data-gen-ebd/tests/test_report.py
amos 5938cbd86f Keep a batch running when a grader log exceeds 1 MiB
report.py read grade.stderr with the default 1 MiB cap only to keep the
last line of a FAIL. A SPIFFS grader logged 1.1 MiB of proof data there,
the read raised ValueError, and the whole batch was rejected mid-run,
interrupting the attempts still in flight. Read it with the trace cap
instead, and fall back to the verdict reason if it is larger still.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 03:57:20 +08:00

155 lines
11 KiB
Python

import json
from pathlib import Path
import shutil
import tempfile
import unittest
from rollout import core, report
ROOT = Path(__file__).resolve().parents[1]
class ReportTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name).resolve()
self.batch, self.tasks = self.root / "batch", self.root / "tasks"
for task, family in (("a", "abi"), ("b", "firmware")):
self.tasks.joinpath(task).mkdir(parents=True)
self.tasks.joinpath(task, "task.json").write_text(json.dumps(dict(id=task, family=family)))
def controller(self, name, task, status, evidence="VERIFIED", cleanup="confirmed", tokens=(10, 5), end=None,
broker=None, grade=None, calibration=None, **fields):
control = self.batch / name
run = control / "run-x"
(run / "trace").mkdir(parents=True)
if broker is not None:
control.joinpath("broker.jsonl").write_text("".join(json.dumps(event) + "\n" for event in broker))
if grade is not None:
run.joinpath("receipt.json").write_text(json.dumps(dict(grade=dict(process=grade))))
if calibration is not None:
control.joinpath("calibration-x").mkdir()
control.joinpath("calibration-x/receipt.json").write_text(json.dumps(calibration))
run.joinpath("events.jsonl").write_text("".join(json.dumps(dict(event=kind, time=time)) + "\n" for kind, time in
(("created", 100.0), ("graded", 150.0), ("finished", 160.5))))
trace = [dict(type="message_end", message=dict(role="assistant", usage=dict(input=tokens[0], output=tokens[1],
cacheRead=1, totalTokens=sum(tokens) + 1))),
dict(type="message_end", message=dict(role="toolResult")),
dict(type="tool_execution_end"), dict(type="tool_execution_end"),
end or dict(type="session_end", final_stop_reason="stop", cost_available=False)]
run.joinpath("trace/native.jsonl").write_text("".join(json.dumps(event) + "\n" for event in trace))
receipt = dict(task_id=task, task_digest="d", model_id="glm-5.3", status=status, task_verdict=status,
cleanup=cleanup, agent_exit=0, agent_termination="exit", attempt=str(run),
pi_evidence=dict(status=evidence, model_requests=3))
receipt.update(fields)
control.joinpath("receipt.json").write_text(json.dumps(receipt))
def test_only_verified_cleaned_task_outcomes_count_as_complete(self):
self.controller("pi-1", "a", "PASS")
self.controller("pi-2", "a", "FAIL")
self.batch.joinpath("pi-2/run-x/grade.stderr").write_text("Traceback ...\nAttributeError: no attribute 'process'\n\n")
self.controller("pi-6", "a", "FAIL")
self.batch.joinpath("pi-6/run-x/receipt.json").write_text(json.dumps(dict(grade=dict(verdict=dict(
status="FAIL", checks=1, reason="FAIL: AssertionError case 4 mismatch")))))
self.controller("pi-3", "b", "UNVERIFIED", evidence="UNVERIFIED")
self.controller("pi-4", "b", "INFRA_ERROR", cleanup="unknown")
self.batch.joinpath("calibration-x").mkdir()
self.batch.joinpath("pi-5-running").mkdir()
for name, controller, consistent, cleanup in (("regrade-1", "pi-1", True, "confirmed"),
("regrade-2", "pi-2", False, "confirmed"),
("regrade-3", "pi-2", True, "unknown")):
self.batch.joinpath("regrade", name).mkdir(parents=True)
self.batch.joinpath("regrade", name, "receipt.json").write_text(json.dumps(dict(
controller=controller, consistent=consistent, cleanup=cleanup)))
result = report.report(self.batch, self.tasks)
self.assertEqual((result["regraded"], result["regrade_consistent"]), (2, 1))
self.assertEqual((result["attempts"], result["complete_trajectories"], result["passed"], result["failed"]),
(6, 3, 1, 2))
self.assertEqual(result["excluded"], {"UNVERIFIED/UNVERIFIED": 1, "INFRA_ERROR/VERIFIED": 1,
"NO_RECEIPT/None": 1})
self.assertEqual(result["exclusions"], {"evidence": 1, "cleanup": 1, "no_receipt": 1})
self.assertEqual(result["tasks"], {"a": {"PASS": 1, "FAIL": 2}, "b": {"incomplete": 2}, "unknown": {"incomplete": 1}})
self.assertEqual(result["families"]["abi"], dict(tasks=1, attempts=3, complete=3, passed=1))
self.assertEqual(result["tokens"], dict(input=50, output=25, cacheRead=5, totalTokens=80))
self.assertEqual(result["tool_calls"]["p50"], 2)
self.assertEqual(result["seconds"]["max"], 60.5)
self.assertEqual(result["cost"], "unknown")
details = {row["controller"]: row["fail_detail"] for row in result["rows"]}
self.assertEqual((details["pi-2"], details["pi-6"]), ("AttributeError: no attribute 'process'",
"FAIL: AssertionError case 4 mismatch"))
output = self.root / "report.json"
self.assertEqual(report.main([str(self.batch), "--tasks", str(self.tasks), "--output", str(output)]), 0)
self.assertEqual(report.main([str(self.batch), "--tasks", str(self.tasks), "--output", str(output)]), 2)
def test_large_grade_log_keeps_its_last_line(self):
self.controller("pi-1", "a", "FAIL")
self.batch.joinpath("pi-1/run-x/grade.stderr").write_text("PROOF " + "x" * (2 * 1024 * 1024) + "\nFAIL: case 3\n")
row = report.report(self.batch, self.tasks)["rows"][0]
self.assertEqual((row["task_verdict"], row["fail_detail"]), ("FAIL", "FAIL: case 3"))
def test_exclusion_names_the_first_recorded_cause(self):
cut = dict(type="session_end", complete=False, error="TRACE_LIMIT_EXCEEDED")
done = dict(status=200, responses_terminal="response.completed", responses_stream_finished=True,
response_status="completed")
infra = dict(status="INFRA_ERROR", cleanup="unknown")
self.controller("pi-ok", "a", "PASS", broker=[dict(status=200)])
self.controller("pi-interrupted", "a", agent_termination="interrupted", **infra)
self.controller("pi-calibration-interrupted", "a", "INFRA_ERROR", error_phase="calibration", attempt=None,
calibration=dict(status="UNVERIFIED", termination="interrupted"))
self.controller("pi-timeout", "a", agent_termination="timeout", **infra)
self.controller("pi-disk", "a", agent_termination="attempt_size", **infra)
self.controller("pi-disk-low", "a", agent_termination="disk_low", **infra)
self.controller("pi-grade-timeout", "a", grade=dict(termination="timeout", returncode=None), **infra)
self.controller("pi-cleanup", "a", **infra)
self.controller("pi-trace", "a", "UNVERIFIED", "UNVERIFIED", end=cut, agent_exit=1, broker=[done])
self.controller("pi-http", "a", "INFRA_ERROR", "UNVERIFIED", agent_exit=1, broker=[done, dict(status=400)])
self.controller("pi-context", "a", "INFRA_ERROR", "UNVERIFIED", agent_exit=1,
broker=[done, dict(status=400, upstream_statuses=[400], request_bytes=650_000)])
self.controller("pi-cut", "a", "UNVERIFIED", "UNVERIFIED", agent_exit=1,
broker=[done, dict(done, responses_terminal=None, response_status=None)])
self.controller("pi-length", "a", "UNVERIFIED", "UNVERIFIED", agent_exit=1,
broker=[done, dict(done, response_status="incomplete")])
self.controller("pi-length-native", "a", "UNVERIFIED", "UNVERIFIED", agent_exit=1, broker=[dict(status=200)],
end=dict(type="session_end", complete=False, final_stop_reason="length"))
self.controller("pi-calibration", "a", "INFRA_ERROR", error_phase="calibration", attempt=None)
self.controller("pi-network", "a", "INFRA_ERROR", error_phase="network", attempt=None)
# A Chat Completions stream carries no terminal fields; an error stop there stays an evidence gap.
self.controller("pi-stalled", "a", "UNVERIFIED", "UNVERIFIED", agent_exit=1,
broker=[dict(status=200, upstream_statuses=["stalled"] * 3)])
self.controller("pi-evidence", "a", "UNVERIFIED", "UNVERIFIED", agent_exit=1, broker=[dict(status=200)],
end=dict(type="session_end", complete=False, final_stop_reason="error"))
self.batch.joinpath("pi-killed").mkdir()
rows = {row["controller"]: row["exclusion"] for row in report.report(self.batch, self.tasks)["rows"]}
self.assertEqual(rows, {
"pi-ok": None, "pi-interrupted": "interrupted", "pi-calibration-interrupted": "interrupted",
"pi-timeout": "timeout", "pi-disk": "disk_limit", "pi-disk-low": "disk_low", "pi-grade-timeout": "grade_timeout",
"pi-cleanup": "cleanup", "pi-trace": "trace_limit", "pi-http": "model_http", "pi-context": "context_limit", "pi-cut": "stream_cut",
"pi-length": "output_limit", "pi-length-native": "output_limit", "pi-calibration": "calibration",
"pi-network": "infra", "pi-stalled": "stream_cut", "pi-evidence": "evidence", "pi-killed": "no_receipt"})
def test_pending_reruns_only_attempts_that_did_not_finish(self):
tasks = self.root / "pool"
shutil.copytree(ROOT / "tasks/frame-decoder", tasks / "frame-decoder")
digest = core.load_task(tasks / "frame-decoder")["_digest"]
mine = dict(task_digest=digest)
self.controller("pi-1", "frame-decoder", "PASS", **mine)
self.controller("pi-2", "frame-decoder", "INFRA_ERROR", "UNVERIFIED", agent_exit=1, broker=[dict(status=429)],
**mine)
self.controller("pi-3", "frame-decoder", "INFRA_ERROR", cleanup="unknown", agent_termination="interrupted",
**mine)
self.controller("pi-4", "frame-decoder", "PASS", task_digest="older-version")
self.controller("pi-5", "frame-decoder", "INFRA_ERROR", error_phase="network", attempt=None, **mine)
self.batch.joinpath("pi-6").mkdir()
lines = ["frame-decoder /calibration/{}".format(index) for index in range(1, 5)]
self.assertEqual(report.pending(self.batch, tasks, "\n".join(lines[:2] + [""] + lines[2:]) + "\n"),
lines[2:])
# One retry per line lets the 429 attempt be resampled; an attempt in flight takes one line back.
self.assertEqual(report.pending(self.batch, tasks, "\n".join(lines), retries=1), lines[1:])
self.assertEqual(report.pending(self.batch, tasks, "\n".join(lines), 1, ["frame-decoder"]), lines[2:])
self.assertEqual(report.pending(self.batch, tasks, lines[0], retries=5), [])
if __name__ == "__main__":
unittest.main()