Batch 6 lost 26 of 103 attempts to Qwen requests that sent their preamble and then stalled; Pi's own Node fetch gave up after 300 s and reported "terminated". - model_proxy: hold the SSE preamble until the first generation event, retry a stall (180 s) within the existing retry budget, and send SSE keepalive comments so Pi never times out first; record first_event_seconds. - report: name one exclusion reason per incomplete attempt and compute the JOBS lines still owed, with a per-line resample budget. - batch: run a JOBS file to completion, resume from receipts, resample infrastructure failures, pause after three environment failures in a row, forward SIGTERM to running controllers, and refuse a second runner. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
146 lines
10 KiB
Python
146 lines
10 KiB
Python
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
import tempfile
|
|
import unittest
|
|
|
|
from rollout import core, report
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
class ReportTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name).resolve()
|
|
self.batch, self.tasks = self.root / "batch", self.root / "tasks"
|
|
for task, family in (("a", "abi"), ("b", "firmware")):
|
|
self.tasks.joinpath(task).mkdir(parents=True)
|
|
self.tasks.joinpath(task, "task.json").write_text(json.dumps(dict(id=task, family=family)))
|
|
|
|
def controller(self, name, task, status, evidence="VERIFIED", cleanup="confirmed", tokens=(10, 5), end=None,
|
|
broker=None, grade=None, calibration=None, **fields):
|
|
control = self.batch / name
|
|
run = control / "run-x"
|
|
(run / "trace").mkdir(parents=True)
|
|
if broker is not None:
|
|
control.joinpath("broker.jsonl").write_text("".join(json.dumps(event) + "\n" for event in broker))
|
|
if grade is not None:
|
|
run.joinpath("receipt.json").write_text(json.dumps(dict(grade=dict(process=grade))))
|
|
if calibration is not None:
|
|
control.joinpath("calibration-x").mkdir()
|
|
control.joinpath("calibration-x/receipt.json").write_text(json.dumps(calibration))
|
|
run.joinpath("events.jsonl").write_text("".join(json.dumps(dict(event=kind, time=time)) + "\n" for kind, time in
|
|
(("created", 100.0), ("graded", 150.0), ("finished", 160.5))))
|
|
trace = [dict(type="message_end", message=dict(role="assistant", usage=dict(input=tokens[0], output=tokens[1],
|
|
cacheRead=1, totalTokens=sum(tokens) + 1))),
|
|
dict(type="message_end", message=dict(role="toolResult")),
|
|
dict(type="tool_execution_end"), dict(type="tool_execution_end"),
|
|
end or dict(type="session_end", final_stop_reason="stop", cost_available=False)]
|
|
run.joinpath("trace/native.jsonl").write_text("".join(json.dumps(event) + "\n" for event in trace))
|
|
receipt = dict(task_id=task, task_digest="d", model_id="glm-5.3", status=status, task_verdict=status,
|
|
cleanup=cleanup, agent_exit=0, agent_termination="exit", attempt=str(run),
|
|
pi_evidence=dict(status=evidence, model_requests=3))
|
|
receipt.update(fields)
|
|
control.joinpath("receipt.json").write_text(json.dumps(receipt))
|
|
|
|
def test_only_verified_cleaned_task_outcomes_count_as_complete(self):
|
|
self.controller("pi-1", "a", "PASS")
|
|
self.controller("pi-2", "a", "FAIL")
|
|
self.batch.joinpath("pi-2/run-x/grade.stderr").write_text("Traceback ...\nAttributeError: no attribute 'process'\n\n")
|
|
self.controller("pi-6", "a", "FAIL")
|
|
self.batch.joinpath("pi-6/run-x/receipt.json").write_text(json.dumps(dict(grade=dict(verdict=dict(
|
|
status="FAIL", checks=1, reason="FAIL: AssertionError case 4 mismatch")))))
|
|
self.controller("pi-3", "b", "UNVERIFIED", evidence="UNVERIFIED")
|
|
self.controller("pi-4", "b", "INFRA_ERROR", cleanup="unknown")
|
|
self.batch.joinpath("calibration-x").mkdir()
|
|
self.batch.joinpath("pi-5-running").mkdir()
|
|
for name, controller, consistent, cleanup in (("regrade-1", "pi-1", True, "confirmed"),
|
|
("regrade-2", "pi-2", False, "confirmed"),
|
|
("regrade-3", "pi-2", True, "unknown")):
|
|
self.batch.joinpath("regrade", name).mkdir(parents=True)
|
|
self.batch.joinpath("regrade", name, "receipt.json").write_text(json.dumps(dict(
|
|
controller=controller, consistent=consistent, cleanup=cleanup)))
|
|
result = report.report(self.batch, self.tasks)
|
|
self.assertEqual((result["regraded"], result["regrade_consistent"]), (2, 1))
|
|
self.assertEqual((result["attempts"], result["complete_trajectories"], result["passed"], result["failed"]),
|
|
(6, 3, 1, 2))
|
|
self.assertEqual(result["excluded"], {"UNVERIFIED/UNVERIFIED": 1, "INFRA_ERROR/VERIFIED": 1,
|
|
"NO_RECEIPT/None": 1})
|
|
self.assertEqual(result["exclusions"], {"evidence": 1, "cleanup": 1, "no_receipt": 1})
|
|
self.assertEqual(result["tasks"], {"a": {"PASS": 1, "FAIL": 2}, "b": {"incomplete": 2}, "unknown": {"incomplete": 1}})
|
|
self.assertEqual(result["families"]["abi"], dict(tasks=1, attempts=3, complete=3, passed=1))
|
|
self.assertEqual(result["tokens"], dict(input=50, output=25, cacheRead=5, totalTokens=80))
|
|
self.assertEqual(result["tool_calls"]["p50"], 2)
|
|
self.assertEqual(result["seconds"]["max"], 60.5)
|
|
self.assertEqual(result["cost"], "unknown")
|
|
details = {row["controller"]: row["fail_detail"] for row in result["rows"]}
|
|
self.assertEqual((details["pi-2"], details["pi-6"]), ("AttributeError: no attribute 'process'",
|
|
"FAIL: AssertionError case 4 mismatch"))
|
|
output = self.root / "report.json"
|
|
self.assertEqual(report.main([str(self.batch), "--tasks", str(self.tasks), "--output", str(output)]), 0)
|
|
self.assertEqual(report.main([str(self.batch), "--tasks", str(self.tasks), "--output", str(output)]), 2)
|
|
|
|
def test_exclusion_names_the_first_recorded_cause(self):
|
|
cut = dict(type="session_end", complete=False, error="TRACE_LIMIT_EXCEEDED")
|
|
done = dict(status=200, responses_terminal="response.completed", responses_stream_finished=True,
|
|
response_status="completed")
|
|
infra = dict(status="INFRA_ERROR", cleanup="unknown")
|
|
self.controller("pi-ok", "a", "PASS", broker=[dict(status=200)])
|
|
self.controller("pi-interrupted", "a", agent_termination="interrupted", **infra)
|
|
self.controller("pi-calibration-interrupted", "a", "INFRA_ERROR", error_phase="calibration", attempt=None,
|
|
calibration=dict(status="UNVERIFIED", termination="interrupted"))
|
|
self.controller("pi-timeout", "a", agent_termination="timeout", **infra)
|
|
self.controller("pi-disk", "a", agent_termination="attempt_size", **infra)
|
|
self.controller("pi-grade-timeout", "a", grade=dict(termination="timeout", returncode=None), **infra)
|
|
self.controller("pi-cleanup", "a", **infra)
|
|
self.controller("pi-trace", "a", "UNVERIFIED", "UNVERIFIED", end=cut, agent_exit=1, broker=[done])
|
|
self.controller("pi-http", "a", "INFRA_ERROR", "UNVERIFIED", agent_exit=1, broker=[done, dict(status=400)])
|
|
self.controller("pi-cut", "a", "UNVERIFIED", "UNVERIFIED", agent_exit=1,
|
|
broker=[done, dict(done, responses_terminal=None, response_status=None)])
|
|
self.controller("pi-length", "a", "UNVERIFIED", "UNVERIFIED", agent_exit=1,
|
|
broker=[done, dict(done, response_status="incomplete")])
|
|
self.controller("pi-length-native", "a", "UNVERIFIED", "UNVERIFIED", agent_exit=1, broker=[dict(status=200)],
|
|
end=dict(type="session_end", complete=False, final_stop_reason="length"))
|
|
self.controller("pi-calibration", "a", "INFRA_ERROR", error_phase="calibration", attempt=None)
|
|
self.controller("pi-network", "a", "INFRA_ERROR", error_phase="network", attempt=None)
|
|
# A Chat Completions stream carries no terminal fields; an error stop there stays an evidence gap.
|
|
self.controller("pi-stalled", "a", "UNVERIFIED", "UNVERIFIED", agent_exit=1,
|
|
broker=[dict(status=200, upstream_statuses=["stalled"] * 3)])
|
|
self.controller("pi-evidence", "a", "UNVERIFIED", "UNVERIFIED", agent_exit=1, broker=[dict(status=200)],
|
|
end=dict(type="session_end", complete=False, final_stop_reason="error"))
|
|
self.batch.joinpath("pi-killed").mkdir()
|
|
rows = {row["controller"]: row["exclusion"] for row in report.report(self.batch, self.tasks)["rows"]}
|
|
self.assertEqual(rows, {
|
|
"pi-ok": None, "pi-interrupted": "interrupted", "pi-calibration-interrupted": "interrupted",
|
|
"pi-timeout": "timeout", "pi-disk": "disk_limit", "pi-grade-timeout": "grade_timeout",
|
|
"pi-cleanup": "cleanup", "pi-trace": "trace_limit", "pi-http": "model_http", "pi-cut": "stream_cut",
|
|
"pi-length": "output_limit", "pi-length-native": "output_limit", "pi-calibration": "calibration",
|
|
"pi-network": "infra", "pi-stalled": "stream_cut", "pi-evidence": "evidence", "pi-killed": "no_receipt"})
|
|
|
|
def test_pending_reruns_only_attempts_that_did_not_finish(self):
|
|
tasks = self.root / "pool"
|
|
shutil.copytree(ROOT / "tasks/frame-decoder", tasks / "frame-decoder")
|
|
digest = core.load_task(tasks / "frame-decoder")["_digest"]
|
|
mine = dict(task_digest=digest)
|
|
self.controller("pi-1", "frame-decoder", "PASS", **mine)
|
|
self.controller("pi-2", "frame-decoder", "INFRA_ERROR", "UNVERIFIED", agent_exit=1, broker=[dict(status=429)],
|
|
**mine)
|
|
self.controller("pi-3", "frame-decoder", "INFRA_ERROR", cleanup="unknown", agent_termination="interrupted",
|
|
**mine)
|
|
self.controller("pi-4", "frame-decoder", "PASS", task_digest="older-version")
|
|
self.controller("pi-5", "frame-decoder", "INFRA_ERROR", error_phase="network", attempt=None, **mine)
|
|
self.batch.joinpath("pi-6").mkdir()
|
|
lines = ["frame-decoder /calibration/{}".format(index) for index in range(1, 5)]
|
|
self.assertEqual(report.pending(self.batch, tasks, "\n".join(lines[:2] + [""] + lines[2:]) + "\n"),
|
|
lines[2:])
|
|
# One retry per line lets the 429 attempt be resampled; an attempt in flight takes one line back.
|
|
self.assertEqual(report.pending(self.batch, tasks, "\n".join(lines), retries=1), lines[1:])
|
|
self.assertEqual(report.pending(self.batch, tasks, "\n".join(lines), 1, ["frame-decoder"]), lines[2:])
|
|
self.assertEqual(report.pending(self.batch, tasks, lines[0], retries=5), [])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|