fix: report counts controllers without a receipt instead of failing
A running or killed controller has no receipt yet; it is now an attempt with status NO_RECEIPT and listed among the exclusions. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
d5c6bb2773
commit
f20316fa4d
@ -16,7 +16,10 @@ def _jsonl(root, name, limit=core.MAX_FILE_BYTES):
|
||||
|
||||
def attempt(controller):
|
||||
"""One row per controller, read only from host-held receipts and native output."""
|
||||
control = json.loads(core._read(controller, "receipt.json"))
|
||||
try:
|
||||
control = json.loads(core._read(controller, "receipt.json"))
|
||||
except ValueError:
|
||||
control = dict(status="NO_RECEIPT", task_id="unknown") # Still running, or the controller died before writing.
|
||||
evidence = control.get("pi_evidence", {})
|
||||
row = dict(controller=controller.name, task_id=control.get("task_id"), task_digest=control.get("task_digest"),
|
||||
model_id=control.get("model_id", "glm-5.3"), status=control.get("status"),
|
||||
|
||||
@ -39,6 +39,7 @@ class ReportTests(unittest.TestCase):
|
||||
self.controller("pi-3", "b", "UNVERIFIED", evidence="UNVERIFIED")
|
||||
self.controller("pi-4", "b", "INFRA_ERROR", cleanup="unknown")
|
||||
self.batch.joinpath("calibration-x").mkdir()
|
||||
self.batch.joinpath("pi-5-running").mkdir()
|
||||
for name, controller, consistent, cleanup in (("regrade-1", "pi-1", True, "confirmed"),
|
||||
("regrade-2", "pi-2", False, "confirmed"),
|
||||
("regrade-3", "pi-2", True, "unknown")):
|
||||
@ -48,9 +49,10 @@ class ReportTests(unittest.TestCase):
|
||||
result = report.report(self.batch, self.tasks)
|
||||
self.assertEqual((result["regraded"], result["regrade_consistent"]), (2, 1))
|
||||
self.assertEqual((result["attempts"], result["complete_trajectories"], result["passed"], result["failed"]),
|
||||
(4, 2, 1, 1))
|
||||
self.assertEqual(result["excluded"], {"UNVERIFIED/UNVERIFIED": 1, "INFRA_ERROR/VERIFIED": 1})
|
||||
self.assertEqual(result["tasks"], {"a": {"PASS": 1, "FAIL": 1}, "b": {"incomplete": 2}})
|
||||
(5, 2, 1, 1))
|
||||
self.assertEqual(result["excluded"], {"UNVERIFIED/UNVERIFIED": 1, "INFRA_ERROR/VERIFIED": 1,
|
||||
"NO_RECEIPT/None": 1})
|
||||
self.assertEqual(result["tasks"], {"a": {"PASS": 1, "FAIL": 1}, "b": {"incomplete": 2}, "unknown": {"incomplete": 1}})
|
||||
self.assertEqual(result["families"]["abi"], dict(tasks=1, attempts=2, complete=2, passed=1))
|
||||
self.assertEqual(result["tokens"], dict(input=40, output=20, cacheRead=4, totalTokens=64))
|
||||
self.assertEqual(result["tool_calls"]["p50"], 2)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user