data-gen-ebd/tests/test_regrade.py
amos 3e19424391 feat: regrade frozen candidates and report rerun consistency
rollout.regrade checks the original task, image and candidate digests,
grades the frozen candidate again in a fresh grader container and records
whether the verdict matches. rollout.report counts confirmed regrades under
the batch's regrade/ directory.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 21:52:29 +08:00

65 lines
3.0 KiB
Python

import json
import unittest
from unittest.mock import patch
import test_core as core_fixture
from rollout import core, regrade
IMAGE = "sha256:" + "a" * 64
class RegradeTests(unittest.TestCase):
def setUp(self):
helper = core_fixture.CoreTests("runTest")
helper.setUp()
self.addCleanup(helper.doCleanups)
self.root, self.task = helper.root, helper.task
calibration = helper.mock_calibration()
self.controller = self.root / "pi-1"
self.controller.mkdir()
def docker(attempt, image_id, argv, mounts, prefix, timeout, *args, **kwargs):
attempt.joinpath("workspace/decoder.c").write_text("gold")
return dict(returncode=0, termination="exit", cleanup="confirmed")
def grade(task, attempt, candidate, prefix, image_id):
ok = candidate.joinpath("decoder.c").read_text() == "gold"
return dict(verdict=dict(status="PASS" if ok else "FAIL", checks=1),
process=dict(returncode=0 if ok else 1, termination="exit", cleanup="confirmed"))
self.grade = grade
with patch.object(core, "_image", return_value=IMAGE), patch.object(core, "_docker", side_effect=docker), \
patch.object(core, "_grade", side_effect=grade):
attempt = core.run(self.task, self.controller, "tag", ["agent"], "fake@1", calibration)
self.attempt = attempt
self.controller.joinpath("receipt.json").write_text(json.dumps(dict(attempt=str(attempt))))
def run_regrade(self, grade=None):
with patch.object(core, "_image", return_value=IMAGE), patch.object(core, "_grade", side_effect=grade or self.grade):
target = regrade.regrade(self.controller, self.task, "tag", self.root / "regrade")
return json.loads(target.joinpath("receipt.json").read_text())
def test_fresh_grade_is_compared_with_the_original_verdict(self):
receipt = self.run_regrade()
self.assertEqual((receipt["original_verdict"], receipt["task_verdict"], receipt["consistent"]),
("PASS", "PASS", True))
self.assertEqual((receipt["controller"], receipt["cleanup"]), ("pi-1", "confirmed"))
def flaky(task, attempt, candidate, prefix, image_id):
return dict(verdict=dict(status="FAIL", checks=1),
process=dict(returncode=1, termination="exit", cleanup="confirmed"))
self.assertIs(self.run_regrade(flaky)["consistent"], False)
def test_changed_candidate_or_identity_is_rejected(self):
self.attempt.joinpath("candidate/decoder.c").write_text("edited")
with self.assertRaises(ValueError):
self.run_regrade()
self.attempt.joinpath("candidate/decoder.c").write_text("gold")
with patch.object(core, "_image", return_value="sha256:" + "b" * 64):
with self.assertRaises(ValueError):
regrade.regrade(self.controller, self.task, "tag", self.root / "regrade")
if __name__ == "__main__":
unittest.main()