rollout.regrade checks the original task, image and candidate digests, grades the frozen candidate again in a fresh grader container and records whether the verdict matches. rollout.report counts confirmed regrades under the batch's regrade/ directory. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
65 lines
3.0 KiB
Python
65 lines
3.0 KiB
Python
import json
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import test_core as core_fixture
|
|
from rollout import core, regrade
|
|
|
|
IMAGE = "sha256:" + "a" * 64
|
|
|
|
|
|
class RegradeTests(unittest.TestCase):
|
|
def setUp(self):
|
|
helper = core_fixture.CoreTests("runTest")
|
|
helper.setUp()
|
|
self.addCleanup(helper.doCleanups)
|
|
self.root, self.task = helper.root, helper.task
|
|
calibration = helper.mock_calibration()
|
|
self.controller = self.root / "pi-1"
|
|
self.controller.mkdir()
|
|
|
|
def docker(attempt, image_id, argv, mounts, prefix, timeout, *args, **kwargs):
|
|
attempt.joinpath("workspace/decoder.c").write_text("gold")
|
|
return dict(returncode=0, termination="exit", cleanup="confirmed")
|
|
|
|
def grade(task, attempt, candidate, prefix, image_id):
|
|
ok = candidate.joinpath("decoder.c").read_text() == "gold"
|
|
return dict(verdict=dict(status="PASS" if ok else "FAIL", checks=1),
|
|
process=dict(returncode=0 if ok else 1, termination="exit", cleanup="confirmed"))
|
|
|
|
self.grade = grade
|
|
with patch.object(core, "_image", return_value=IMAGE), patch.object(core, "_docker", side_effect=docker), \
|
|
patch.object(core, "_grade", side_effect=grade):
|
|
attempt = core.run(self.task, self.controller, "tag", ["agent"], "fake@1", calibration)
|
|
self.attempt = attempt
|
|
self.controller.joinpath("receipt.json").write_text(json.dumps(dict(attempt=str(attempt))))
|
|
|
|
def run_regrade(self, grade=None):
|
|
with patch.object(core, "_image", return_value=IMAGE), patch.object(core, "_grade", side_effect=grade or self.grade):
|
|
target = regrade.regrade(self.controller, self.task, "tag", self.root / "regrade")
|
|
return json.loads(target.joinpath("receipt.json").read_text())
|
|
|
|
def test_fresh_grade_is_compared_with_the_original_verdict(self):
|
|
receipt = self.run_regrade()
|
|
self.assertEqual((receipt["original_verdict"], receipt["task_verdict"], receipt["consistent"]),
|
|
("PASS", "PASS", True))
|
|
self.assertEqual((receipt["controller"], receipt["cleanup"]), ("pi-1", "confirmed"))
|
|
|
|
def flaky(task, attempt, candidate, prefix, image_id):
|
|
return dict(verdict=dict(status="FAIL", checks=1),
|
|
process=dict(returncode=1, termination="exit", cleanup="confirmed"))
|
|
self.assertIs(self.run_regrade(flaky)["consistent"], False)
|
|
|
|
def test_changed_candidate_or_identity_is_rejected(self):
|
|
self.attempt.joinpath("candidate/decoder.c").write_text("edited")
|
|
with self.assertRaises(ValueError):
|
|
self.run_regrade()
|
|
self.attempt.joinpath("candidate/decoder.c").write_text("gold")
|
|
with patch.object(core, "_image", return_value="sha256:" + "b" * 64):
|
|
with self.assertRaises(ValueError):
|
|
regrade.regrade(self.controller, self.task, "tag", self.root / "regrade")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|