data-gen-ebd/rollout/regrade.py
amos 3e19424391 feat: regrade frozen candidates and report rerun consistency
rollout.regrade checks the original task, image and candidate digests,
grades the frozen candidate again in a fresh grader container and records
whether the verdict matches. rollout.report counts confirmed regrades under
the batch's regrade/ directory.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 21:52:29 +08:00

60 lines
2.8 KiB
Python

"""Grade one frozen candidate again in a fresh container and compare with the original verdict."""
import argparse
import json
from pathlib import Path
import sys
from . import core
def regrade(controller, task_path, image, output_root):
controller = core._path(controller)
task = core.load_task(task_path)
control = json.loads(core._read(controller, "receipt.json"))
attempt = core._path(control["attempt"])
if attempt.parent != controller:
raise ValueError("attempt must belong to the controller")
original = json.loads(core._read(attempt, "receipt.json"))
image_id = core._image(image)
if original.get("task_digest") != task["_digest"] or original.get("image_id") != image_id:
raise ValueError("regrade needs the original task and image identity")
files = {name: core._read(attempt / "candidate", name) for name in task["candidate_files"]}
if core._digest(files) != original.get("candidate_digest"):
raise ValueError("frozen candidate changed")
target = core._attempt(task, output_root, "regrade")
receipt = core._receipt(task, "regrade", image_id)
receipt.update(controller=controller.name, candidate_digest=original["candidate_digest"],
original_verdict=original["task_verdict"], consistent=None)
with core._execution(target, receipt):
candidate = target / "candidate"
core._write_tree(candidate, files, 0o644, 0o755)
receipt["cleanup"] = "unknown"
grade = core._grade(task, target, candidate, "grade", image_id)
receipt.update(status=grade["verdict"]["status"], task_verdict=grade["verdict"]["status"], grade=grade,
cleanup=grade["process"]["cleanup"],
consistent=grade["verdict"]["status"] == original["task_verdict"])
return target
def main(argv=None):
parser = argparse.ArgumentParser(description="Regrade a frozen candidate in a fresh grader container")
parser.add_argument("controller", type=Path)
parser.add_argument("--task", type=Path, required=True)
parser.add_argument("--image", required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args(argv)
try:
target = regrade(args.controller, args.task, args.image, args.output)
receipt = json.loads((target / "receipt.json").read_text())
print(json.dumps({key: receipt.get(key) for key in ("controller", "original_verdict", "task_verdict",
"consistent", "cleanup")}))
return 0 if receipt.get("consistent") else 1
except (ValueError, RuntimeError, OSError, KeyError) as error:
print("regrade rejected: " + str(error), file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())