rollout.regrade checks the original task, image and candidate digests, grades the frozen candidate again in a fresh grader container and records whether the verdict matches. rollout.report counts confirmed regrades under the batch's regrade/ directory. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
60 lines
2.8 KiB
Python
60 lines
2.8 KiB
Python
"""Grade one frozen candidate again in a fresh container and compare with the original verdict."""
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
from . import core
|
|
|
|
|
|
def regrade(controller, task_path, image, output_root):
|
|
controller = core._path(controller)
|
|
task = core.load_task(task_path)
|
|
control = json.loads(core._read(controller, "receipt.json"))
|
|
attempt = core._path(control["attempt"])
|
|
if attempt.parent != controller:
|
|
raise ValueError("attempt must belong to the controller")
|
|
original = json.loads(core._read(attempt, "receipt.json"))
|
|
image_id = core._image(image)
|
|
if original.get("task_digest") != task["_digest"] or original.get("image_id") != image_id:
|
|
raise ValueError("regrade needs the original task and image identity")
|
|
files = {name: core._read(attempt / "candidate", name) for name in task["candidate_files"]}
|
|
if core._digest(files) != original.get("candidate_digest"):
|
|
raise ValueError("frozen candidate changed")
|
|
target = core._attempt(task, output_root, "regrade")
|
|
receipt = core._receipt(task, "regrade", image_id)
|
|
receipt.update(controller=controller.name, candidate_digest=original["candidate_digest"],
|
|
original_verdict=original["task_verdict"], consistent=None)
|
|
with core._execution(target, receipt):
|
|
candidate = target / "candidate"
|
|
core._write_tree(candidate, files, 0o644, 0o755)
|
|
receipt["cleanup"] = "unknown"
|
|
grade = core._grade(task, target, candidate, "grade", image_id)
|
|
receipt.update(status=grade["verdict"]["status"], task_verdict=grade["verdict"]["status"], grade=grade,
|
|
cleanup=grade["process"]["cleanup"],
|
|
consistent=grade["verdict"]["status"] == original["task_verdict"])
|
|
return target
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description="Regrade a frozen candidate in a fresh grader container")
|
|
parser.add_argument("controller", type=Path)
|
|
parser.add_argument("--task", type=Path, required=True)
|
|
parser.add_argument("--image", required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
target = regrade(args.controller, args.task, args.image, args.output)
|
|
receipt = json.loads((target / "receipt.json").read_text())
|
|
print(json.dumps({key: receipt.get(key) for key in ("controller", "original_verdict", "task_verdict",
|
|
"consistent", "cleanup")}))
|
|
return 0 if receipt.get("consistent") else 1
|
|
except (ValueError, RuntimeError, OSError, KeyError) as error:
|
|
print("regrade rejected: " + str(error), file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|