Output limits were baked into pi.mjs inside the runtime image, so every change (GLM 32000 -> 131072, Qwen 8192 -> 65536) needed a new image and a full 785-trial recalibration, and the Qwen baseline ran on an image the repo could not rebuild. - MODELS holds max_tokens and its wire field per model; pi_run passes --model and --max-tokens to the adapter, which has no defaults left. - Receipts record model_max_tokens and model_sampling=service-default; evidence requires every payload to carry exactly that limit, which would have caught the SDK's silent 32000 cap. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
162 lines
8.8 KiB
Python
162 lines
8.8 KiB
Python
"""Recheck one completed Pi attempt and archive only its explicit evidence files."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import sys
|
|
import tarfile
|
|
|
|
from . import core, pi_run, export
|
|
|
|
|
|
def _object(raw):
|
|
value = json.loads(raw)
|
|
if not isinstance(value, dict):
|
|
raise ValueError("receipt must be a JSON object")
|
|
return value
|
|
|
|
|
|
def _expect(value, expected, label):
|
|
if not isinstance(value, dict) or any(value.get(key) != wanted for key, wanted in expected.items()):
|
|
raise ValueError(label + " does not match the completed attempt")
|
|
|
|
|
|
def archive(controller_path, task_path, output_path):
|
|
controller = core._path(controller_path)
|
|
if not controller.is_dir():
|
|
raise ValueError("controller must be a directory")
|
|
task = core.load_task(task_path)
|
|
controller_raw = core._read(controller, "receipt.json")
|
|
control = _object(controller_raw)
|
|
verdict = control.get("status")
|
|
if verdict not in ("PASS", "FAIL"):
|
|
raise ValueError("only completed PASS/FAIL attempts can be archived")
|
|
image = control.get("image_id")
|
|
if not isinstance(image, str) or not re.fullmatch(r"sha256:[a-f0-9]{64}", image):
|
|
raise ValueError("a fixed Docker image identity is required")
|
|
_expect(control, dict(schema="embedded-rollout.pi-control/1", task_id=task["id"],
|
|
task_digest=task["_digest"], task_verdict=verdict, cleanup="confirmed",
|
|
network_cleanup="confirmed", proxy_cleanup="confirmed", agent_cleanup="confirmed",
|
|
agent_exit=0, agent_termination="exit"), "controller receipt")
|
|
if type(control.get("agent_exit")) is not int:
|
|
raise ValueError("Agent exit code must be an integer")
|
|
_expect(control.get("pi_evidence"), dict(status="VERIFIED"), "native evidence")
|
|
if not isinstance(control.get("attempt"), str) or not control["attempt"]:
|
|
raise ValueError("controller must identify its attempt")
|
|
attempt = core._path(control["attempt"])
|
|
if attempt.parent != controller or not attempt.is_dir() or not attempt.name.startswith("run-"):
|
|
raise ValueError("attempt must be a direct run directory of the controller")
|
|
runs = [core._path(entry) for entry in controller.iterdir() if entry.name.startswith("run-")]
|
|
if runs != [attempt]:
|
|
raise ValueError("controller must contain one unique run directory")
|
|
attempt_raw = core._read(attempt, "receipt.json")
|
|
receipt = _object(attempt_raw)
|
|
identity = core._receipt(task, "run", image)
|
|
_expect(receipt, {key: identity[key] for key in (
|
|
"mode", "task_id", "task_version", "task_digest", "manifest_digest", "verifier_digest", "image_id")},
|
|
"attempt identity")
|
|
_expect(receipt, dict(status=verdict, task_verdict=verdict, cleanup="confirmed",
|
|
agent_exit=0, termination="exit"), "attempt receipt")
|
|
if type(receipt.get("agent_exit")) is not int:
|
|
raise ValueError("Agent exit code must be an integer")
|
|
_expect(receipt.get("agent_process"), dict(returncode=0, termination="exit", cleanup="confirmed"),
|
|
"Agent process")
|
|
calibration_ref = control.get("calibration")
|
|
if not isinstance(calibration_ref, dict) or not isinstance(calibration_ref.get("path"), str):
|
|
raise ValueError("controller must identify its calibration")
|
|
calibration = core._calibration(task, image, calibration_ref["path"])
|
|
if calibration != calibration_ref or calibration != receipt.get("calibration"):
|
|
raise ValueError("calibration identity or receipt digest changed")
|
|
calibration_file = core._path(calibration["path"])
|
|
output = core._path(output_path)
|
|
if not output.name.endswith(".tar.gz"):
|
|
raise ValueError("output must end in .tar.gz")
|
|
for source in (controller, task["_root"], calibration_file.parent):
|
|
if output == source or source in output.parents:
|
|
raise ValueError("archive output must be outside its input directories")
|
|
|
|
files = {"controller/receipt.json": controller_raw, "attempt/receipt.json": attempt_raw}
|
|
for name in ("events.jsonl", "broker.jsonl"):
|
|
files["controller/" + name] = core._read(controller, name,
|
|
core.MAX_TRACE_BYTES if name == "broker.jsonl" else core.MAX_FILE_BYTES)
|
|
candidates = ["candidate/" + name for name in task["candidate_files"]]
|
|
for name in ("events.jsonl", "prompt.md", "trace/native.jsonl", "agent.stdout", "agent.stderr",
|
|
"grade.stdout", "grade.stderr", *candidates):
|
|
limit = core.MAX_TRACE_BYTES if name in ("trace/native.jsonl", "agent.stdout") else core.MAX_FILE_BYTES
|
|
files["attempt/" + name] = core._read(attempt, name, limit)
|
|
for name in ("task.json", "prompt.md", *("template/" + name for name in core._template(task))):
|
|
files["task/" + name] = task["_files"][name]
|
|
files["calibration/receipt.json"] = core._read(calibration_file.parent, calibration_file.name)
|
|
if hashlib.sha256(files["calibration/receipt.json"]).hexdigest() != calibration["receipt_digest"]:
|
|
raise ValueError("calibration changed while collecting evidence")
|
|
if files["attempt/prompt.md"] != files["task/prompt.md"]:
|
|
raise ValueError("attempt prompt differs from the task")
|
|
if core._digest({name: files["attempt/candidate/" + name] for name in task["candidate_files"]}) != receipt.get(
|
|
"candidate_digest"):
|
|
raise ValueError("frozen candidate digest changed")
|
|
grade = receipt.get("grade")
|
|
if not isinstance(grade, dict):
|
|
raise ValueError("independent grade evidence is missing")
|
|
_expect(grade.get("process"), dict(returncode=0 if verdict == "PASS" else 1,
|
|
termination="exit", cleanup="confirmed"), "grade process")
|
|
if type(grade["process"].get("returncode")) is not int:
|
|
raise ValueError("grade exit code must be an integer")
|
|
checked = core._checker(grade["process"], attempt / "grade.stdout")
|
|
if checked != grade.get("verdict") or checked.get("status") != verdict:
|
|
raise ValueError("grade stdout differs from its recorded verdict")
|
|
model = control.get("model_id", pi_run.MODEL)
|
|
if model not in pi_run.MODEL_IDS:
|
|
raise ValueError("unsupported archived model identity")
|
|
observed = pi_run.evidence(attempt, controller / "broker.jsonl", model, control.get("model_max_tokens"))
|
|
if observed.get("status") != "VERIFIED" or observed != control["pi_evidence"]:
|
|
raise ValueError("native model and tool evidence no longer verifies")
|
|
if observed.get("trace_sha256") != hashlib.sha256(files["attempt/trace/native.jsonl"]).hexdigest():
|
|
raise ValueError("native trace changed while collecting evidence")
|
|
# Recheck the captured evidence after helpers read the same host files.
|
|
if core._read(controller, "broker.jsonl", core.MAX_TRACE_BYTES) != files["controller/broker.jsonl"]:
|
|
raise ValueError("broker evidence changed while collecting evidence")
|
|
for name, limit in (("agent.stdout", core.MAX_TRACE_BYTES), ("grade.stdout", core.MAX_FILE_BYTES)):
|
|
if core._read(attempt, name, limit) != files["attempt/" + name]:
|
|
raise ValueError("host evidence changed while collecting evidence")
|
|
|
|
row = export.record(task)
|
|
files["dataset.jsonl"] = (json.dumps(row, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n").encode("utf-8")
|
|
|
|
manifest = dict(schema="embedded-rollout.archive/1", task_id=task["id"],
|
|
model_id=model, model_api=pi_run.MODELS[model]["api"],
|
|
dataset_layout="ultradata-five-fields", dataset_call_type=row["ground_truth"]["call_type"],
|
|
task_version=task["version"], task_digest=task["_digest"], image_id=image,
|
|
task_verdict=verdict, pi_evidence="VERIFIED", cleanup="confirmed",
|
|
training_disposition="NOT_TRAIN_READY", files={name: dict(bytes=len(data),
|
|
sha256=hashlib.sha256(data).hexdigest()) for name, data in sorted(files.items())})
|
|
files["manifest.json"] = (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode()
|
|
buffer = io.BytesIO()
|
|
with tarfile.open(fileobj=buffer, mode="w:gz") as bundle:
|
|
for name, data in sorted(files.items()):
|
|
info = tarfile.TarInfo(name)
|
|
info.size, info.mode = len(data), 0o600
|
|
bundle.addfile(info, io.BytesIO(data))
|
|
return core._create(output, buffer.getvalue())
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description="Recheck and archive one completed Pi rollout")
|
|
parser.add_argument("controller", type=Path)
|
|
parser.add_argument("--task", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args(argv)
|
|
try:
|
|
output = archive(args.controller, args.task, args.output)
|
|
print(json.dumps(dict(archive=str(output)), indent=2))
|
|
return 0
|
|
except (ValueError, OSError, RuntimeError) as error:
|
|
print("archive rejected: " + str(error), file=sys.stderr)
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|