data-gen-ebd/rollout/archive.py
amos f0c277a0d1 fix: controller owns model output limits; verify them on the wire
Output limits were baked into pi.mjs inside the runtime image, so every
change (GLM 32000 -> 131072, Qwen 8192 -> 65536) needed a new image and a
full 785-trial recalibration, and the Qwen baseline ran on an image the
repo could not rebuild.
- MODELS holds max_tokens and its wire field per model; pi_run passes
  --model and --max-tokens to the adapter, which has no defaults left.
- Receipts record model_max_tokens and model_sampling=service-default;
  evidence requires every payload to carry exactly that limit, which
  would have caught the SDK's silent 32000 cap.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 20:27:29 +08:00

162 lines
8.8 KiB
Python

"""Recheck one completed Pi attempt and archive only its explicit evidence files."""
import argparse
import hashlib
import io
import json
from pathlib import Path
import re
import sys
import tarfile
from . import core, pi_run, export
def _object(raw):
value = json.loads(raw)
if not isinstance(value, dict):
raise ValueError("receipt must be a JSON object")
return value
def _expect(value, expected, label):
if not isinstance(value, dict) or any(value.get(key) != wanted for key, wanted in expected.items()):
raise ValueError(label + " does not match the completed attempt")
def archive(controller_path, task_path, output_path):
controller = core._path(controller_path)
if not controller.is_dir():
raise ValueError("controller must be a directory")
task = core.load_task(task_path)
controller_raw = core._read(controller, "receipt.json")
control = _object(controller_raw)
verdict = control.get("status")
if verdict not in ("PASS", "FAIL"):
raise ValueError("only completed PASS/FAIL attempts can be archived")
image = control.get("image_id")
if not isinstance(image, str) or not re.fullmatch(r"sha256:[a-f0-9]{64}", image):
raise ValueError("a fixed Docker image identity is required")
_expect(control, dict(schema="embedded-rollout.pi-control/1", task_id=task["id"],
task_digest=task["_digest"], task_verdict=verdict, cleanup="confirmed",
network_cleanup="confirmed", proxy_cleanup="confirmed", agent_cleanup="confirmed",
agent_exit=0, agent_termination="exit"), "controller receipt")
if type(control.get("agent_exit")) is not int:
raise ValueError("Agent exit code must be an integer")
_expect(control.get("pi_evidence"), dict(status="VERIFIED"), "native evidence")
if not isinstance(control.get("attempt"), str) or not control["attempt"]:
raise ValueError("controller must identify its attempt")
attempt = core._path(control["attempt"])
if attempt.parent != controller or not attempt.is_dir() or not attempt.name.startswith("run-"):
raise ValueError("attempt must be a direct run directory of the controller")
runs = [core._path(entry) for entry in controller.iterdir() if entry.name.startswith("run-")]
if runs != [attempt]:
raise ValueError("controller must contain one unique run directory")
attempt_raw = core._read(attempt, "receipt.json")
receipt = _object(attempt_raw)
identity = core._receipt(task, "run", image)
_expect(receipt, {key: identity[key] for key in (
"mode", "task_id", "task_version", "task_digest", "manifest_digest", "verifier_digest", "image_id")},
"attempt identity")
_expect(receipt, dict(status=verdict, task_verdict=verdict, cleanup="confirmed",
agent_exit=0, termination="exit"), "attempt receipt")
if type(receipt.get("agent_exit")) is not int:
raise ValueError("Agent exit code must be an integer")
_expect(receipt.get("agent_process"), dict(returncode=0, termination="exit", cleanup="confirmed"),
"Agent process")
calibration_ref = control.get("calibration")
if not isinstance(calibration_ref, dict) or not isinstance(calibration_ref.get("path"), str):
raise ValueError("controller must identify its calibration")
calibration = core._calibration(task, image, calibration_ref["path"])
if calibration != calibration_ref or calibration != receipt.get("calibration"):
raise ValueError("calibration identity or receipt digest changed")
calibration_file = core._path(calibration["path"])
output = core._path(output_path)
if not output.name.endswith(".tar.gz"):
raise ValueError("output must end in .tar.gz")
for source in (controller, task["_root"], calibration_file.parent):
if output == source or source in output.parents:
raise ValueError("archive output must be outside its input directories")
files = {"controller/receipt.json": controller_raw, "attempt/receipt.json": attempt_raw}
for name in ("events.jsonl", "broker.jsonl"):
files["controller/" + name] = core._read(controller, name,
core.MAX_TRACE_BYTES if name == "broker.jsonl" else core.MAX_FILE_BYTES)
candidates = ["candidate/" + name for name in task["candidate_files"]]
for name in ("events.jsonl", "prompt.md", "trace/native.jsonl", "agent.stdout", "agent.stderr",
"grade.stdout", "grade.stderr", *candidates):
limit = core.MAX_TRACE_BYTES if name in ("trace/native.jsonl", "agent.stdout") else core.MAX_FILE_BYTES
files["attempt/" + name] = core._read(attempt, name, limit)
for name in ("task.json", "prompt.md", *("template/" + name for name in core._template(task))):
files["task/" + name] = task["_files"][name]
files["calibration/receipt.json"] = core._read(calibration_file.parent, calibration_file.name)
if hashlib.sha256(files["calibration/receipt.json"]).hexdigest() != calibration["receipt_digest"]:
raise ValueError("calibration changed while collecting evidence")
if files["attempt/prompt.md"] != files["task/prompt.md"]:
raise ValueError("attempt prompt differs from the task")
if core._digest({name: files["attempt/candidate/" + name] for name in task["candidate_files"]}) != receipt.get(
"candidate_digest"):
raise ValueError("frozen candidate digest changed")
grade = receipt.get("grade")
if not isinstance(grade, dict):
raise ValueError("independent grade evidence is missing")
_expect(grade.get("process"), dict(returncode=0 if verdict == "PASS" else 1,
termination="exit", cleanup="confirmed"), "grade process")
if type(grade["process"].get("returncode")) is not int:
raise ValueError("grade exit code must be an integer")
checked = core._checker(grade["process"], attempt / "grade.stdout")
if checked != grade.get("verdict") or checked.get("status") != verdict:
raise ValueError("grade stdout differs from its recorded verdict")
model = control.get("model_id", pi_run.MODEL)
if model not in pi_run.MODEL_IDS:
raise ValueError("unsupported archived model identity")
observed = pi_run.evidence(attempt, controller / "broker.jsonl", model, control.get("model_max_tokens"))
if observed.get("status") != "VERIFIED" or observed != control["pi_evidence"]:
raise ValueError("native model and tool evidence no longer verifies")
if observed.get("trace_sha256") != hashlib.sha256(files["attempt/trace/native.jsonl"]).hexdigest():
raise ValueError("native trace changed while collecting evidence")
# Recheck the captured evidence after helpers read the same host files.
if core._read(controller, "broker.jsonl", core.MAX_TRACE_BYTES) != files["controller/broker.jsonl"]:
raise ValueError("broker evidence changed while collecting evidence")
for name, limit in (("agent.stdout", core.MAX_TRACE_BYTES), ("grade.stdout", core.MAX_FILE_BYTES)):
if core._read(attempt, name, limit) != files["attempt/" + name]:
raise ValueError("host evidence changed while collecting evidence")
row = export.record(task)
files["dataset.jsonl"] = (json.dumps(row, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n").encode("utf-8")
manifest = dict(schema="embedded-rollout.archive/1", task_id=task["id"],
model_id=model, model_api=pi_run.MODELS[model]["api"],
dataset_layout="ultradata-five-fields", dataset_call_type=row["ground_truth"]["call_type"],
task_version=task["version"], task_digest=task["_digest"], image_id=image,
task_verdict=verdict, pi_evidence="VERIFIED", cleanup="confirmed",
training_disposition="NOT_TRAIN_READY", files={name: dict(bytes=len(data),
sha256=hashlib.sha256(data).hexdigest()) for name, data in sorted(files.items())})
files["manifest.json"] = (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode()
buffer = io.BytesIO()
with tarfile.open(fileobj=buffer, mode="w:gz") as bundle:
for name, data in sorted(files.items()):
info = tarfile.TarInfo(name)
info.size, info.mode = len(data), 0o600
bundle.addfile(info, io.BytesIO(data))
return core._create(output, buffer.getvalue())
def main(argv=None):
parser = argparse.ArgumentParser(description="Recheck and archive one completed Pi rollout")
parser.add_argument("controller", type=Path)
parser.add_argument("--task", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args(argv)
try:
output = archive(args.controller, args.task, args.output)
print(json.dumps(dict(archive=str(output)), indent=2))
return 0
except (ValueError, OSError, RuntimeError) as error:
print("archive rejected: " + str(error), file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())