data-gen-ebd/tests/test_pi_run.py
amos b773033658 Record model requests and payloads as prefix deltas in the native trace
Each model_request/model_payload repeated the full context, so the trace
grew with the square of the request count: batch 6 Qwen runs hit 64 MiB
at ~75 requests and 14 attempts (13 graded PASS) were discarded.
Requests now record only items after the common prefix with the previous
one (schema embedded-rollout.pi/2), and pi_run.expand rebuilds full
payloads before the unchanged broker/feedback checks. Needs a new image
(trace6) and recalibration before use.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-18 10:50:56 +08:00

310 lines
19 KiB
Python

import hashlib
import json
from pathlib import Path
import subprocess
import tempfile
import unittest
from unittest.mock import patch
from rollout import pi_run
IMAGE = "sha256:" + "a" * 64
TASK = Path(__file__).resolve().parents[1] / "tasks/frame-decoder"
def canonical(value):
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"),
ensure_ascii=False).encode()).hexdigest()
def fixture(attempt, broker_log, bad_digest=False, consumed=True, model="glm-5.3", verdict="PASS", upstream=200,
max_tokens=None):
attempt.mkdir()
(attempt / "trace").mkdir()
route = pi_run.MODELS[model]
limit = {route["max_tokens_field"]: route["max_tokens"] if max_tokens is None else max_tokens}
payloads = [dict(model=model, messages=[dict(role="user", content="fix")], **limit),
dict(model=model, messages=[dict(role="tool", content="written")], **limit)]
result = dict(role="toolResult", toolCallId="write-1", toolName="write",
content=[dict(type="text", text="written")], isError=False)
identity = dict(id=model, api="openai-responses" if model == pi_run.QWEN_MODEL else "openai-completions")
records = [dict(type="trace_header", model=identity), dict(type="model_request", generation_id=0, model=identity,
context=dict(messages=[])), dict(type="model_payload", generation_id=0, payload=payloads[0]),
dict(type="tool_execution_start", toolCallId="write-1", toolName="write"),
dict(type="tool_execution_end", toolCallId="write-1", toolName="write",
result=dict(content=result["content"]), isError=False),
dict(type="model_request", generation_id=1, model=identity, context=dict(messages=[result] if consumed else [])),
dict(type="model_payload", generation_id=1, payload=payloads[1]),
dict(type="session_end", complete=True, closed=True, interrupted=False, final_stop_reason="stop")]
raw = "".join(json.dumps(record) + "\n" for record in records)
(attempt / "agent.stdout").write_text(raw)
(attempt / "trace/native.jsonl").write_text(raw)
completed = dict(responses_terminal="response.completed", responses_terminal_count=1,
responses_stream_valid=True, responses_stream_finished=True, response_model=model, response_status="completed")
broker_log.write_text("".join(json.dumps(dict(sequence=index + 1, status=upstream if index else 200,
request_payload_sha256="bad" if bad_digest else canonical(payload), **completed)) + "\n"
for index, payload in enumerate(payloads)))
(attempt / "receipt.json").write_text(json.dumps(dict(status=verdict, task_verdict=verdict,
cleanup="confirmed", agent_exit=0, termination="exit")))
def delta_records(attempt, broker_log, skip_result=False):
"""A three-request Qwen session recorded as pi/2 prefix deltas, as agent/pi.mjs writes it."""
attempt.mkdir()
(attempt / "trace").mkdir()
model = pi_run.QWEN_MODEL
identity = dict(id=model, api="openai-responses")
result = dict(role="toolResult", toolCallId="write-1", toolName="write",
content=[dict(type="text", text="written")], isError=False)
contexts = [[dict(role="user", content="fix")]]
contexts.append(contexts[0] + [dict(role="assistant", content=[dict(type="toolCall", id="write-1")]), result])
contexts.append(contexts[1] + [dict(role="assistant", content=[dict(type="text", text="checking")])])
if skip_result:
contexts = [[message for message in context if message is not result] for context in contexts]
inputs = [[dict(role="user", content="fix")]]
inputs.append(inputs[0] + [dict(type="function_call", call_id="write-1"),
dict(type="function_call_output", call_id="write-1", output="written")])
inputs.append(inputs[1] + [dict(type="message", role="assistant", content="checking")])
payloads = [dict(model=model, input=items, max_output_tokens=65536) for items in inputs]
records, sent = [dict(type="trace_header", schema="embedded-rollout.pi/2", model=identity)], {}
for generation, (context, payload) in enumerate(zip(contexts, payloads)):
for kind, body, field, items in (("model_request", "context", "messages", context),
("model_payload", "payload", "input", payload["input"])):
prior = sent.get(kind, [])
start = next((index for index, (old, new) in enumerate(zip(prior, items)) if old != new),
min(len(prior), len(items)))
sent[kind] = items
recorded = items[start:]
records.append({"type": kind, "generation_id": generation, "delta": {"field": field, "from": start},
body: dict(messages=recorded) if kind == "model_request" else dict(payload, input=recorded),
**({"model": identity} if kind == "model_request" else {})})
if generation == 0:
records += [dict(type="tool_execution_start", toolCallId="write-1", toolName="write"),
dict(type="tool_execution_end", toolCallId="write-1", toolName="write",
result=dict(content=result["content"]), isError=False)]
records.append(dict(type="session_end", complete=True, closed=True, interrupted=False, final_stop_reason="stop"))
raw = "".join(json.dumps(record) + "\n" for record in records)
(attempt / "agent.stdout").write_text(raw)
(attempt / "trace/native.jsonl").write_text(raw)
completed = dict(status=200, responses_terminal="response.completed", responses_terminal_count=1,
responses_stream_valid=True, responses_stream_finished=True, response_model=model, response_status="completed")
broker_log.write_text("".join(json.dumps(dict(sequence=index + 1, request_payload_sha256=canonical(payload),
**completed)) + "\n" for index, payload in enumerate(payloads)))
return records
class PiRunTests(unittest.TestCase):
def test_prefix_delta_trace_rebuilds_full_payloads_and_feedback(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory).resolve()
records = delta_records(root / "ok", root / "broker.jsonl")
self.assertEqual([record["delta"]["from"] for record in records if "delta" in record], [0, 0, 1, 1, 3, 3])
result = pi_run.evidence(root / "ok", root / "broker.jsonl", pi_run.QWEN_MODEL, 65536)
self.assertEqual((result["status"], result["model_requests"]), ("VERIFIED", 3))
# Feedback missing from every rebuilt context still fails the feedback check.
delta_records(root / "skipped", root / "broker2.jsonl", skip_result=True)
self.assertEqual(pi_run.evidence(root / "skipped", root / "broker2.jsonl", pi_run.QWEN_MODEL)["reason"],
"tool feedback did not enter a later model request")
raw = (root / "ok/agent.stdout").read_text()
for index, (old, new, reason) in enumerate((
('"from": 3', '"from": 4', "invalid model trace delta"),
('"field": "input", "from": 1', '"field": "messages", "from": 1', "invalid model trace delta"),
('"output": "written"', '"output": "forged"', "proxy did not confirm the recorded model payload"))):
broken = root / ("broken-" + str(index))
(broken / "trace").mkdir(parents=True)
(broken / "agent.stdout").write_text(raw.replace(old, new, 1))
(broken / "trace/native.jsonl").write_text(raw.replace(old, new, 1))
self.assertEqual(pi_run.evidence(broken, root / "broker.jsonl", pi_run.QWEN_MODEL)["reason"], reason)
def test_payload_and_native_feedback_must_both_match(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory).resolve()
fixture(root / "ok", root / "broker.jsonl")
self.assertEqual(pi_run.evidence(root / "ok", root / "broker.jsonl")["status"], "VERIFIED")
fixture(root / "wrong", root / "broker2.jsonl", bad_digest=True)
self.assertEqual(pi_run.evidence(root / "wrong", root / "broker2.jsonl")["status"], "UNVERIFIED")
fixture(root / "missing", root / "broker3.jsonl", consumed=False)
self.assertEqual(pi_run.evidence(root / "missing", root / "broker3.jsonl")["status"], "UNVERIFIED")
(root / "ok/trace/native.jsonl").write_text('{"type":"fake"}\n')
self.assertEqual(pi_run.evidence(root / "ok", root / "broker.jsonl")["status"], "UNVERIFIED")
raw = (root / "missing/agent.stdout").read_text().replace('"messages": []', '"messages": [null]')
(root / "missing/agent.stdout").write_text(raw)
(root / "missing/trace/native.jsonl").write_text(raw)
self.assertEqual(pi_run.evidence(root / "missing", root / "broker3.jsonl")["status"], "UNVERIFIED")
def _scope(self, directory, failure=None, bad_digest=False, cleanup_unknown=False, calibration=True, model="glm-5.3",
calibration_cleanup="confirmed", calibration_failure=None, network_create=0, verdict="PASS",
upstream=200, max_tokens=None):
directory = str(Path(directory).resolve())
state = dict(created=False, removed=False, closed=False, name=None, owner=None)
def docker(argv):
if argv[:3] == ["docker", "network", "create"]:
state["create"] = argv
state.update(name=argv[-1], owner=argv[-2].split("=", 1)[1])
if isinstance(network_create, Exception):
raise network_create
state["created"] = network_create == 0
return subprocess.CompletedProcess(argv, network_create, "network-id", "")
if argv[:3] == ["docker", "network", "rm"]:
state["removed"] = not cleanup_unknown
return subprocess.CompletedProcess(argv, 0, "", "")
if state["removed"] or not state["created"]:
return subprocess.CompletedProcess(argv, 1, "", "Error: No such network")
network = dict(Name=state["name"], Driver="bridge",
Labels={"embedded-rollout.controller": state["owner"]},
IPAM=dict(Config=[dict(Subnet="172.22.0.0/16", Gateway="172.22.0.1")]))
return subprocess.CompletedProcess(argv, 0, json.dumps([network]), "")
class Proxy:
def __init__(self, credential, bind, subnet, log, **options):
state.update(log=log, credential=credential)
state["model"] = options["model"]
self.base_url = "http://172.22.0.1:1234/v1"
def __enter__(self):
return self
def __exit__(self, *args):
state["closed"] = True
def agent(task, output, image, argv, identity, calibration, **options):
self.assertEqual(image, IMAGE)
self.assertEqual(argv, pi_run.PI_ARGV + ["--model", model, "--max-tokens",
str(pi_run.MODELS[model]["max_tokens"])])
self.assertEqual(identity, "pi-0.73.1/" + model)
self.assertEqual(options["model_base_url"], "http://172.22.0.1:1234/v1")
self.assertEqual(options["agent_network"], state["name"])
if failure:
raise failure
attempt = output / "run-mock"
fixture(attempt, state["log"], bad_digest=bad_digest, model=model, verdict=verdict, upstream=upstream,
max_tokens=max_tokens)
return attempt
def calibrate(task, output, image):
if calibration_failure:
raise calibration_failure
path = Path(directory) / "calibration"
path.mkdir()
(path / "receipt.json").write_text(json.dumps(dict(cleanup=calibration_cleanup)))
return path
def checked_calibration(*args):
if not calibration and calibration_cleanup != "confirmed":
raise ValueError("calibration cleanup was not confirmed")
return dict(path="calibration/receipt.json")
with patch.object(pi_run.core, "_image", return_value=IMAGE), \
patch.object(pi_run.core, "_calibration", side_effect=checked_calibration), \
patch.object(pi_run.core, "calibrate", side_effect=calibrate) as calibration_call, \
patch.object(pi_run.core, "_cli", side_effect=docker), \
patch.object(pi_run, "ModelProxy", Proxy), \
patch.object(pi_run.core, "run", side_effect=agent):
control = pi_run.run(TASK, "tag-mutable", Path(directory) / "secret.json", directory,
Path(directory) / "existing" if calibration else None, model=model)
receipt = json.loads((control / "receipt.json").read_text())
if not calibration:
self.assertEqual(calibration_call.call_args.kwargs["image"], IMAGE)
return state, receipt
def test_image_frozen_before_auto_calibration_and_all_resources_cleaned(self):
with tempfile.TemporaryDirectory() as directory:
state, receipt = self._scope(directory, calibration=False)
self.assertTrue(state["closed"] and state["removed"])
self.assertIn("--internal", state["create"])
self.assertEqual(receipt["image_id"], IMAGE)
self.assertEqual(receipt["status"], "PASS")
self.assertEqual(receipt["task_verdict"], "PASS")
self.assertEqual(receipt["cleanup"], "confirmed")
self.assertEqual(receipt["calibration_cleanup"], "confirmed")
self.assertEqual(receipt["training_disposition"], "NOT_TRAIN_READY")
self.assertNotIn("credential", json.dumps(receipt))
def test_auto_calibration_unknown_cleanup_reaches_controller(self):
for options in (dict(calibration_cleanup="unknown"),
dict(calibration_failure=RuntimeError("calibration launch failed"))):
with self.subTest(options=options), tempfile.TemporaryDirectory() as directory:
state, receipt = self._scope(directory, calibration=False, **options)
self.assertEqual(receipt["status"], "INFRA_ERROR")
self.assertEqual(receipt["cleanup"], "unknown")
self.assertEqual(receipt["calibration_cleanup"], "unknown")
self.assertEqual(receipt["error_phase"], "calibration")
self.assertIsNone(receipt["attempt"])
self.assertFalse(state["created"] or state["closed"])
def test_early_network_absence_cannot_confirm_an_unfinished_create(self):
for created, expected in ((RuntimeError("Docker create timed out"), "unknown"),
(-9, "unknown"), (125, "confirmed")):
with self.subTest(created=created), tempfile.TemporaryDirectory() as directory:
state, receipt = self._scope(directory, network_create=created)
self.assertEqual(receipt["status"], "INFRA_ERROR")
self.assertEqual(receipt["error_phase"], "network")
self.assertEqual(receipt["network_cleanup"], expected)
self.assertEqual(receipt["cleanup"], expected)
self.assertFalse(state["created"] or state["closed"])
def test_failure_and_interrupt_still_close_proxy_and_network(self):
for failure in (RuntimeError("launch failed"), KeyboardInterrupt()):
with self.subTest(failure=type(failure).__name__), tempfile.TemporaryDirectory() as directory:
state, receipt = self._scope(directory, failure=failure)
self.assertTrue(state["closed"] and state["removed"])
self.assertEqual(receipt["cleanup"], "confirmed")
self.assertIn(receipt["status"], ("INFRA_ERROR", "UNVERIFIED"))
self.assertEqual(receipt["task_verdict"], "UNVERIFIED")
def test_bad_evidence_or_unknown_cleanup_cannot_make_controller_pass(self):
for options, expected in ((dict(bad_digest=True), "UNVERIFIED"),
(dict(cleanup_unknown=True), "INFRA_ERROR")):
with self.subTest(options=options), tempfile.TemporaryDirectory() as directory:
_, receipt = self._scope(directory, **options)
self.assertEqual(receipt["status"], expected)
self.assertEqual(receipt["task_verdict"], "PASS")
def test_incomplete_session_is_not_a_task_outcome_and_channel_errors_are_infra(self):
for options, expected in ((dict(), "FAIL"), (dict(bad_digest=True), "UNVERIFIED"),
(dict(upstream=500), "INFRA_ERROR")):
with self.subTest(options=options), tempfile.TemporaryDirectory() as directory:
_, receipt = self._scope(directory, verdict="FAIL", **options)
self.assertEqual((receipt["status"], receipt["task_verdict"]), (expected, "FAIL"))
def test_wire_output_limit_must_match_the_configured_model(self):
for model in ("glm-5.3", pi_run.QWEN_MODEL):
with self.subTest(model=model), tempfile.TemporaryDirectory() as directory:
_, receipt = self._scope(directory, model=model, max_tokens=32000)
self.assertEqual(receipt["model_max_tokens"], pi_run.MODELS[model]["max_tokens"])
self.assertEqual(receipt["pi_evidence"]["status"], "UNVERIFIED")
self.assertIn("output limit", receipt["pi_evidence"]["reason"])
self.assertEqual(receipt["status"], "UNVERIFIED")
def test_qwen_controller_uses_responses_and_records_model_identity(self):
with tempfile.TemporaryDirectory() as directory:
state, receipt = self._scope(directory, model=pi_run.QWEN_MODEL)
self.assertEqual(state["model"], pi_run.QWEN_MODEL)
self.assertEqual(receipt["model_id"], pi_run.QWEN_MODEL)
self.assertEqual(receipt["model_api"], "openai-responses")
self.assertEqual(receipt["status"], "PASS")
def test_responses_incomplete_truncated_or_wrong_model_is_not_verified(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory).resolve()
fixture(root / "attempt", root / "broker.jsonl", model=pi_run.QWEN_MODEL)
raw = (root / "broker.jsonl").read_text()
self.assertEqual(pi_run.evidence(root / "attempt", root / "broker.jsonl", pi_run.QWEN_MODEL)["status"], "VERIFIED")
for key, value in (("responses_terminal", "response.incomplete"), ("responses_terminal_count", 2),
("responses_stream_finished", False), ("responses_stream_valid", False),
("response_model", "other"), ("response_status", "incomplete")):
records = [json.loads(line) for line in raw.splitlines()]
records[0][key] = value
(root / "broker.jsonl").write_text("".join(json.dumps(record) + "\n" for record in records))
self.assertEqual(pi_run.evidence(root / "attempt", root / "broker.jsonl", pi_run.QWEN_MODEL)["status"], "UNVERIFIED")
(root / "broker.jsonl").write_text(raw)
self.assertEqual(pi_run.evidence(root / "attempt", root / "broker.jsonl", "glm-5.3")["status"], "UNVERIFIED")
if __name__ == "__main__":
unittest.main()