Each model_request/model_payload repeated the full context, so the trace grew with the square of the request count: batch 6 Qwen runs hit 64 MiB at ~75 requests and 14 attempts (13 graded PASS) were discarded. Requests now record only items after the common prefix with the previous one (schema embedded-rollout.pi/2), and pi_run.expand rebuilds full payloads before the unchanged broker/feedback checks. Needs a new image (trace6) and recalibration before use. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
310 lines
19 KiB
Python
310 lines
19 KiB
Python
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from rollout import pi_run
|
|
|
|
IMAGE = "sha256:" + "a" * 64
|
|
TASK = Path(__file__).resolve().parents[1] / "tasks/frame-decoder"
|
|
|
|
|
|
def canonical(value):
|
|
return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"),
|
|
ensure_ascii=False).encode()).hexdigest()
|
|
|
|
|
|
def fixture(attempt, broker_log, bad_digest=False, consumed=True, model="glm-5.3", verdict="PASS", upstream=200,
|
|
max_tokens=None):
|
|
attempt.mkdir()
|
|
(attempt / "trace").mkdir()
|
|
route = pi_run.MODELS[model]
|
|
limit = {route["max_tokens_field"]: route["max_tokens"] if max_tokens is None else max_tokens}
|
|
payloads = [dict(model=model, messages=[dict(role="user", content="fix")], **limit),
|
|
dict(model=model, messages=[dict(role="tool", content="written")], **limit)]
|
|
result = dict(role="toolResult", toolCallId="write-1", toolName="write",
|
|
content=[dict(type="text", text="written")], isError=False)
|
|
identity = dict(id=model, api="openai-responses" if model == pi_run.QWEN_MODEL else "openai-completions")
|
|
records = [dict(type="trace_header", model=identity), dict(type="model_request", generation_id=0, model=identity,
|
|
context=dict(messages=[])), dict(type="model_payload", generation_id=0, payload=payloads[0]),
|
|
dict(type="tool_execution_start", toolCallId="write-1", toolName="write"),
|
|
dict(type="tool_execution_end", toolCallId="write-1", toolName="write",
|
|
result=dict(content=result["content"]), isError=False),
|
|
dict(type="model_request", generation_id=1, model=identity, context=dict(messages=[result] if consumed else [])),
|
|
dict(type="model_payload", generation_id=1, payload=payloads[1]),
|
|
dict(type="session_end", complete=True, closed=True, interrupted=False, final_stop_reason="stop")]
|
|
raw = "".join(json.dumps(record) + "\n" for record in records)
|
|
(attempt / "agent.stdout").write_text(raw)
|
|
(attempt / "trace/native.jsonl").write_text(raw)
|
|
completed = dict(responses_terminal="response.completed", responses_terminal_count=1,
|
|
responses_stream_valid=True, responses_stream_finished=True, response_model=model, response_status="completed")
|
|
broker_log.write_text("".join(json.dumps(dict(sequence=index + 1, status=upstream if index else 200,
|
|
request_payload_sha256="bad" if bad_digest else canonical(payload), **completed)) + "\n"
|
|
for index, payload in enumerate(payloads)))
|
|
(attempt / "receipt.json").write_text(json.dumps(dict(status=verdict, task_verdict=verdict,
|
|
cleanup="confirmed", agent_exit=0, termination="exit")))
|
|
|
|
|
|
def delta_records(attempt, broker_log, skip_result=False):
|
|
"""A three-request Qwen session recorded as pi/2 prefix deltas, as agent/pi.mjs writes it."""
|
|
attempt.mkdir()
|
|
(attempt / "trace").mkdir()
|
|
model = pi_run.QWEN_MODEL
|
|
identity = dict(id=model, api="openai-responses")
|
|
result = dict(role="toolResult", toolCallId="write-1", toolName="write",
|
|
content=[dict(type="text", text="written")], isError=False)
|
|
contexts = [[dict(role="user", content="fix")]]
|
|
contexts.append(contexts[0] + [dict(role="assistant", content=[dict(type="toolCall", id="write-1")]), result])
|
|
contexts.append(contexts[1] + [dict(role="assistant", content=[dict(type="text", text="checking")])])
|
|
if skip_result:
|
|
contexts = [[message for message in context if message is not result] for context in contexts]
|
|
inputs = [[dict(role="user", content="fix")]]
|
|
inputs.append(inputs[0] + [dict(type="function_call", call_id="write-1"),
|
|
dict(type="function_call_output", call_id="write-1", output="written")])
|
|
inputs.append(inputs[1] + [dict(type="message", role="assistant", content="checking")])
|
|
payloads = [dict(model=model, input=items, max_output_tokens=65536) for items in inputs]
|
|
records, sent = [dict(type="trace_header", schema="embedded-rollout.pi/2", model=identity)], {}
|
|
for generation, (context, payload) in enumerate(zip(contexts, payloads)):
|
|
for kind, body, field, items in (("model_request", "context", "messages", context),
|
|
("model_payload", "payload", "input", payload["input"])):
|
|
prior = sent.get(kind, [])
|
|
start = next((index for index, (old, new) in enumerate(zip(prior, items)) if old != new),
|
|
min(len(prior), len(items)))
|
|
sent[kind] = items
|
|
recorded = items[start:]
|
|
records.append({"type": kind, "generation_id": generation, "delta": {"field": field, "from": start},
|
|
body: dict(messages=recorded) if kind == "model_request" else dict(payload, input=recorded),
|
|
**({"model": identity} if kind == "model_request" else {})})
|
|
if generation == 0:
|
|
records += [dict(type="tool_execution_start", toolCallId="write-1", toolName="write"),
|
|
dict(type="tool_execution_end", toolCallId="write-1", toolName="write",
|
|
result=dict(content=result["content"]), isError=False)]
|
|
records.append(dict(type="session_end", complete=True, closed=True, interrupted=False, final_stop_reason="stop"))
|
|
raw = "".join(json.dumps(record) + "\n" for record in records)
|
|
(attempt / "agent.stdout").write_text(raw)
|
|
(attempt / "trace/native.jsonl").write_text(raw)
|
|
completed = dict(status=200, responses_terminal="response.completed", responses_terminal_count=1,
|
|
responses_stream_valid=True, responses_stream_finished=True, response_model=model, response_status="completed")
|
|
broker_log.write_text("".join(json.dumps(dict(sequence=index + 1, request_payload_sha256=canonical(payload),
|
|
**completed)) + "\n" for index, payload in enumerate(payloads)))
|
|
return records
|
|
|
|
|
|
class PiRunTests(unittest.TestCase):
|
|
def test_prefix_delta_trace_rebuilds_full_payloads_and_feedback(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory).resolve()
|
|
records = delta_records(root / "ok", root / "broker.jsonl")
|
|
self.assertEqual([record["delta"]["from"] for record in records if "delta" in record], [0, 0, 1, 1, 3, 3])
|
|
result = pi_run.evidence(root / "ok", root / "broker.jsonl", pi_run.QWEN_MODEL, 65536)
|
|
self.assertEqual((result["status"], result["model_requests"]), ("VERIFIED", 3))
|
|
|
|
# Feedback missing from every rebuilt context still fails the feedback check.
|
|
delta_records(root / "skipped", root / "broker2.jsonl", skip_result=True)
|
|
self.assertEqual(pi_run.evidence(root / "skipped", root / "broker2.jsonl", pi_run.QWEN_MODEL)["reason"],
|
|
"tool feedback did not enter a later model request")
|
|
|
|
raw = (root / "ok/agent.stdout").read_text()
|
|
for index, (old, new, reason) in enumerate((
|
|
('"from": 3', '"from": 4', "invalid model trace delta"),
|
|
('"field": "input", "from": 1', '"field": "messages", "from": 1', "invalid model trace delta"),
|
|
('"output": "written"', '"output": "forged"', "proxy did not confirm the recorded model payload"))):
|
|
broken = root / ("broken-" + str(index))
|
|
(broken / "trace").mkdir(parents=True)
|
|
(broken / "agent.stdout").write_text(raw.replace(old, new, 1))
|
|
(broken / "trace/native.jsonl").write_text(raw.replace(old, new, 1))
|
|
self.assertEqual(pi_run.evidence(broken, root / "broker.jsonl", pi_run.QWEN_MODEL)["reason"], reason)
|
|
|
|
|
|
def test_payload_and_native_feedback_must_both_match(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory).resolve()
|
|
fixture(root / "ok", root / "broker.jsonl")
|
|
self.assertEqual(pi_run.evidence(root / "ok", root / "broker.jsonl")["status"], "VERIFIED")
|
|
fixture(root / "wrong", root / "broker2.jsonl", bad_digest=True)
|
|
self.assertEqual(pi_run.evidence(root / "wrong", root / "broker2.jsonl")["status"], "UNVERIFIED")
|
|
fixture(root / "missing", root / "broker3.jsonl", consumed=False)
|
|
self.assertEqual(pi_run.evidence(root / "missing", root / "broker3.jsonl")["status"], "UNVERIFIED")
|
|
(root / "ok/trace/native.jsonl").write_text('{"type":"fake"}\n')
|
|
self.assertEqual(pi_run.evidence(root / "ok", root / "broker.jsonl")["status"], "UNVERIFIED")
|
|
raw = (root / "missing/agent.stdout").read_text().replace('"messages": []', '"messages": [null]')
|
|
(root / "missing/agent.stdout").write_text(raw)
|
|
(root / "missing/trace/native.jsonl").write_text(raw)
|
|
self.assertEqual(pi_run.evidence(root / "missing", root / "broker3.jsonl")["status"], "UNVERIFIED")
|
|
|
|
def _scope(self, directory, failure=None, bad_digest=False, cleanup_unknown=False, calibration=True, model="glm-5.3",
|
|
calibration_cleanup="confirmed", calibration_failure=None, network_create=0, verdict="PASS",
|
|
upstream=200, max_tokens=None):
|
|
directory = str(Path(directory).resolve())
|
|
state = dict(created=False, removed=False, closed=False, name=None, owner=None)
|
|
|
|
def docker(argv):
|
|
if argv[:3] == ["docker", "network", "create"]:
|
|
state["create"] = argv
|
|
state.update(name=argv[-1], owner=argv[-2].split("=", 1)[1])
|
|
if isinstance(network_create, Exception):
|
|
raise network_create
|
|
state["created"] = network_create == 0
|
|
return subprocess.CompletedProcess(argv, network_create, "network-id", "")
|
|
if argv[:3] == ["docker", "network", "rm"]:
|
|
state["removed"] = not cleanup_unknown
|
|
return subprocess.CompletedProcess(argv, 0, "", "")
|
|
if state["removed"] or not state["created"]:
|
|
return subprocess.CompletedProcess(argv, 1, "", "Error: No such network")
|
|
network = dict(Name=state["name"], Driver="bridge",
|
|
Labels={"embedded-rollout.controller": state["owner"]},
|
|
IPAM=dict(Config=[dict(Subnet="172.22.0.0/16", Gateway="172.22.0.1")]))
|
|
return subprocess.CompletedProcess(argv, 0, json.dumps([network]), "")
|
|
|
|
class Proxy:
|
|
def __init__(self, credential, bind, subnet, log, **options):
|
|
state.update(log=log, credential=credential)
|
|
state["model"] = options["model"]
|
|
self.base_url = "http://172.22.0.1:1234/v1"
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
state["closed"] = True
|
|
|
|
def agent(task, output, image, argv, identity, calibration, **options):
|
|
self.assertEqual(image, IMAGE)
|
|
self.assertEqual(argv, pi_run.PI_ARGV + ["--model", model, "--max-tokens",
|
|
str(pi_run.MODELS[model]["max_tokens"])])
|
|
self.assertEqual(identity, "pi-0.73.1/" + model)
|
|
self.assertEqual(options["model_base_url"], "http://172.22.0.1:1234/v1")
|
|
self.assertEqual(options["agent_network"], state["name"])
|
|
if failure:
|
|
raise failure
|
|
attempt = output / "run-mock"
|
|
fixture(attempt, state["log"], bad_digest=bad_digest, model=model, verdict=verdict, upstream=upstream,
|
|
max_tokens=max_tokens)
|
|
return attempt
|
|
|
|
def calibrate(task, output, image):
|
|
if calibration_failure:
|
|
raise calibration_failure
|
|
path = Path(directory) / "calibration"
|
|
path.mkdir()
|
|
(path / "receipt.json").write_text(json.dumps(dict(cleanup=calibration_cleanup)))
|
|
return path
|
|
|
|
def checked_calibration(*args):
|
|
if not calibration and calibration_cleanup != "confirmed":
|
|
raise ValueError("calibration cleanup was not confirmed")
|
|
return dict(path="calibration/receipt.json")
|
|
|
|
with patch.object(pi_run.core, "_image", return_value=IMAGE), \
|
|
patch.object(pi_run.core, "_calibration", side_effect=checked_calibration), \
|
|
patch.object(pi_run.core, "calibrate", side_effect=calibrate) as calibration_call, \
|
|
patch.object(pi_run.core, "_cli", side_effect=docker), \
|
|
patch.object(pi_run, "ModelProxy", Proxy), \
|
|
patch.object(pi_run.core, "run", side_effect=agent):
|
|
control = pi_run.run(TASK, "tag-mutable", Path(directory) / "secret.json", directory,
|
|
Path(directory) / "existing" if calibration else None, model=model)
|
|
receipt = json.loads((control / "receipt.json").read_text())
|
|
if not calibration:
|
|
self.assertEqual(calibration_call.call_args.kwargs["image"], IMAGE)
|
|
return state, receipt
|
|
|
|
def test_image_frozen_before_auto_calibration_and_all_resources_cleaned(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
state, receipt = self._scope(directory, calibration=False)
|
|
self.assertTrue(state["closed"] and state["removed"])
|
|
self.assertIn("--internal", state["create"])
|
|
self.assertEqual(receipt["image_id"], IMAGE)
|
|
self.assertEqual(receipt["status"], "PASS")
|
|
self.assertEqual(receipt["task_verdict"], "PASS")
|
|
self.assertEqual(receipt["cleanup"], "confirmed")
|
|
self.assertEqual(receipt["calibration_cleanup"], "confirmed")
|
|
self.assertEqual(receipt["training_disposition"], "NOT_TRAIN_READY")
|
|
self.assertNotIn("credential", json.dumps(receipt))
|
|
|
|
def test_auto_calibration_unknown_cleanup_reaches_controller(self):
|
|
for options in (dict(calibration_cleanup="unknown"),
|
|
dict(calibration_failure=RuntimeError("calibration launch failed"))):
|
|
with self.subTest(options=options), tempfile.TemporaryDirectory() as directory:
|
|
state, receipt = self._scope(directory, calibration=False, **options)
|
|
self.assertEqual(receipt["status"], "INFRA_ERROR")
|
|
self.assertEqual(receipt["cleanup"], "unknown")
|
|
self.assertEqual(receipt["calibration_cleanup"], "unknown")
|
|
self.assertEqual(receipt["error_phase"], "calibration")
|
|
self.assertIsNone(receipt["attempt"])
|
|
self.assertFalse(state["created"] or state["closed"])
|
|
|
|
def test_early_network_absence_cannot_confirm_an_unfinished_create(self):
|
|
for created, expected in ((RuntimeError("Docker create timed out"), "unknown"),
|
|
(-9, "unknown"), (125, "confirmed")):
|
|
with self.subTest(created=created), tempfile.TemporaryDirectory() as directory:
|
|
state, receipt = self._scope(directory, network_create=created)
|
|
self.assertEqual(receipt["status"], "INFRA_ERROR")
|
|
self.assertEqual(receipt["error_phase"], "network")
|
|
self.assertEqual(receipt["network_cleanup"], expected)
|
|
self.assertEqual(receipt["cleanup"], expected)
|
|
self.assertFalse(state["created"] or state["closed"])
|
|
|
|
def test_failure_and_interrupt_still_close_proxy_and_network(self):
|
|
for failure in (RuntimeError("launch failed"), KeyboardInterrupt()):
|
|
with self.subTest(failure=type(failure).__name__), tempfile.TemporaryDirectory() as directory:
|
|
state, receipt = self._scope(directory, failure=failure)
|
|
self.assertTrue(state["closed"] and state["removed"])
|
|
self.assertEqual(receipt["cleanup"], "confirmed")
|
|
self.assertIn(receipt["status"], ("INFRA_ERROR", "UNVERIFIED"))
|
|
self.assertEqual(receipt["task_verdict"], "UNVERIFIED")
|
|
|
|
def test_bad_evidence_or_unknown_cleanup_cannot_make_controller_pass(self):
|
|
for options, expected in ((dict(bad_digest=True), "UNVERIFIED"),
|
|
(dict(cleanup_unknown=True), "INFRA_ERROR")):
|
|
with self.subTest(options=options), tempfile.TemporaryDirectory() as directory:
|
|
_, receipt = self._scope(directory, **options)
|
|
self.assertEqual(receipt["status"], expected)
|
|
self.assertEqual(receipt["task_verdict"], "PASS")
|
|
|
|
def test_incomplete_session_is_not_a_task_outcome_and_channel_errors_are_infra(self):
|
|
for options, expected in ((dict(), "FAIL"), (dict(bad_digest=True), "UNVERIFIED"),
|
|
(dict(upstream=500), "INFRA_ERROR")):
|
|
with self.subTest(options=options), tempfile.TemporaryDirectory() as directory:
|
|
_, receipt = self._scope(directory, verdict="FAIL", **options)
|
|
self.assertEqual((receipt["status"], receipt["task_verdict"]), (expected, "FAIL"))
|
|
|
|
def test_wire_output_limit_must_match_the_configured_model(self):
|
|
for model in ("glm-5.3", pi_run.QWEN_MODEL):
|
|
with self.subTest(model=model), tempfile.TemporaryDirectory() as directory:
|
|
_, receipt = self._scope(directory, model=model, max_tokens=32000)
|
|
self.assertEqual(receipt["model_max_tokens"], pi_run.MODELS[model]["max_tokens"])
|
|
self.assertEqual(receipt["pi_evidence"]["status"], "UNVERIFIED")
|
|
self.assertIn("output limit", receipt["pi_evidence"]["reason"])
|
|
self.assertEqual(receipt["status"], "UNVERIFIED")
|
|
|
|
def test_qwen_controller_uses_responses_and_records_model_identity(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
state, receipt = self._scope(directory, model=pi_run.QWEN_MODEL)
|
|
self.assertEqual(state["model"], pi_run.QWEN_MODEL)
|
|
self.assertEqual(receipt["model_id"], pi_run.QWEN_MODEL)
|
|
self.assertEqual(receipt["model_api"], "openai-responses")
|
|
self.assertEqual(receipt["status"], "PASS")
|
|
|
|
def test_responses_incomplete_truncated_or_wrong_model_is_not_verified(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory).resolve()
|
|
fixture(root / "attempt", root / "broker.jsonl", model=pi_run.QWEN_MODEL)
|
|
raw = (root / "broker.jsonl").read_text()
|
|
self.assertEqual(pi_run.evidence(root / "attempt", root / "broker.jsonl", pi_run.QWEN_MODEL)["status"], "VERIFIED")
|
|
for key, value in (("responses_terminal", "response.incomplete"), ("responses_terminal_count", 2),
|
|
("responses_stream_finished", False), ("responses_stream_valid", False),
|
|
("response_model", "other"), ("response_status", "incomplete")):
|
|
records = [json.loads(line) for line in raw.splitlines()]
|
|
records[0][key] = value
|
|
(root / "broker.jsonl").write_text("".join(json.dumps(record) + "\n" for record in records))
|
|
self.assertEqual(pi_run.evidence(root / "attempt", root / "broker.jsonl", pi_run.QWEN_MODEL)["status"], "UNVERIFIED")
|
|
(root / "broker.jsonl").write_text(raw)
|
|
self.assertEqual(pi_run.evidence(root / "attempt", root / "broker.jsonl", "glm-5.3")["status"], "UNVERIFIED")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|