import hashlib import json from pathlib import Path import subprocess import tempfile import unittest from unittest.mock import patch from rollout import pi_run IMAGE = "sha256:" + "a" * 64 TASK = Path(__file__).resolve().parents[1] / "tasks/frame-decoder" def canonical(value): return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode()).hexdigest() def fixture(attempt, broker_log, bad_digest=False, consumed=True, model="glm-5.3", verdict="PASS", upstream=200, max_tokens=None): attempt.mkdir() (attempt / "trace").mkdir() route = pi_run.MODELS[model] limit = {route["max_tokens_field"]: route["max_tokens"] if max_tokens is None else max_tokens} payloads = [dict(model=model, messages=[dict(role="user", content="fix")], **limit), dict(model=model, messages=[dict(role="tool", content="written")], **limit)] result = dict(role="toolResult", toolCallId="write-1", toolName="write", content=[dict(type="text", text="written")], isError=False) identity = dict(id=model, api="openai-responses" if model == pi_run.QWEN_MODEL else "openai-completions") records = [dict(type="trace_header", model=identity), dict(type="model_request", generation_id=0, model=identity, context=dict(messages=[])), dict(type="model_payload", generation_id=0, payload=payloads[0]), dict(type="tool_execution_start", toolCallId="write-1", toolName="write"), dict(type="tool_execution_end", toolCallId="write-1", toolName="write", result=dict(content=result["content"]), isError=False), dict(type="model_request", generation_id=1, model=identity, context=dict(messages=[result] if consumed else [])), dict(type="model_payload", generation_id=1, payload=payloads[1]), dict(type="session_end", complete=True, closed=True, interrupted=False, final_stop_reason="stop")] raw = "".join(json.dumps(record) + "\n" for record in records) (attempt / "agent.stdout").write_text(raw) (attempt / "trace/native.jsonl").write_text(raw) completed = dict(responses_terminal="response.completed", responses_terminal_count=1, responses_stream_valid=True, responses_stream_finished=True, response_model=model, response_status="completed") broker_log.write_text("".join(json.dumps(dict(sequence=index + 1, status=upstream if index else 200, request_payload_sha256="bad" if bad_digest else canonical(payload), **completed)) + "\n" for index, payload in enumerate(payloads))) (attempt / "receipt.json").write_text(json.dumps(dict(status=verdict, task_verdict=verdict, cleanup="confirmed", agent_exit=0, termination="exit"))) def delta_records(attempt, broker_log, skip_result=False): """A three-request Qwen session recorded as pi/2 prefix deltas, as agent/pi.mjs writes it.""" attempt.mkdir() (attempt / "trace").mkdir() model = pi_run.QWEN_MODEL identity = dict(id=model, api="openai-responses") result = dict(role="toolResult", toolCallId="write-1", toolName="write", content=[dict(type="text", text="written")], isError=False) contexts = [[dict(role="user", content="fix")]] contexts.append(contexts[0] + [dict(role="assistant", content=[dict(type="toolCall", id="write-1")]), result]) contexts.append(contexts[1] + [dict(role="assistant", content=[dict(type="text", text="checking")])]) if skip_result: contexts = [[message for message in context if message is not result] for context in contexts] inputs = [[dict(role="user", content="fix")]] inputs.append(inputs[0] + [dict(type="function_call", call_id="write-1"), dict(type="function_call_output", call_id="write-1", output="written")]) inputs.append(inputs[1] + [dict(type="message", role="assistant", content="checking")]) payloads = [dict(model=model, input=items, max_output_tokens=65536) for items in inputs] records, sent = [dict(type="trace_header", schema="embedded-rollout.pi/2", model=identity)], {} for generation, (context, payload) in enumerate(zip(contexts, payloads)): for kind, body, field, items in (("model_request", "context", "messages", context), ("model_payload", "payload", "input", payload["input"])): prior = sent.get(kind, []) start = next((index for index, (old, new) in enumerate(zip(prior, items)) if old != new), min(len(prior), len(items))) sent[kind] = items recorded = items[start:] records.append({"type": kind, "generation_id": generation, "delta": {"field": field, "from": start}, body: dict(messages=recorded) if kind == "model_request" else dict(payload, input=recorded), **({"model": identity} if kind == "model_request" else {})}) if generation == 0: records += [dict(type="tool_execution_start", toolCallId="write-1", toolName="write"), dict(type="tool_execution_end", toolCallId="write-1", toolName="write", result=dict(content=result["content"]), isError=False)] records.append(dict(type="session_end", complete=True, closed=True, interrupted=False, final_stop_reason="stop")) raw = "".join(json.dumps(record) + "\n" for record in records) (attempt / "agent.stdout").write_text(raw) (attempt / "trace/native.jsonl").write_text(raw) completed = dict(status=200, responses_terminal="response.completed", responses_terminal_count=1, responses_stream_valid=True, responses_stream_finished=True, response_model=model, response_status="completed") broker_log.write_text("".join(json.dumps(dict(sequence=index + 1, request_payload_sha256=canonical(payload), **completed)) + "\n" for index, payload in enumerate(payloads))) return records class PiRunTests(unittest.TestCase): def test_prefix_delta_trace_rebuilds_full_payloads_and_feedback(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory).resolve() records = delta_records(root / "ok", root / "broker.jsonl") self.assertEqual([record["delta"]["from"] for record in records if "delta" in record], [0, 0, 1, 1, 3, 3]) result = pi_run.evidence(root / "ok", root / "broker.jsonl", pi_run.QWEN_MODEL, 65536) self.assertEqual((result["status"], result["model_requests"]), ("VERIFIED", 3)) # Feedback missing from every rebuilt context still fails the feedback check. delta_records(root / "skipped", root / "broker2.jsonl", skip_result=True) self.assertEqual(pi_run.evidence(root / "skipped", root / "broker2.jsonl", pi_run.QWEN_MODEL)["reason"], "tool feedback did not enter a later model request") raw = (root / "ok/agent.stdout").read_text() for index, (old, new, reason) in enumerate(( ('"from": 3', '"from": 4', "invalid model trace delta"), ('"field": "input", "from": 1', '"field": "messages", "from": 1', "invalid model trace delta"), ('"output": "written"', '"output": "forged"', "proxy did not confirm the recorded model payload"))): broken = root / ("broken-" + str(index)) (broken / "trace").mkdir(parents=True) (broken / "agent.stdout").write_text(raw.replace(old, new, 1)) (broken / "trace/native.jsonl").write_text(raw.replace(old, new, 1)) self.assertEqual(pi_run.evidence(broken, root / "broker.jsonl", pi_run.QWEN_MODEL)["reason"], reason) def test_payload_and_native_feedback_must_both_match(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory).resolve() fixture(root / "ok", root / "broker.jsonl") self.assertEqual(pi_run.evidence(root / "ok", root / "broker.jsonl")["status"], "VERIFIED") fixture(root / "wrong", root / "broker2.jsonl", bad_digest=True) self.assertEqual(pi_run.evidence(root / "wrong", root / "broker2.jsonl")["status"], "UNVERIFIED") fixture(root / "missing", root / "broker3.jsonl", consumed=False) self.assertEqual(pi_run.evidence(root / "missing", root / "broker3.jsonl")["status"], "UNVERIFIED") (root / "ok/trace/native.jsonl").write_text('{"type":"fake"}\n') self.assertEqual(pi_run.evidence(root / "ok", root / "broker.jsonl")["status"], "UNVERIFIED") raw = (root / "missing/agent.stdout").read_text().replace('"messages": []', '"messages": [null]') (root / "missing/agent.stdout").write_text(raw) (root / "missing/trace/native.jsonl").write_text(raw) self.assertEqual(pi_run.evidence(root / "missing", root / "broker3.jsonl")["status"], "UNVERIFIED") def _scope(self, directory, failure=None, bad_digest=False, cleanup_unknown=False, calibration=True, model="glm-5.3", calibration_cleanup="confirmed", calibration_failure=None, network_create=0, verdict="PASS", upstream=200, max_tokens=None): directory = str(Path(directory).resolve()) state = dict(created=False, removed=False, closed=False, name=None, owner=None) def docker(argv): if argv[:3] == ["docker", "network", "create"]: state["create"] = argv state.update(name=argv[-1], owner=argv[-2].split("=", 1)[1]) if isinstance(network_create, Exception): raise network_create state["created"] = network_create == 0 return subprocess.CompletedProcess(argv, network_create, "network-id", "") if argv[:3] == ["docker", "network", "rm"]: state["removed"] = not cleanup_unknown return subprocess.CompletedProcess(argv, 0, "", "") if state["removed"] or not state["created"]: return subprocess.CompletedProcess(argv, 1, "", "Error: No such network") network = dict(Name=state["name"], Driver="bridge", Labels={"embedded-rollout.controller": state["owner"]}, IPAM=dict(Config=[dict(Subnet="172.22.0.0/16", Gateway="172.22.0.1")])) return subprocess.CompletedProcess(argv, 0, json.dumps([network]), "") class Proxy: def __init__(self, credential, bind, subnet, log, **options): state.update(log=log, credential=credential) state["model"] = options["model"] self.base_url = "http://172.22.0.1:1234/v1" def __enter__(self): return self def __exit__(self, *args): state["closed"] = True def agent(task, output, image, argv, identity, calibration, **options): self.assertEqual(image, IMAGE) self.assertEqual(argv, pi_run.PI_ARGV + ["--model", model, "--max-tokens", str(pi_run.MODELS[model]["max_tokens"])]) self.assertEqual(identity, "pi-0.73.1/" + model) self.assertEqual(options["model_base_url"], "http://172.22.0.1:1234/v1") self.assertEqual(options["agent_network"], state["name"]) if failure: raise failure attempt = output / "run-mock" fixture(attempt, state["log"], bad_digest=bad_digest, model=model, verdict=verdict, upstream=upstream, max_tokens=max_tokens) return attempt def calibrate(task, output, image): if calibration_failure: raise calibration_failure path = Path(directory) / "calibration" path.mkdir() (path / "receipt.json").write_text(json.dumps(dict(cleanup=calibration_cleanup))) return path def checked_calibration(*args): if not calibration and calibration_cleanup != "confirmed": raise ValueError("calibration cleanup was not confirmed") return dict(path="calibration/receipt.json") with patch.object(pi_run.core, "_image", return_value=IMAGE), \ patch.object(pi_run.core, "_calibration", side_effect=checked_calibration), \ patch.object(pi_run.core, "calibrate", side_effect=calibrate) as calibration_call, \ patch.object(pi_run.core, "_cli", side_effect=docker), \ patch.object(pi_run, "ModelProxy", Proxy), \ patch.object(pi_run.core, "run", side_effect=agent): control = pi_run.run(TASK, "tag-mutable", Path(directory) / "secret.json", directory, Path(directory) / "existing" if calibration else None, model=model) receipt = json.loads((control / "receipt.json").read_text()) if not calibration: self.assertEqual(calibration_call.call_args.kwargs["image"], IMAGE) return state, receipt def test_image_frozen_before_auto_calibration_and_all_resources_cleaned(self): with tempfile.TemporaryDirectory() as directory: state, receipt = self._scope(directory, calibration=False) self.assertTrue(state["closed"] and state["removed"]) self.assertIn("--internal", state["create"]) self.assertEqual(receipt["image_id"], IMAGE) self.assertEqual(receipt["status"], "PASS") self.assertEqual(receipt["task_verdict"], "PASS") self.assertEqual(receipt["cleanup"], "confirmed") self.assertEqual(receipt["calibration_cleanup"], "confirmed") self.assertEqual(receipt["training_disposition"], "NOT_TRAIN_READY") self.assertNotIn("credential", json.dumps(receipt)) def test_auto_calibration_unknown_cleanup_reaches_controller(self): for options in (dict(calibration_cleanup="unknown"), dict(calibration_failure=RuntimeError("calibration launch failed"))): with self.subTest(options=options), tempfile.TemporaryDirectory() as directory: state, receipt = self._scope(directory, calibration=False, **options) self.assertEqual(receipt["status"], "INFRA_ERROR") self.assertEqual(receipt["cleanup"], "unknown") self.assertEqual(receipt["calibration_cleanup"], "unknown") self.assertEqual(receipt["error_phase"], "calibration") self.assertIsNone(receipt["attempt"]) self.assertFalse(state["created"] or state["closed"]) def test_early_network_absence_cannot_confirm_an_unfinished_create(self): for created, expected in ((RuntimeError("Docker create timed out"), "unknown"), (-9, "unknown"), (125, "confirmed")): with self.subTest(created=created), tempfile.TemporaryDirectory() as directory: state, receipt = self._scope(directory, network_create=created) self.assertEqual(receipt["status"], "INFRA_ERROR") self.assertEqual(receipt["error_phase"], "network") self.assertEqual(receipt["network_cleanup"], expected) self.assertEqual(receipt["cleanup"], expected) self.assertFalse(state["created"] or state["closed"]) def test_failure_and_interrupt_still_close_proxy_and_network(self): for failure in (RuntimeError("launch failed"), KeyboardInterrupt()): with self.subTest(failure=type(failure).__name__), tempfile.TemporaryDirectory() as directory: state, receipt = self._scope(directory, failure=failure) self.assertTrue(state["closed"] and state["removed"]) self.assertEqual(receipt["cleanup"], "confirmed") self.assertIn(receipt["status"], ("INFRA_ERROR", "UNVERIFIED")) self.assertEqual(receipt["task_verdict"], "UNVERIFIED") def test_bad_evidence_or_unknown_cleanup_cannot_make_controller_pass(self): for options, expected in ((dict(bad_digest=True), "UNVERIFIED"), (dict(cleanup_unknown=True), "INFRA_ERROR")): with self.subTest(options=options), tempfile.TemporaryDirectory() as directory: _, receipt = self._scope(directory, **options) self.assertEqual(receipt["status"], expected) self.assertEqual(receipt["task_verdict"], "PASS") def test_incomplete_session_is_not_a_task_outcome_and_channel_errors_are_infra(self): for options, expected in ((dict(), "FAIL"), (dict(bad_digest=True), "UNVERIFIED"), (dict(upstream=500), "INFRA_ERROR")): with self.subTest(options=options), tempfile.TemporaryDirectory() as directory: _, receipt = self._scope(directory, verdict="FAIL", **options) self.assertEqual((receipt["status"], receipt["task_verdict"]), (expected, "FAIL")) def test_wire_output_limit_must_match_the_configured_model(self): for model in ("glm-5.3", pi_run.QWEN_MODEL): with self.subTest(model=model), tempfile.TemporaryDirectory() as directory: _, receipt = self._scope(directory, model=model, max_tokens=32000) self.assertEqual(receipt["model_max_tokens"], pi_run.MODELS[model]["max_tokens"]) self.assertEqual(receipt["pi_evidence"]["status"], "UNVERIFIED") self.assertIn("output limit", receipt["pi_evidence"]["reason"]) self.assertEqual(receipt["status"], "UNVERIFIED") def test_qwen_controller_uses_responses_and_records_model_identity(self): with tempfile.TemporaryDirectory() as directory: state, receipt = self._scope(directory, model=pi_run.QWEN_MODEL) self.assertEqual(state["model"], pi_run.QWEN_MODEL) self.assertEqual(receipt["model_id"], pi_run.QWEN_MODEL) self.assertEqual(receipt["model_api"], "openai-responses") self.assertEqual(receipt["status"], "PASS") def test_responses_incomplete_truncated_or_wrong_model_is_not_verified(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory).resolve() fixture(root / "attempt", root / "broker.jsonl", model=pi_run.QWEN_MODEL) raw = (root / "broker.jsonl").read_text() self.assertEqual(pi_run.evidence(root / "attempt", root / "broker.jsonl", pi_run.QWEN_MODEL)["status"], "VERIFIED") for key, value in (("responses_terminal", "response.incomplete"), ("responses_terminal_count", 2), ("responses_stream_finished", False), ("responses_stream_valid", False), ("response_model", "other"), ("response_status", "incomplete")): records = [json.loads(line) for line in raw.splitlines()] records[0][key] = value (root / "broker.jsonl").write_text("".join(json.dumps(record) + "\n" for record in records)) self.assertEqual(pi_run.evidence(root / "attempt", root / "broker.jsonl", pi_run.QWEN_MODEL)["status"], "UNVERIFIED") (root / "broker.jsonl").write_text(raw) self.assertEqual(pi_run.evidence(root / "attempt", root / "broker.jsonl", "glm-5.3")["status"], "UNVERIFIED") if __name__ == "__main__": unittest.main()