import json from pathlib import Path import shutil import signal import subprocess import sys import tempfile import time import unittest from unittest.mock import patch from rollout import batch, core ROOT = Path(__file__).resolve().parents[1] # Stands in for rollout.pi_run: the n-th launch writes the n-th planned controller and prints its path. FAKE = """ import json, sys from pathlib import Path batch, plan, task = Path(sys.argv[1]), json.loads(Path(sys.argv[2]).read_text()), sys.argv[3] index = len(list(batch.glob("pi-*"))) outcome = plan["outcomes"][index] if outcome == "crash": sys.exit("pi_run could not start") control = batch / ("pi-%03d" % index) run = control / "run-x" run.mkdir(parents=True) (run / "events.jsonl").write_text("") receipt = dict(task_id=task, task_digest=plan["digest"], model_id="glm-5.3", attempt=str(run), cleanup="confirmed", agent_termination="exit", agent_exit=0, status=outcome, task_verdict=outcome, pi_evidence=dict(status="VERIFIED")) if outcome == "stream_cut": receipt.update(status="UNVERIFIED", agent_exit=1, pi_evidence=dict(status="UNVERIFIED")) (control / "broker.jsonl").write_text(json.dumps(dict(status=200, responses_terminal=None)) + "\\n") if outcome == "model_http": receipt.update(status="INFRA_ERROR", agent_exit=1, pi_evidence=dict(status="UNVERIFIED")) (control / "broker.jsonl").write_text(json.dumps(dict(status=503)) + "\\n") if outcome == "disk_low": receipt.update(status="INFRA_ERROR", agent_termination="disk_low", agent_exit=None) (control / "receipt.json").write_text(json.dumps(receipt)) print(json.dumps(dict(controller=str(control)))) """ class BatchTests(unittest.TestCase): def setUp(self): temporary = tempfile.TemporaryDirectory() self.addCleanup(temporary.cleanup) self.root = Path(temporary.name).resolve() self.tasks, self.batch = self.root / "tasks", self.root / "batch" shutil.copytree(ROOT / "tasks/frame-decoder", self.tasks / "frame-decoder") self.digest = core.load_task(self.tasks / "frame-decoder")["_digest"] self.jobs = self.root / "JOBS" self.jobs.write_text("frame-decoder /calibration\nframe-decoder /calibration\n") self.floor = 0 def run_batch(self, outcomes, keys=("key",), parallel=1): plan = self.root / "plan.json" plan.write_text(json.dumps(dict(outcomes=outcomes, digest=self.digest))) self.keys_used = [] def fake(task, calibration, args, key): self.keys_used.append(key.name) return [sys.executable, "-c", FAKE, str(args.output), str(plan), task.name] credentials = [argument for key in keys for argument in ("--credential-file", str(self.root / key))] # The free-space floor is tested on its own; build hosts may have less than it in /tmp. with patch.object(batch, "_argv", fake), patch("rollout.batch.time.sleep"), \ patch.object(core, "MIN_FREE_BYTES", self.floor): code = batch.main([str(self.jobs), "--image", "image", *credentials, "--output", str(self.batch), "--tasks", str(self.tasks), "--parallel", str(parallel)]) return code, sorted(path.name for path in self.batch.glob("pi-*")) def test_attempts_spread_over_keys_and_the_batch_exports_its_rollouts(self): code, controllers = self.run_batch(["PASS", "PASS"], keys=("glm-1", "glm-2"), parallel=2) self.assertEqual((code, sorted(self.keys_used)), (0, ["glm-1", "glm-2"])) exports = list(self.batch.glob("export-*")) self.assertEqual(len(exports), 1) self.assertEqual(sorted(path.name for path in exports[0].iterdir()), ["fail.jsonl", "pass.jsonl", "skipped.json"]) # These stand-in controllers have no native trace, so each is listed with its reason, not exported. summary = json.loads((exports[0] / "skipped.json").read_text()) self.assertEqual((summary["passed"], len(summary["skipped"])), (0, len(controllers))) def test_infrastructure_failures_are_resampled_and_a_rerun_resumes(self): code, controllers = self.run_batch(["stream_cut", "PASS", "FAIL", "PASS"]) # The task FAIL is a real outcome: it pays its line and is not resampled. self.assertEqual((code, controllers), (0, ["pi-000", "pi-001", "pi-002"])) reports = list(self.batch.glob("report-*.json")) self.assertEqual(len(reports), 1) result = json.loads(reports[0].read_text()) self.assertEqual((result["complete_trajectories"], result["exclusions"]), (2, {"stream_cut": 1})) self.assertEqual(self.run_batch(["stream_cut", "PASS", "FAIL", "PASS"]), (0, controllers)) self.assertEqual(len(list(self.batch.glob("report-*.json"))), 2) def test_an_attempt_stopped_for_host_disk_is_owed_again_without_using_a_retry(self): code, controllers = self.run_batch(["disk_low", "PASS", "PASS"]) self.assertEqual((code, controllers), (0, ["pi-000", "pi-001", "pi-002"])) def test_consecutive_environment_failures_pause_the_batch(self): code, controllers = self.run_batch(["model_http"] * 6) self.assertEqual((code, controllers), (3, ["pi-000", "pi-001", "pi-002"])) # A launcher that never writes a controller pauses too, instead of relaunching forever. shutil.rmtree(self.batch) self.assertEqual(self.run_batch(["crash"] * 9), (3, [])) def test_low_disk_pauses_before_launching(self): self.floor = 1 << 62 self.assertEqual(self.run_batch(["PASS", "PASS"]), (3, [])) def test_retry_budget_bounds_resampling(self): with patch.object(batch, "PAUSE_AFTER", 99): code, controllers = self.run_batch(["model_http"] * 9) # Two lines, two resamples each: six attempts, then nothing is owed. self.assertEqual((code, len(controllers)), (0, 6)) def test_sigterm_reaches_running_controllers_before_the_runner_exits(self): child = """ import json, signal, sys, time from pathlib import Path control = Path(sys.argv[1]) / "pi-running" control.mkdir() def stop(*_): (control / "receipt.json").write_text(json.dumps(dict(status="UNVERIFIED", termination="interrupted"))) sys.exit(2) signal.signal(signal.SIGTERM, stop) (control / "started").write_text("") time.sleep(30) """ runner = "import sys\nfrom rollout import batch\nbatch.core.MIN_FREE_BYTES = 0\n" \ "batch._argv = lambda task, calibration, args, key: " \ "[sys.executable, '-c', %r, str(args.output)]\nsys.exit(batch.main(sys.argv[1:]))" % child proc = subprocess.Popen([sys.executable, "-c", runner, str(self.jobs), "--image", "image", "--credential-file", "key", "--output", str(self.batch), "--tasks", str(self.tasks)], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) started = self.batch / "pi-running/started" deadline = time.monotonic() + 10 while not started.exists() and time.monotonic() < deadline: time.sleep(0.05) proc.send_signal(signal.SIGTERM) self.assertEqual(proc.wait(10), 2) receipt = json.loads((self.batch / "pi-running/receipt.json").read_text()) self.assertEqual(receipt["termination"], "interrupted") self.assertEqual(len(list(self.batch.glob("report-*.json"))), 1) def test_a_second_runner_on_the_same_batch_is_refused(self): self.batch.mkdir() import fcntl import os held = os.open(self.batch / ".batch.lock", os.O_WRONLY | os.O_CREAT) self.addCleanup(os.close, held) fcntl.flock(held, fcntl.LOCK_EX) self.assertEqual(self.run_batch(["PASS", "PASS"]), (2, [])) if __name__ == "__main__": unittest.main()