import json from pathlib import Path import shutil import signal import subprocess import sys import tempfile import time import unittest from unittest.mock import patch from rollout import batch, core ROOT = Path(__file__).resolve().parents[1] # Stands in for rollout.pi_run: the n-th launch writes the n-th planned controller and prints its path. FAKE = """ import json, sys from pathlib import Path batch, plan, task = Path(sys.argv[1]), json.loads(Path(sys.argv[2]).read_text()), sys.argv[3] index = len(list(batch.glob("pi-*"))) outcome = plan["outcomes"][index] if outcome == "crash": sys.exit("pi_run could not start") control = batch / ("pi-%03d" % index) run = control / "run-x" run.mkdir(parents=True) (run / "events.jsonl").write_text("") receipt = dict(task_id=task, task_digest=plan["digest"], model_id="glm-5.3", attempt=str(run), cleanup="confirmed", agent_termination="exit", agent_exit=0, status=outcome, task_verdict=outcome, pi_evidence=dict(status="VERIFIED")) if outcome == "stream_cut": receipt.update(status="UNVERIFIED", agent_exit=1, pi_evidence=dict(status="UNVERIFIED")) (control / "broker.jsonl").write_text(json.dumps(dict(status=200, responses_terminal=None)) + "\\n") if outcome == "model_http": receipt.update(status="INFRA_ERROR", agent_exit=1, pi_evidence=dict(status="UNVERIFIED")) (control / "broker.jsonl").write_text(json.dumps(dict(status=503)) + "\\n") (control / "receipt.json").write_text(json.dumps(receipt)) print(json.dumps(dict(controller=str(control)))) """ class BatchTests(unittest.TestCase): def setUp(self): temporary = tempfile.TemporaryDirectory() self.addCleanup(temporary.cleanup) self.root = Path(temporary.name).resolve() self.tasks, self.batch = self.root / "tasks", self.root / "batch" shutil.copytree(ROOT / "tasks/frame-decoder", self.tasks / "frame-decoder") self.digest = core.load_task(self.tasks / "frame-decoder")["_digest"] self.jobs = self.root / "JOBS" self.jobs.write_text("frame-decoder /calibration\nframe-decoder /calibration\n") self.floor = 0 def run_batch(self, outcomes): plan = self.root / "plan.json" plan.write_text(json.dumps(dict(outcomes=outcomes, digest=self.digest))) fake = lambda task, calibration, args: [sys.executable, "-c", FAKE, str(args.output), str(plan), task.name] # The free-space floor is tested on its own; build hosts may have less than it in /tmp. with patch.object(batch, "_argv", fake), patch("rollout.batch.time.sleep"), \ patch.object(core, "MIN_FREE_BYTES", self.floor): code = batch.main([str(self.jobs), "--image", "image", "--credential-file", str(self.root / "key"), "--output", str(self.batch), "--tasks", str(self.tasks), "--parallel", "1"]) return code, sorted(path.name for path in self.batch.glob("pi-*")) def test_infrastructure_failures_are_resampled_and_a_rerun_resumes(self): code, controllers = self.run_batch(["stream_cut", "PASS", "FAIL", "PASS"]) # The task FAIL is a real outcome: it pays its line and is not resampled. self.assertEqual((code, controllers), (0, ["pi-000", "pi-001", "pi-002"])) reports = list(self.batch.glob("report-*.json")) self.assertEqual(len(reports), 1) result = json.loads(reports[0].read_text()) self.assertEqual((result["complete_trajectories"], result["exclusions"]), (2, {"stream_cut": 1})) self.assertEqual(self.run_batch(["stream_cut", "PASS", "FAIL", "PASS"]), (0, controllers)) self.assertEqual(len(list(self.batch.glob("report-*.json"))), 2) def test_consecutive_environment_failures_pause_the_batch(self): code, controllers = self.run_batch(["model_http"] * 6) self.assertEqual((code, controllers), (3, ["pi-000", "pi-001", "pi-002"])) # A launcher that never writes a controller pauses too, instead of relaunching forever. shutil.rmtree(self.batch) self.assertEqual(self.run_batch(["crash"] * 9), (3, [])) def test_low_disk_pauses_before_launching(self): self.floor = 1 << 62 self.assertEqual(self.run_batch(["PASS", "PASS"]), (3, [])) def test_retry_budget_bounds_resampling(self): with patch.object(batch, "PAUSE_AFTER", 99): code, controllers = self.run_batch(["model_http"] * 9) # Two lines, two resamples each: six attempts, then nothing is owed. self.assertEqual((code, len(controllers)), (0, 6)) def test_sigterm_reaches_running_controllers_before_the_runner_exits(self): child = """ import json, signal, sys, time from pathlib import Path control = Path(sys.argv[1]) / "pi-running" control.mkdir() def stop(*_): (control / "receipt.json").write_text(json.dumps(dict(status="UNVERIFIED", termination="interrupted"))) sys.exit(2) signal.signal(signal.SIGTERM, stop) (control / "started").write_text("") time.sleep(30) """ runner = "import sys\nfrom rollout import batch\nbatch.core.MIN_FREE_BYTES = 0\n" \ "batch._argv = lambda task, calibration, args: " \ "[sys.executable, '-c', %r, str(args.output)]\nsys.exit(batch.main(sys.argv[1:]))" % child proc = subprocess.Popen([sys.executable, "-c", runner, str(self.jobs), "--image", "image", "--credential-file", "key", "--output", str(self.batch), "--tasks", str(self.tasks)], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) started = self.batch / "pi-running/started" deadline = time.monotonic() + 10 while not started.exists() and time.monotonic() < deadline: time.sleep(0.05) proc.send_signal(signal.SIGTERM) self.assertEqual(proc.wait(10), 2) receipt = json.loads((self.batch / "pi-running/receipt.json").read_text()) self.assertEqual(receipt["termination"], "interrupted") self.assertEqual(len(list(self.batch.glob("report-*.json"))), 1) def test_a_second_runner_on_the_same_batch_is_refused(self): self.batch.mkdir() import fcntl import os held = os.open(self.batch / ".batch.lock", os.O_WRONLY | os.O_CREAT) self.addCleanup(os.close, held) fcntl.flock(held, fcntl.LOCK_EX) self.assertEqual(self.run_batch(["PASS", "PASS"]), (2, [])) if __name__ == "__main__": unittest.main()