36's /tmp has 28 GiB free, below the 30 GiB floor, so the runner correctly paused before launching and four tests failed there. Pin the floor to 0 in those tests and cover the low-disk pause on its own. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
134 lines
6.4 KiB
Python
134 lines
6.4 KiB
Python
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from rollout import batch, core
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
# Stands in for rollout.pi_run: the n-th launch writes the n-th planned controller and prints its path.
|
|
FAKE = """
|
|
import json, sys
|
|
from pathlib import Path
|
|
batch, plan, task = Path(sys.argv[1]), json.loads(Path(sys.argv[2]).read_text()), sys.argv[3]
|
|
index = len(list(batch.glob("pi-*")))
|
|
outcome = plan["outcomes"][index]
|
|
if outcome == "crash":
|
|
sys.exit("pi_run could not start")
|
|
control = batch / ("pi-%03d" % index)
|
|
run = control / "run-x"
|
|
run.mkdir(parents=True)
|
|
(run / "events.jsonl").write_text("")
|
|
receipt = dict(task_id=task, task_digest=plan["digest"], model_id="glm-5.3", attempt=str(run), cleanup="confirmed",
|
|
agent_termination="exit", agent_exit=0, status=outcome, task_verdict=outcome,
|
|
pi_evidence=dict(status="VERIFIED"))
|
|
if outcome == "stream_cut":
|
|
receipt.update(status="UNVERIFIED", agent_exit=1, pi_evidence=dict(status="UNVERIFIED"))
|
|
(control / "broker.jsonl").write_text(json.dumps(dict(status=200, responses_terminal=None)) + "\\n")
|
|
if outcome == "model_http":
|
|
receipt.update(status="INFRA_ERROR", agent_exit=1, pi_evidence=dict(status="UNVERIFIED"))
|
|
(control / "broker.jsonl").write_text(json.dumps(dict(status=503)) + "\\n")
|
|
(control / "receipt.json").write_text(json.dumps(receipt))
|
|
print(json.dumps(dict(controller=str(control))))
|
|
"""
|
|
|
|
|
|
class BatchTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name).resolve()
|
|
self.tasks, self.batch = self.root / "tasks", self.root / "batch"
|
|
shutil.copytree(ROOT / "tasks/frame-decoder", self.tasks / "frame-decoder")
|
|
self.digest = core.load_task(self.tasks / "frame-decoder")["_digest"]
|
|
self.jobs = self.root / "JOBS"
|
|
self.jobs.write_text("frame-decoder /calibration\nframe-decoder /calibration\n")
|
|
self.floor = 0
|
|
|
|
def run_batch(self, outcomes):
|
|
plan = self.root / "plan.json"
|
|
plan.write_text(json.dumps(dict(outcomes=outcomes, digest=self.digest)))
|
|
fake = lambda task, calibration, args: [sys.executable, "-c", FAKE, str(args.output), str(plan), task.name]
|
|
# The free-space floor is tested on its own; build hosts may have less than it in /tmp.
|
|
with patch.object(batch, "_argv", fake), patch("rollout.batch.time.sleep"), \
|
|
patch.object(core, "MIN_FREE_BYTES", self.floor):
|
|
code = batch.main([str(self.jobs), "--image", "image", "--credential-file", str(self.root / "key"),
|
|
"--output", str(self.batch), "--tasks", str(self.tasks), "--parallel", "1"])
|
|
return code, sorted(path.name for path in self.batch.glob("pi-*"))
|
|
|
|
def test_infrastructure_failures_are_resampled_and_a_rerun_resumes(self):
|
|
code, controllers = self.run_batch(["stream_cut", "PASS", "FAIL", "PASS"])
|
|
# The task FAIL is a real outcome: it pays its line and is not resampled.
|
|
self.assertEqual((code, controllers), (0, ["pi-000", "pi-001", "pi-002"]))
|
|
reports = list(self.batch.glob("report-*.json"))
|
|
self.assertEqual(len(reports), 1)
|
|
result = json.loads(reports[0].read_text())
|
|
self.assertEqual((result["complete_trajectories"], result["exclusions"]), (2, {"stream_cut": 1}))
|
|
self.assertEqual(self.run_batch(["stream_cut", "PASS", "FAIL", "PASS"]), (0, controllers))
|
|
self.assertEqual(len(list(self.batch.glob("report-*.json"))), 2)
|
|
|
|
def test_consecutive_environment_failures_pause_the_batch(self):
|
|
code, controllers = self.run_batch(["model_http"] * 6)
|
|
self.assertEqual((code, controllers), (3, ["pi-000", "pi-001", "pi-002"]))
|
|
# A launcher that never writes a controller pauses too, instead of relaunching forever.
|
|
shutil.rmtree(self.batch)
|
|
self.assertEqual(self.run_batch(["crash"] * 9), (3, []))
|
|
|
|
def test_low_disk_pauses_before_launching(self):
|
|
self.floor = 1 << 62
|
|
self.assertEqual(self.run_batch(["PASS", "PASS"]), (3, []))
|
|
|
|
def test_retry_budget_bounds_resampling(self):
|
|
with patch.object(batch, "PAUSE_AFTER", 99):
|
|
code, controllers = self.run_batch(["model_http"] * 9)
|
|
# Two lines, two resamples each: six attempts, then nothing is owed.
|
|
self.assertEqual((code, len(controllers)), (0, 6))
|
|
|
|
def test_sigterm_reaches_running_controllers_before_the_runner_exits(self):
|
|
child = """
|
|
import json, signal, sys, time
|
|
from pathlib import Path
|
|
control = Path(sys.argv[1]) / "pi-running"
|
|
control.mkdir()
|
|
def stop(*_):
|
|
(control / "receipt.json").write_text(json.dumps(dict(status="UNVERIFIED", termination="interrupted")))
|
|
sys.exit(2)
|
|
signal.signal(signal.SIGTERM, stop)
|
|
(control / "started").write_text("")
|
|
time.sleep(30)
|
|
"""
|
|
runner = "import sys\nfrom rollout import batch\nbatch.core.MIN_FREE_BYTES = 0\n" \
|
|
"batch._argv = lambda task, calibration, args: " \
|
|
"[sys.executable, '-c', %r, str(args.output)]\nsys.exit(batch.main(sys.argv[1:]))" % child
|
|
proc = subprocess.Popen([sys.executable, "-c", runner, str(self.jobs), "--image", "image",
|
|
"--credential-file", "key", "--output", str(self.batch), "--tasks", str(self.tasks)],
|
|
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
started = self.batch / "pi-running/started"
|
|
deadline = time.monotonic() + 10
|
|
while not started.exists() and time.monotonic() < deadline:
|
|
time.sleep(0.05)
|
|
proc.send_signal(signal.SIGTERM)
|
|
self.assertEqual(proc.wait(10), 2)
|
|
receipt = json.loads((self.batch / "pi-running/receipt.json").read_text())
|
|
self.assertEqual(receipt["termination"], "interrupted")
|
|
self.assertEqual(len(list(self.batch.glob("report-*.json"))), 1)
|
|
|
|
def test_a_second_runner_on_the_same_batch_is_refused(self):
|
|
self.batch.mkdir()
|
|
import fcntl
|
|
import os
|
|
held = os.open(self.batch / ".batch.lock", os.O_WRONLY | os.O_CREAT)
|
|
self.addCleanup(os.close, held)
|
|
fcntl.flock(held, fcntl.LOCK_EX)
|
|
self.assertEqual(self.run_batch(["PASS", "PASS"]), (2, []))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|