data-gen-ebd/tests/test_batch.py
amos a3365b5583 Keep batch tests independent of the host's free disk
36's /tmp has 28 GiB free, below the 30 GiB floor, so the runner correctly
paused before launching and four tests failed there. Pin the floor to 0 in
those tests and cover the low-disk pause on its own.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 08:31:29 +08:00

134 lines
6.4 KiB
Python

import json
from pathlib import Path
import shutil
import signal
import subprocess
import sys
import tempfile
import time
import unittest
from unittest.mock import patch
from rollout import batch, core
ROOT = Path(__file__).resolve().parents[1]
# Stands in for rollout.pi_run: the n-th launch writes the n-th planned controller and prints its path.
FAKE = """
import json, sys
from pathlib import Path
batch, plan, task = Path(sys.argv[1]), json.loads(Path(sys.argv[2]).read_text()), sys.argv[3]
index = len(list(batch.glob("pi-*")))
outcome = plan["outcomes"][index]
if outcome == "crash":
sys.exit("pi_run could not start")
control = batch / ("pi-%03d" % index)
run = control / "run-x"
run.mkdir(parents=True)
(run / "events.jsonl").write_text("")
receipt = dict(task_id=task, task_digest=plan["digest"], model_id="glm-5.3", attempt=str(run), cleanup="confirmed",
agent_termination="exit", agent_exit=0, status=outcome, task_verdict=outcome,
pi_evidence=dict(status="VERIFIED"))
if outcome == "stream_cut":
receipt.update(status="UNVERIFIED", agent_exit=1, pi_evidence=dict(status="UNVERIFIED"))
(control / "broker.jsonl").write_text(json.dumps(dict(status=200, responses_terminal=None)) + "\\n")
if outcome == "model_http":
receipt.update(status="INFRA_ERROR", agent_exit=1, pi_evidence=dict(status="UNVERIFIED"))
(control / "broker.jsonl").write_text(json.dumps(dict(status=503)) + "\\n")
(control / "receipt.json").write_text(json.dumps(receipt))
print(json.dumps(dict(controller=str(control))))
"""
class BatchTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name).resolve()
self.tasks, self.batch = self.root / "tasks", self.root / "batch"
shutil.copytree(ROOT / "tasks/frame-decoder", self.tasks / "frame-decoder")
self.digest = core.load_task(self.tasks / "frame-decoder")["_digest"]
self.jobs = self.root / "JOBS"
self.jobs.write_text("frame-decoder /calibration\nframe-decoder /calibration\n")
self.floor = 0
def run_batch(self, outcomes):
plan = self.root / "plan.json"
plan.write_text(json.dumps(dict(outcomes=outcomes, digest=self.digest)))
fake = lambda task, calibration, args: [sys.executable, "-c", FAKE, str(args.output), str(plan), task.name]
# The free-space floor is tested on its own; build hosts may have less than it in /tmp.
with patch.object(batch, "_argv", fake), patch("rollout.batch.time.sleep"), \
patch.object(core, "MIN_FREE_BYTES", self.floor):
code = batch.main([str(self.jobs), "--image", "image", "--credential-file", str(self.root / "key"),
"--output", str(self.batch), "--tasks", str(self.tasks), "--parallel", "1"])
return code, sorted(path.name for path in self.batch.glob("pi-*"))
def test_infrastructure_failures_are_resampled_and_a_rerun_resumes(self):
code, controllers = self.run_batch(["stream_cut", "PASS", "FAIL", "PASS"])
# The task FAIL is a real outcome: it pays its line and is not resampled.
self.assertEqual((code, controllers), (0, ["pi-000", "pi-001", "pi-002"]))
reports = list(self.batch.glob("report-*.json"))
self.assertEqual(len(reports), 1)
result = json.loads(reports[0].read_text())
self.assertEqual((result["complete_trajectories"], result["exclusions"]), (2, {"stream_cut": 1}))
self.assertEqual(self.run_batch(["stream_cut", "PASS", "FAIL", "PASS"]), (0, controllers))
self.assertEqual(len(list(self.batch.glob("report-*.json"))), 2)
def test_consecutive_environment_failures_pause_the_batch(self):
code, controllers = self.run_batch(["model_http"] * 6)
self.assertEqual((code, controllers), (3, ["pi-000", "pi-001", "pi-002"]))
# A launcher that never writes a controller pauses too, instead of relaunching forever.
shutil.rmtree(self.batch)
self.assertEqual(self.run_batch(["crash"] * 9), (3, []))
def test_low_disk_pauses_before_launching(self):
self.floor = 1 << 62
self.assertEqual(self.run_batch(["PASS", "PASS"]), (3, []))
def test_retry_budget_bounds_resampling(self):
with patch.object(batch, "PAUSE_AFTER", 99):
code, controllers = self.run_batch(["model_http"] * 9)
# Two lines, two resamples each: six attempts, then nothing is owed.
self.assertEqual((code, len(controllers)), (0, 6))
def test_sigterm_reaches_running_controllers_before_the_runner_exits(self):
child = """
import json, signal, sys, time
from pathlib import Path
control = Path(sys.argv[1]) / "pi-running"
control.mkdir()
def stop(*_):
(control / "receipt.json").write_text(json.dumps(dict(status="UNVERIFIED", termination="interrupted")))
sys.exit(2)
signal.signal(signal.SIGTERM, stop)
(control / "started").write_text("")
time.sleep(30)
"""
runner = "import sys\nfrom rollout import batch\nbatch.core.MIN_FREE_BYTES = 0\n" \
"batch._argv = lambda task, calibration, args: " \
"[sys.executable, '-c', %r, str(args.output)]\nsys.exit(batch.main(sys.argv[1:]))" % child
proc = subprocess.Popen([sys.executable, "-c", runner, str(self.jobs), "--image", "image",
"--credential-file", "key", "--output", str(self.batch), "--tasks", str(self.tasks)],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
started = self.batch / "pi-running/started"
deadline = time.monotonic() + 10
while not started.exists() and time.monotonic() < deadline:
time.sleep(0.05)
proc.send_signal(signal.SIGTERM)
self.assertEqual(proc.wait(10), 2)
receipt = json.loads((self.batch / "pi-running/receipt.json").read_text())
self.assertEqual(receipt["termination"], "interrupted")
self.assertEqual(len(list(self.batch.glob("report-*.json"))), 1)
def test_a_second_runner_on_the_same_batch_is_refused(self):
self.batch.mkdir()
import fcntl
import os
held = os.open(self.batch / ".batch.lock", os.O_WRONLY | os.O_CREAT)
self.addCleanup(os.close, held)
fcntl.flock(held, fcntl.LOCK_EX)
self.assertEqual(self.run_batch(["PASS", "PASS"]), (2, []))
if __name__ == "__main__":
unittest.main()