data-gen-ebd/tests/test_batch.py
amos ef3c173fa6 Owe again the attempts stopped because the host disk ran low
A batch paused at the free-space floor recorded its in-flight attempts as disk_limit,
which paid their JOBS lines, so rerunning the same command would have skipped them.
Host disk stops are now their own exclusion, disk_low, and count like interrupted
attempts; attempt_size stays disk_limit and still pays its line.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 11:13:19 +08:00

156 lines
7.7 KiB
Python

import json
from pathlib import Path
import shutil
import signal
import subprocess
import sys
import tempfile
import time
import unittest
from unittest.mock import patch
from rollout import batch, core
ROOT = Path(__file__).resolve().parents[1]
# Stands in for rollout.pi_run: the n-th launch writes the n-th planned controller and prints its path.
FAKE = """
import json, sys
from pathlib import Path
batch, plan, task = Path(sys.argv[1]), json.loads(Path(sys.argv[2]).read_text()), sys.argv[3]
index = len(list(batch.glob("pi-*")))
outcome = plan["outcomes"][index]
if outcome == "crash":
sys.exit("pi_run could not start")
control = batch / ("pi-%03d" % index)
run = control / "run-x"
run.mkdir(parents=True)
(run / "events.jsonl").write_text("")
receipt = dict(task_id=task, task_digest=plan["digest"], model_id="glm-5.3", attempt=str(run), cleanup="confirmed",
agent_termination="exit", agent_exit=0, status=outcome, task_verdict=outcome,
pi_evidence=dict(status="VERIFIED"))
if outcome == "stream_cut":
receipt.update(status="UNVERIFIED", agent_exit=1, pi_evidence=dict(status="UNVERIFIED"))
(control / "broker.jsonl").write_text(json.dumps(dict(status=200, responses_terminal=None)) + "\\n")
if outcome == "model_http":
receipt.update(status="INFRA_ERROR", agent_exit=1, pi_evidence=dict(status="UNVERIFIED"))
(control / "broker.jsonl").write_text(json.dumps(dict(status=503)) + "\\n")
if outcome == "disk_low":
receipt.update(status="INFRA_ERROR", agent_termination="disk_low", agent_exit=None)
(control / "receipt.json").write_text(json.dumps(receipt))
print(json.dumps(dict(controller=str(control))))
"""
class BatchTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name).resolve()
self.tasks, self.batch = self.root / "tasks", self.root / "batch"
shutil.copytree(ROOT / "tasks/frame-decoder", self.tasks / "frame-decoder")
self.digest = core.load_task(self.tasks / "frame-decoder")["_digest"]
self.jobs = self.root / "JOBS"
self.jobs.write_text("frame-decoder /calibration\nframe-decoder /calibration\n")
self.floor = 0
def run_batch(self, outcomes, keys=("key",), parallel=1):
plan = self.root / "plan.json"
plan.write_text(json.dumps(dict(outcomes=outcomes, digest=self.digest)))
self.keys_used = []
def fake(task, calibration, args, key):
self.keys_used.append(key.name)
return [sys.executable, "-c", FAKE, str(args.output), str(plan), task.name]
credentials = [argument for key in keys for argument in ("--credential-file", str(self.root / key))]
# The free-space floor is tested on its own; build hosts may have less than it in /tmp.
with patch.object(batch, "_argv", fake), patch("rollout.batch.time.sleep"), \
patch.object(core, "MIN_FREE_BYTES", self.floor):
code = batch.main([str(self.jobs), "--image", "image", *credentials,
"--output", str(self.batch), "--tasks", str(self.tasks), "--parallel", str(parallel)])
return code, sorted(path.name for path in self.batch.glob("pi-*"))
def test_attempts_spread_over_keys_and_the_batch_exports_its_rollouts(self):
code, controllers = self.run_batch(["PASS", "PASS"], keys=("glm-1", "glm-2"), parallel=2)
self.assertEqual((code, sorted(self.keys_used)), (0, ["glm-1", "glm-2"]))
exports = list(self.batch.glob("export-*"))
self.assertEqual(len(exports), 1)
self.assertEqual(sorted(path.name for path in exports[0].iterdir()),
["fail.jsonl", "pass.jsonl", "skipped.json"])
# These stand-in controllers have no native trace, so each is listed with its reason, not exported.
summary = json.loads((exports[0] / "skipped.json").read_text())
self.assertEqual((summary["passed"], len(summary["skipped"])), (0, len(controllers)))
def test_infrastructure_failures_are_resampled_and_a_rerun_resumes(self):
code, controllers = self.run_batch(["stream_cut", "PASS", "FAIL", "PASS"])
# The task FAIL is a real outcome: it pays its line and is not resampled.
self.assertEqual((code, controllers), (0, ["pi-000", "pi-001", "pi-002"]))
reports = list(self.batch.glob("report-*.json"))
self.assertEqual(len(reports), 1)
result = json.loads(reports[0].read_text())
self.assertEqual((result["complete_trajectories"], result["exclusions"]), (2, {"stream_cut": 1}))
self.assertEqual(self.run_batch(["stream_cut", "PASS", "FAIL", "PASS"]), (0, controllers))
self.assertEqual(len(list(self.batch.glob("report-*.json"))), 2)
def test_an_attempt_stopped_for_host_disk_is_owed_again_without_using_a_retry(self):
code, controllers = self.run_batch(["disk_low", "PASS", "PASS"])
self.assertEqual((code, controllers), (0, ["pi-000", "pi-001", "pi-002"]))
def test_consecutive_environment_failures_pause_the_batch(self):
code, controllers = self.run_batch(["model_http"] * 6)
self.assertEqual((code, controllers), (3, ["pi-000", "pi-001", "pi-002"]))
# A launcher that never writes a controller pauses too, instead of relaunching forever.
shutil.rmtree(self.batch)
self.assertEqual(self.run_batch(["crash"] * 9), (3, []))
def test_low_disk_pauses_before_launching(self):
self.floor = 1 << 62
self.assertEqual(self.run_batch(["PASS", "PASS"]), (3, []))
def test_retry_budget_bounds_resampling(self):
with patch.object(batch, "PAUSE_AFTER", 99):
code, controllers = self.run_batch(["model_http"] * 9)
# Two lines, two resamples each: six attempts, then nothing is owed.
self.assertEqual((code, len(controllers)), (0, 6))
def test_sigterm_reaches_running_controllers_before_the_runner_exits(self):
child = """
import json, signal, sys, time
from pathlib import Path
control = Path(sys.argv[1]) / "pi-running"
control.mkdir()
def stop(*_):
(control / "receipt.json").write_text(json.dumps(dict(status="UNVERIFIED", termination="interrupted")))
sys.exit(2)
signal.signal(signal.SIGTERM, stop)
(control / "started").write_text("")
time.sleep(30)
"""
runner = "import sys\nfrom rollout import batch\nbatch.core.MIN_FREE_BYTES = 0\n" \
"batch._argv = lambda task, calibration, args, key: " \
"[sys.executable, '-c', %r, str(args.output)]\nsys.exit(batch.main(sys.argv[1:]))" % child
proc = subprocess.Popen([sys.executable, "-c", runner, str(self.jobs), "--image", "image",
"--credential-file", "key", "--output", str(self.batch), "--tasks", str(self.tasks)],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
started = self.batch / "pi-running/started"
deadline = time.monotonic() + 10
while not started.exists() and time.monotonic() < deadline:
time.sleep(0.05)
proc.send_signal(signal.SIGTERM)
self.assertEqual(proc.wait(10), 2)
receipt = json.loads((self.batch / "pi-running/receipt.json").read_text())
self.assertEqual(receipt["termination"], "interrupted")
self.assertEqual(len(list(self.batch.glob("report-*.json"))), 1)
def test_a_second_runner_on_the_same_batch_is_refused(self):
self.batch.mkdir()
import fcntl
import os
held = os.open(self.batch / ".batch.lock", os.O_WRONLY | os.O_CREAT)
self.addCleanup(os.close, held)
fcntl.flock(held, fcntl.LOCK_EX)
self.assertEqual(self.run_batch(["PASS", "PASS"]), (2, []))
if __name__ == "__main__":
unittest.main()