data-gen-ebd/tests/test_batch.py
amos c64c79b31d Run the Pi agent in Salty jobs over a one-time SSH tunnel and export SFT rows
rollout.salty_run keeps the gateway, grading and evidence on 36 and moves only the
agent step into a Salty job. The gateway listens on 127.0.0.1 with a per-attempt
bearer token; the job reaches it over port 22 with a restricted one-time key that is
removed after each attempt. rollout.batch gains --backend salty and repeatable
--credential-file, and writes pass/fail SFT exports through rollout.sft at the end.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-28 17:45:28 +08:00

150 lines
7.3 KiB
Python

import json
from pathlib import Path
import shutil
import signal
import subprocess
import sys
import tempfile
import time
import unittest
from unittest.mock import patch
from rollout import batch, core
ROOT = Path(__file__).resolve().parents[1]
# Stands in for rollout.pi_run: the n-th launch writes the n-th planned controller and prints its path.
FAKE = """
import json, sys
from pathlib import Path
batch, plan, task = Path(sys.argv[1]), json.loads(Path(sys.argv[2]).read_text()), sys.argv[3]
index = len(list(batch.glob("pi-*")))
outcome = plan["outcomes"][index]
if outcome == "crash":
sys.exit("pi_run could not start")
control = batch / ("pi-%03d" % index)
run = control / "run-x"
run.mkdir(parents=True)
(run / "events.jsonl").write_text("")
receipt = dict(task_id=task, task_digest=plan["digest"], model_id="glm-5.3", attempt=str(run), cleanup="confirmed",
agent_termination="exit", agent_exit=0, status=outcome, task_verdict=outcome,
pi_evidence=dict(status="VERIFIED"))
if outcome == "stream_cut":
receipt.update(status="UNVERIFIED", agent_exit=1, pi_evidence=dict(status="UNVERIFIED"))
(control / "broker.jsonl").write_text(json.dumps(dict(status=200, responses_terminal=None)) + "\\n")
if outcome == "model_http":
receipt.update(status="INFRA_ERROR", agent_exit=1, pi_evidence=dict(status="UNVERIFIED"))
(control / "broker.jsonl").write_text(json.dumps(dict(status=503)) + "\\n")
(control / "receipt.json").write_text(json.dumps(receipt))
print(json.dumps(dict(controller=str(control))))
"""
class BatchTests(unittest.TestCase):
def setUp(self):
temporary = tempfile.TemporaryDirectory()
self.addCleanup(temporary.cleanup)
self.root = Path(temporary.name).resolve()
self.tasks, self.batch = self.root / "tasks", self.root / "batch"
shutil.copytree(ROOT / "tasks/frame-decoder", self.tasks / "frame-decoder")
self.digest = core.load_task(self.tasks / "frame-decoder")["_digest"]
self.jobs = self.root / "JOBS"
self.jobs.write_text("frame-decoder /calibration\nframe-decoder /calibration\n")
self.floor = 0
def run_batch(self, outcomes, keys=("key",), parallel=1):
plan = self.root / "plan.json"
plan.write_text(json.dumps(dict(outcomes=outcomes, digest=self.digest)))
self.keys_used = []
def fake(task, calibration, args, key):
self.keys_used.append(key.name)
return [sys.executable, "-c", FAKE, str(args.output), str(plan), task.name]
credentials = [argument for key in keys for argument in ("--credential-file", str(self.root / key))]
# The free-space floor is tested on its own; build hosts may have less than it in /tmp.
with patch.object(batch, "_argv", fake), patch("rollout.batch.time.sleep"), \
patch.object(core, "MIN_FREE_BYTES", self.floor):
code = batch.main([str(self.jobs), "--image", "image", *credentials,
"--output", str(self.batch), "--tasks", str(self.tasks), "--parallel", str(parallel)])
return code, sorted(path.name for path in self.batch.glob("pi-*"))
def test_attempts_spread_over_keys_and_the_batch_exports_its_rollouts(self):
code, controllers = self.run_batch(["PASS", "PASS"], keys=("glm-1", "glm-2"), parallel=2)
self.assertEqual((code, sorted(self.keys_used)), (0, ["glm-1", "glm-2"]))
exports = list(self.batch.glob("export-*"))
self.assertEqual(len(exports), 1)
self.assertEqual(sorted(path.name for path in exports[0].iterdir()),
["fail.jsonl", "pass.jsonl", "skipped.json"])
# These stand-in controllers have no native trace, so each is listed with its reason, not exported.
summary = json.loads((exports[0] / "skipped.json").read_text())
self.assertEqual((summary["passed"], len(summary["skipped"])), (0, len(controllers)))
def test_infrastructure_failures_are_resampled_and_a_rerun_resumes(self):
code, controllers = self.run_batch(["stream_cut", "PASS", "FAIL", "PASS"])
# The task FAIL is a real outcome: it pays its line and is not resampled.
self.assertEqual((code, controllers), (0, ["pi-000", "pi-001", "pi-002"]))
reports = list(self.batch.glob("report-*.json"))
self.assertEqual(len(reports), 1)
result = json.loads(reports[0].read_text())
self.assertEqual((result["complete_trajectories"], result["exclusions"]), (2, {"stream_cut": 1}))
self.assertEqual(self.run_batch(["stream_cut", "PASS", "FAIL", "PASS"]), (0, controllers))
self.assertEqual(len(list(self.batch.glob("report-*.json"))), 2)
def test_consecutive_environment_failures_pause_the_batch(self):
code, controllers = self.run_batch(["model_http"] * 6)
self.assertEqual((code, controllers), (3, ["pi-000", "pi-001", "pi-002"]))
# A launcher that never writes a controller pauses too, instead of relaunching forever.
shutil.rmtree(self.batch)
self.assertEqual(self.run_batch(["crash"] * 9), (3, []))
def test_low_disk_pauses_before_launching(self):
self.floor = 1 << 62
self.assertEqual(self.run_batch(["PASS", "PASS"]), (3, []))
def test_retry_budget_bounds_resampling(self):
with patch.object(batch, "PAUSE_AFTER", 99):
code, controllers = self.run_batch(["model_http"] * 9)
# Two lines, two resamples each: six attempts, then nothing is owed.
self.assertEqual((code, len(controllers)), (0, 6))
def test_sigterm_reaches_running_controllers_before_the_runner_exits(self):
child = """
import json, signal, sys, time
from pathlib import Path
control = Path(sys.argv[1]) / "pi-running"
control.mkdir()
def stop(*_):
(control / "receipt.json").write_text(json.dumps(dict(status="UNVERIFIED", termination="interrupted")))
sys.exit(2)
signal.signal(signal.SIGTERM, stop)
(control / "started").write_text("")
time.sleep(30)
"""
runner = "import sys\nfrom rollout import batch\nbatch.core.MIN_FREE_BYTES = 0\n" \
"batch._argv = lambda task, calibration, args, key: " \
"[sys.executable, '-c', %r, str(args.output)]\nsys.exit(batch.main(sys.argv[1:]))" % child
proc = subprocess.Popen([sys.executable, "-c", runner, str(self.jobs), "--image", "image",
"--credential-file", "key", "--output", str(self.batch), "--tasks", str(self.tasks)],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
started = self.batch / "pi-running/started"
deadline = time.monotonic() + 10
while not started.exists() and time.monotonic() < deadline:
time.sleep(0.05)
proc.send_signal(signal.SIGTERM)
self.assertEqual(proc.wait(10), 2)
receipt = json.loads((self.batch / "pi-running/receipt.json").read_text())
self.assertEqual(receipt["termination"], "interrupted")
self.assertEqual(len(list(self.batch.glob("report-*.json"))), 1)
def test_a_second_runner_on_the_same_batch_is_refused(self):
self.batch.mkdir()
import fcntl
import os
held = os.open(self.batch / ".batch.lock", os.O_WRONLY | os.O_CREAT)
self.addCleanup(os.close, held)
fcntl.flock(held, fcntl.LOCK_EX)
self.assertEqual(self.run_batch(["PASS", "PASS"]), (2, []))
if __name__ == "__main__":
unittest.main()