rollout.salty_run keeps the gateway, grading and evidence on 36 and moves only the agent step into a Salty job. The gateway listens on 127.0.0.1 with a per-attempt bearer token; the job reaches it over port 22 with a restricted one-time key that is removed after each attempt. rollout.batch gains --backend salty and repeatable --credential-file, and writes pass/fail SFT exports through rollout.sft at the end. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
154 lines
7.8 KiB
Python
154 lines
7.8 KiB
Python
"""Run a JOBS file to completion: resume from receipts, resample infrastructure failures and pause when the
|
||
environment is broken."""
|
||
|
||
import argparse
|
||
import collections
|
||
import datetime
|
||
import fcntl
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
|
||
from . import core, report, sft
|
||
from .model_proxy import MODEL, MODEL_IDS
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
# Failures that say the environment, not the Agent, is broken; this many in a row pauses the batch.
|
||
BROKEN = report.RESAMPLE | {"calibration", "no_receipt"}
|
||
PAUSE_AFTER = 3
|
||
|
||
|
||
def _argv(task, calibration, args, credential):
|
||
module = "rollout.salty_run" if args.backend == "salty" else "rollout.pi_run"
|
||
argv = [sys.executable, "-m", module, str(task), "--image", args.image,
|
||
"--credential-file", str(credential), "--output", str(args.output),
|
||
"--model", args.model, "--agent-timeout", str(args.agent_timeout)]
|
||
return argv + (["--calibration", os.path.abspath(calibration)] if calibration not in (None, "-") else [])
|
||
|
||
|
||
def _finished(proc):
|
||
"""Classify one ended controller from its own receipt."""
|
||
out, err = proc.communicate()
|
||
try:
|
||
row = report.attempt(core._path(json.loads(out)["controller"]))
|
||
except (ValueError, KeyError, TypeError):
|
||
row = dict(status="NO_RECEIPT", exclusion="no_receipt")
|
||
return row, err.strip()[-500:]
|
||
|
||
|
||
def run(args):
|
||
jobs_file = core._path(args.jobs)
|
||
jobs = core._read(jobs_file.parent, jobs_file.name).decode("utf-8")
|
||
batch = core._path(args.output)
|
||
batch.mkdir(parents=True, exist_ok=True)
|
||
lock = os.open(batch / ".batch.lock", os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
|
||
try:
|
||
try:
|
||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) # The kernel drops it however this process ends.
|
||
except BlockingIOError:
|
||
raise ValueError("another rollout.batch is already writing " + str(batch)) from None
|
||
return _drive(args, batch, jobs)
|
||
finally:
|
||
os.close(lock)
|
||
|
||
|
||
def _drive(args, batch, jobs):
|
||
running, recent, pause, interrupted, seen = {}, collections.deque(maxlen=PAUSE_AFTER), None, False, {}
|
||
counts = collections.Counter()
|
||
with core._sigterm():
|
||
try:
|
||
while True:
|
||
if pause is None and core._disk_problem(batch):
|
||
pause = "输出盘剩余不足 {} GiB:腾出空间后重跑同一命令".format(core.MIN_FREE_BYTES // 1024 ** 3)
|
||
owed = [] if pause else report.pending(batch, args.tasks, jobs, args.retries,
|
||
[task for task, _ in running.values()], seen)
|
||
for line in owed[:max(0, args.parallel - len(running))]:
|
||
fields = line.split()
|
||
# Each key has its own rate limit, so a new attempt takes the least busy one.
|
||
busy = collections.Counter(key for _, key in running.values())
|
||
key = min(args.credential_file, key=lambda path: busy[path])
|
||
# Own session: a terminal Ctrl-C reaches only this runner, which forwards one SIGTERM.
|
||
proc = subprocess.Popen(_argv(args.tasks / fields[0], (fields + [None])[1], args, key), cwd=ROOT,
|
||
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
||
stderr=subprocess.PIPE, text=True, start_new_session=True)
|
||
running[proc] = fields[0], key
|
||
if not running:
|
||
break
|
||
while all(proc.poll() is None for proc in running):
|
||
time.sleep(2)
|
||
for proc in [proc for proc in running if proc.poll() is not None]:
|
||
task = running.pop(proc)[0]
|
||
row, err = _finished(proc)
|
||
outcome = row["status"] if row.get("complete") else row["exclusion"]
|
||
counts[outcome] += 1
|
||
recent.append(row["exclusion"] in BROKEN)
|
||
print("{} {} | {}".format(task, outcome, dict(counts)), flush=True)
|
||
if pause is None and len(recent) == PAUSE_AFTER and all(recent):
|
||
pause = ("最近 {} 次尝试都因环境原因失败(最后一次:{}{}):检查模型服务、Docker 与校准后"
|
||
"重跑同一命令").format(PAUSE_AFTER, outcome,
|
||
";" + err if err and outcome == "no_receipt" else "")
|
||
except BaseException as error:
|
||
# Children run in their own sessions, so they are stopped here whatever ended the loop.
|
||
interrupted = True
|
||
for proc in running:
|
||
proc.terminate()
|
||
while running:
|
||
try:
|
||
for proc in list(running):
|
||
proc.wait()
|
||
running.pop(proc)
|
||
except KeyboardInterrupt:
|
||
continue # Never abandon a controller that is still cleaning up.
|
||
if not isinstance(error, KeyboardInterrupt):
|
||
raise
|
||
result = report.report(batch, args.tasks)
|
||
stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") # Reruns never collide.
|
||
core._create(batch / ("report-" + stamp + ".json"),
|
||
(json.dumps(result, indent=2, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8"))
|
||
print(json.dumps(dict(complete=result["complete_trajectories"], passed=result["passed"], failed=result["failed"],
|
||
exclusions=result["exclusions"], report="report-" + stamp + ".json"), ensure_ascii=False))
|
||
_, exported = sft.export([batch], batch / ("export-" + stamp))
|
||
print(json.dumps(dict(export="export-" + stamp, passed=exported["passed"], failed=exported["failed"],
|
||
skipped=len(exported["skipped"]))))
|
||
if interrupted:
|
||
print("已中断:重跑同一命令即可续跑", file=sys.stderr)
|
||
return 2
|
||
if pause:
|
||
print("已暂停:" + pause, file=sys.stderr)
|
||
return 3
|
||
return 0
|
||
|
||
|
||
def main(argv=None):
|
||
parser = argparse.ArgumentParser(description="Run every JOBS line through one controller until it is paid off")
|
||
parser.add_argument("jobs", type=Path, help="lines of 'task calibration'; '-' calibrates automatically")
|
||
parser.add_argument("--image", required=True)
|
||
parser.add_argument("--credential-file", type=Path, required=True, action="append",
|
||
help="repeat to spread attempts over several model keys")
|
||
parser.add_argument("--backend", choices=("docker", "salty"), default="docker",
|
||
help="docker runs the Agent on this host; salty runs it in a Salty job")
|
||
parser.add_argument("--output", type=Path, required=True, help="this batch's own directory")
|
||
parser.add_argument("--tasks", type=Path, default=Path("tasks"))
|
||
parser.add_argument("--model", choices=MODEL_IDS, default=MODEL)
|
||
parser.add_argument("--agent-timeout", type=int, default=900)
|
||
parser.add_argument("--parallel", type=int, default=2)
|
||
parser.add_argument("--retries", type=int, default=2, help="infrastructure resamples allowed per JOBS line")
|
||
args = parser.parse_args(argv)
|
||
if args.parallel < 1 or args.retries < 0 or args.agent_timeout < 1:
|
||
parser.error("--parallel and --agent-timeout must be positive and --retries not negative")
|
||
# Controllers run from the project root, so every path is fixed against the caller's directory first.
|
||
args.tasks, args.output = core._path(args.tasks), core._path(args.output)
|
||
args.credential_file = [Path(os.path.abspath(path)) for path in args.credential_file]
|
||
try:
|
||
return run(args)
|
||
except (ValueError, OSError) as error:
|
||
print("batch rejected: " + str(error), file=sys.stderr)
|
||
return 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|