amos c64c79b31d Run the Pi agent in Salty jobs over a one-time SSH tunnel and export SFT rows
rollout.salty_run keeps the gateway, grading and evidence on 36 and moves only the
agent step into a Salty job. The gateway listens on 127.0.0.1 with a per-attempt
bearer token; the job reaches it over port 22 with a restricted one-time key that is
removed after each attempt. rollout.batch gains --backend salty and repeatable
--credential-file, and writes pass/fail SFT exports through rollout.sft at the end.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-28 17:45:28 +08:00

154 lines
7.8 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Run a JOBS file to completion: resume from receipts, resample infrastructure failures and pause when the
environment is broken."""
import argparse
import collections
import datetime
import fcntl
import json
import os
from pathlib import Path
import subprocess
import sys
import time
from . import core, report, sft
from .model_proxy import MODEL, MODEL_IDS
ROOT = Path(__file__).resolve().parents[1]
# Failures that say the environment, not the Agent, is broken; this many in a row pauses the batch.
BROKEN = report.RESAMPLE | {"calibration", "no_receipt"}
PAUSE_AFTER = 3
def _argv(task, calibration, args, credential):
module = "rollout.salty_run" if args.backend == "salty" else "rollout.pi_run"
argv = [sys.executable, "-m", module, str(task), "--image", args.image,
"--credential-file", str(credential), "--output", str(args.output),
"--model", args.model, "--agent-timeout", str(args.agent_timeout)]
return argv + (["--calibration", os.path.abspath(calibration)] if calibration not in (None, "-") else [])
def _finished(proc):
"""Classify one ended controller from its own receipt."""
out, err = proc.communicate()
try:
row = report.attempt(core._path(json.loads(out)["controller"]))
except (ValueError, KeyError, TypeError):
row = dict(status="NO_RECEIPT", exclusion="no_receipt")
return row, err.strip()[-500:]
def run(args):
jobs_file = core._path(args.jobs)
jobs = core._read(jobs_file.parent, jobs_file.name).decode("utf-8")
batch = core._path(args.output)
batch.mkdir(parents=True, exist_ok=True)
lock = os.open(batch / ".batch.lock", os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
try:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) # The kernel drops it however this process ends.
except BlockingIOError:
raise ValueError("another rollout.batch is already writing " + str(batch)) from None
return _drive(args, batch, jobs)
finally:
os.close(lock)
def _drive(args, batch, jobs):
running, recent, pause, interrupted, seen = {}, collections.deque(maxlen=PAUSE_AFTER), None, False, {}
counts = collections.Counter()
with core._sigterm():
try:
while True:
if pause is None and core._disk_problem(batch):
pause = "输出盘剩余不足 {} GiB:腾出空间后重跑同一命令".format(core.MIN_FREE_BYTES // 1024 ** 3)
owed = [] if pause else report.pending(batch, args.tasks, jobs, args.retries,
[task for task, _ in running.values()], seen)
for line in owed[:max(0, args.parallel - len(running))]:
fields = line.split()
# Each key has its own rate limit, so a new attempt takes the least busy one.
busy = collections.Counter(key for _, key in running.values())
key = min(args.credential_file, key=lambda path: busy[path])
# Own session: a terminal Ctrl-C reaches only this runner, which forwards one SIGTERM.
proc = subprocess.Popen(_argv(args.tasks / fields[0], (fields + [None])[1], args, key), cwd=ROOT,
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, text=True, start_new_session=True)
running[proc] = fields[0], key
if not running:
break
while all(proc.poll() is None for proc in running):
time.sleep(2)
for proc in [proc for proc in running if proc.poll() is not None]:
task = running.pop(proc)[0]
row, err = _finished(proc)
outcome = row["status"] if row.get("complete") else row["exclusion"]
counts[outcome] += 1
recent.append(row["exclusion"] in BROKEN)
print("{} {} | {}".format(task, outcome, dict(counts)), flush=True)
if pause is None and len(recent) == PAUSE_AFTER and all(recent):
pause = ("最近 {} 次尝试都因环境原因失败(最后一次:{}{}):检查模型服务、Docker 与校准后"
"重跑同一命令").format(PAUSE_AFTER, outcome,
";" + err if err and outcome == "no_receipt" else "")
except BaseException as error:
# Children run in their own sessions, so they are stopped here whatever ended the loop.
interrupted = True
for proc in running:
proc.terminate()
while running:
try:
for proc in list(running):
proc.wait()
running.pop(proc)
except KeyboardInterrupt:
continue # Never abandon a controller that is still cleaning up.
if not isinstance(error, KeyboardInterrupt):
raise
result = report.report(batch, args.tasks)
stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") # Reruns never collide.
core._create(batch / ("report-" + stamp + ".json"),
(json.dumps(result, indent=2, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8"))
print(json.dumps(dict(complete=result["complete_trajectories"], passed=result["passed"], failed=result["failed"],
exclusions=result["exclusions"], report="report-" + stamp + ".json"), ensure_ascii=False))
_, exported = sft.export([batch], batch / ("export-" + stamp))
print(json.dumps(dict(export="export-" + stamp, passed=exported["passed"], failed=exported["failed"],
skipped=len(exported["skipped"]))))
if interrupted:
print("已中断:重跑同一命令即可续跑", file=sys.stderr)
return 2
if pause:
print("已暂停:" + pause, file=sys.stderr)
return 3
return 0
def main(argv=None):
parser = argparse.ArgumentParser(description="Run every JOBS line through one controller until it is paid off")
parser.add_argument("jobs", type=Path, help="lines of 'task calibration'; '-' calibrates automatically")
parser.add_argument("--image", required=True)
parser.add_argument("--credential-file", type=Path, required=True, action="append",
help="repeat to spread attempts over several model keys")
parser.add_argument("--backend", choices=("docker", "salty"), default="docker",
help="docker runs the Agent on this host; salty runs it in a Salty job")
parser.add_argument("--output", type=Path, required=True, help="this batch's own directory")
parser.add_argument("--tasks", type=Path, default=Path("tasks"))
parser.add_argument("--model", choices=MODEL_IDS, default=MODEL)
parser.add_argument("--agent-timeout", type=int, default=900)
parser.add_argument("--parallel", type=int, default=2)
parser.add_argument("--retries", type=int, default=2, help="infrastructure resamples allowed per JOBS line")
args = parser.parse_args(argv)
if args.parallel < 1 or args.retries < 0 or args.agent_timeout < 1:
parser.error("--parallel and --agent-timeout must be positive and --retries not negative")
# Controllers run from the project root, so every path is fixed against the caller's directory first.
args.tasks, args.output = core._path(args.tasks), core._path(args.output)
args.credential_file = [Path(os.path.abspath(path)) for path in args.credential_file]
try:
return run(args)
except (ValueError, OSError) as error:
print("batch rejected: " + str(error), file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())