amos 3adab31da7 Self-heal rollouts: retry stalled model streams and run batches from receipts
Batch 6 lost 26 of 103 attempts to Qwen requests that sent their preamble and
then stalled; Pi's own Node fetch gave up after 300 s and reported "terminated".

- model_proxy: hold the SSE preamble until the first generation event, retry a
  stall (180 s) within the existing retry budget, and send SSE keepalive
  comments so Pi never times out first; record first_event_seconds.
- report: name one exclusion reason per incomplete attempt and compute the
  JOBS lines still owed, with a per-line resample budget.
- batch: run a JOBS file to completion, resume from receipts, resample
  infrastructure failures, pause after three environment failures in a row,
  forward SIGTERM to running controllers, and refuse a second runner.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 08:28:53 +08:00

144 lines
6.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Run a JOBS file to completion: resume from receipts, resample infrastructure failures and pause when the
environment is broken."""
import argparse
import collections
import datetime
import fcntl
import json
import os
from pathlib import Path
import subprocess
import sys
import time
from . import core, report
from .model_proxy import MODEL, MODEL_IDS
ROOT = Path(__file__).resolve().parents[1]
# Failures that say the environment, not the Agent, is broken; this many in a row pauses the batch.
BROKEN = report.RESAMPLE | {"calibration", "no_receipt"}
PAUSE_AFTER = 3
def _argv(task, calibration, args):
argv = [sys.executable, "-m", "rollout.pi_run", str(task), "--image", args.image,
"--credential-file", str(args.credential_file), "--output", str(args.output),
"--model", args.model, "--agent-timeout", str(args.agent_timeout)]
return argv + (["--calibration", os.path.abspath(calibration)] if calibration not in (None, "-") else [])
def _finished(proc):
"""Classify one ended controller from its own receipt."""
out, err = proc.communicate()
try:
row = report.attempt(core._path(json.loads(out)["controller"]))
except (ValueError, KeyError, TypeError):
row = dict(status="NO_RECEIPT", exclusion="no_receipt")
return row, err.strip()[-500:]
def run(args):
jobs_file = core._path(args.jobs)
jobs = core._read(jobs_file.parent, jobs_file.name).decode("utf-8")
batch = core._path(args.output)
batch.mkdir(parents=True, exist_ok=True)
lock = os.open(batch / ".batch.lock", os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW, 0o600)
try:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) # The kernel drops it however this process ends.
except BlockingIOError:
raise ValueError("another rollout.batch is already writing " + str(batch)) from None
return _drive(args, batch, jobs)
finally:
os.close(lock)
def _drive(args, batch, jobs):
running, recent, pause, interrupted, seen = {}, collections.deque(maxlen=PAUSE_AFTER), None, False, {}
counts = collections.Counter()
with core._sigterm():
try:
while True:
if pause is None and core._disk_problem(batch):
pause = "输出盘剩余不足 {} GiB:腾出空间后重跑同一命令".format(core.MIN_FREE_BYTES // 1024 ** 3)
owed = [] if pause else report.pending(batch, args.tasks, jobs, args.retries,
list(running.values()), seen)
for line in owed[:max(0, args.parallel - len(running))]:
fields = line.split()
# Own session: a terminal Ctrl-C reaches only this runner, which forwards one SIGTERM.
proc = subprocess.Popen(_argv(args.tasks / fields[0], (fields + [None])[1], args), cwd=ROOT,
stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, text=True, start_new_session=True)
running[proc] = fields[0]
if not running:
break
while all(proc.poll() is None for proc in running):
time.sleep(2)
for proc in [proc for proc in running if proc.poll() is not None]:
task = running.pop(proc)
row, err = _finished(proc)
outcome = row["status"] if row.get("complete") else row["exclusion"]
counts[outcome] += 1
recent.append(row["exclusion"] in BROKEN)
print("{} {} | {}".format(task, outcome, dict(counts)), flush=True)
if pause is None and len(recent) == PAUSE_AFTER and all(recent):
pause = ("最近 {} 次尝试都因环境原因失败(最后一次:{}{}):检查模型服务、Docker 与校准后"
"重跑同一命令").format(PAUSE_AFTER, outcome,
";" + err if err and outcome == "no_receipt" else "")
except BaseException as error:
# Children run in their own sessions, so they are stopped here whatever ended the loop.
interrupted = True
for proc in running:
proc.terminate()
while running:
try:
for proc in list(running):
proc.wait()
running.pop(proc)
except KeyboardInterrupt:
continue # Never abandon a controller that is still cleaning up.
if not isinstance(error, KeyboardInterrupt):
raise
result = report.report(batch, args.tasks)
stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") # Reruns never collide.
core._create(batch / ("report-" + stamp + ".json"),
(json.dumps(result, indent=2, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8"))
print(json.dumps(dict(complete=result["complete_trajectories"], passed=result["passed"], failed=result["failed"],
exclusions=result["exclusions"], report="report-" + stamp + ".json"), ensure_ascii=False))
if interrupted:
print("已中断:重跑同一命令即可续跑", file=sys.stderr)
return 2
if pause:
print("已暂停:" + pause, file=sys.stderr)
return 3
return 0
def main(argv=None):
parser = argparse.ArgumentParser(description="Run every JOBS line through rollout.pi_run until it is paid off")
parser.add_argument("jobs", type=Path, help="lines of 'task calibration'; '-' calibrates automatically")
parser.add_argument("--image", required=True)
parser.add_argument("--credential-file", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True, help="this batch's own directory")
parser.add_argument("--tasks", type=Path, default=Path("tasks"))
parser.add_argument("--model", choices=MODEL_IDS, default=MODEL)
parser.add_argument("--agent-timeout", type=int, default=900)
parser.add_argument("--parallel", type=int, default=2)
parser.add_argument("--retries", type=int, default=2, help="infrastructure resamples allowed per JOBS line")
args = parser.parse_args(argv)
if args.parallel < 1 or args.retries < 0 or args.agent_timeout < 1:
parser.error("--parallel and --agent-timeout must be positive and --retries not negative")
# Controllers run from the project root, so every path is fixed against the caller's directory first.
args.tasks, args.output = core._path(args.tasks), core._path(args.output)
args.credential_file = Path(os.path.abspath(args.credential_file))
try:
return run(args)
except (ValueError, OSError) as error:
print("batch rejected: " + str(error), file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main())