fix: 系统失败不占编译次数、连不上可自动续跑、暂停轨迹不被新题抢名额;加健康检查与排空重启脚本

- stagerun: 容器被杀/执行器退出不计入 5 次编译(每条最多免 3 次)
- quota: CONNECT_FAILED / REQUEST_DEADLINE_BEFORE_SEND 证明未发送,允许自动续跑
- pipeline: 有暂停轨迹只差额度时,空出的名额留给它
- scripts/health_check.py + bench_health_alert.sh: 每 10 分钟检查,异常弹本机通知
- scripts/drain_restart.sh: 排空旧调度程序后重启,排空期间用新代码续跑卡住的轨迹

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bench 2026-10-12 16:38:07 +08:00
parent 54c716fabb
commit 12787a289d
9 changed files with 198 additions and 8 deletions

View File

@ -809,7 +809,8 @@ def collect_rollouts(run_id: str, job_ids: list[str] | None = None, modes: list[
# A fully paused group has no request left to drive its own recovery.
collection_runtime._broker.reconcile()
resumes, resume_pending = _resume_snapshot(run_id, job_ids, modes)
ready = resumes + ready
# 有暂停的轨迹只差额度时,空出的名额留给它;否则新轨迹总抢在它就绪之前,它会一直饿到 24h 过期。
ready = resumes + ([] if resume_pending and not resumes else ready)
pending = pending or resume_pending
active_jobs = {r["job_id"] for r in futures.values()}
free = n - len(futures)

View File

@ -670,7 +670,8 @@ class Broker:
result['generation_id']=gen['generation_id']
policy=self.groups[ex['group_id']]
if (ex['cancelled'] or gen['state']!='PAUSED'
or gen['reason'] not in ('QUOTA_WAIT_BUDGET_EXHAUSTED','REQUEST_OUTCOME_UNKNOWN')
# CONNECT_FAILED / REQUEST_DEADLINE_BEFORE_SEND: _not_sent proved no bytes left, replay is safe.
or gen['reason'] not in ('QUOTA_WAIT_BUDGET_EXHAUSTED','REQUEST_OUTCOME_UNKNOWN','CONNECT_FAILED','REQUEST_DEADLINE_BEFORE_SEND')
# A generation whose own request end is unknown never replays.
or con.execute('SELECT 1 FROM attempts WHERE generation_id=? AND ended_at IS NULL',(gen['generation_id'],)).fetchone()):
return result | {'reason':'RESUME_NOT_ELIGIBLE'}

View File

@ -116,13 +116,19 @@ def _completion_status(ctx, name, out):
MAX_COMPILE_ATTEMPTS = 5 # 每条轨迹的主预算;改这里要同步改 agent/prompts/e2e.md 里的次数
# 容器被杀(多是内存超限,下次自动加倍)、执行器退出:系统的错,不扣模型的次数;
# 每条轨迹最多免这么多次,防止一个怎么加内存都编不过的模型无限重试。
FREE_INFRA_COMPILES = 3
def compile_attempts_used(con, run_id, job_id, rollout) -> int:
"""Compiles that got a slot; a compile that timed out in the queue never ran and is free.
Read from the stage ledger, so resume/recovery cannot reset it."""
return con.execute("SELECT COUNT(*) FROM stages WHERE run_id=? AND job_id=? AND rollout=? "
"AND stage='05-compile' AND lane!='queued'", (run_id, job_id, rollout)).fetchone()[0]
total, infra = con.execute(
"SELECT COUNT(*), COALESCE(SUM(error_class IN ('REMOTE_WORKER_FAILED','STAGE_EXECUTOR_EXIT')),0) FROM stages "
"WHERE run_id=? AND job_id=? AND rollout=? AND stage='05-compile' AND lane!='queued'",
(run_id, job_id, rollout)).fetchone()
return total - min(infra, FREE_INFRA_COMPILES)
def _exec_impl(hd: Path, *, rd=None, publish_status=None, protected=False) -> int:

34
scripts/bench_health_alert.sh Executable file
View File

@ -0,0 +1,34 @@
#!/bin/bash
# 本机每 10 分钟(launchd)查一次 Mac mini 上的采集;有问题弹系统通知,同样的问题每 2 小时再提醒一次,恢复了也通知。
# 安装:cp 到 ~/.bench-health/,plist 放 ~/Library/LaunchAgents/cc.d-robotics.bench-health.plist。记录在 ~/.bench-health/log。
MINI=amos@192.168.66.251 # Mac mini 换了地址改这里
RUNS="r20260929-04 r20260929-05 r20260929-06" # 加新批次时加到这里
D=~/.bench-health; mkdir -p "$D"
notify() {
osascript -e 'on run argv' -e 'display notification (item 1 of argv) with title (item 2 of argv) sound name "Glass"' -e 'end run' "$1" "$2"
printf '%s %s\n%s\n' "$(date '+%m-%d %H:%M')" "$2" "$1" >> "$D/alerts.log"
}
if ! ssh -o ConnectTimeout=15 -o BatchMode=yes "$MINI" true 2>/dev/null; then
fails=$(( $(cat "$D/ssh_fails" 2>/dev/null || echo 0) + 1 )); echo $fails > "$D/ssh_fails"
[ $fails -ge 2 ] && problems="Mac mini($MINI)连不上,已 $((fails * 10)) 分钟。Wi-Fi 掉了、换了地址或关机了" # Wi-Fi 抖一下不报
else
rm -f "$D/ssh_fails"
out=$(ssh -o ConnectTimeout=15 -o BatchMode=yes "$MINI" \
"export PATH=/opt/homebrew/bin:\$PATH; limactl list --format '{{.Status}}' bench && limactl shell bench -- python3 /mnt/data/ming01.wang/model-compile-hbdk/bench/scripts/health_check.py $RUNS" 2>&1)
rc=$?
status=$(head -1 <<<"$out")
if [ "$status" != Running ]; then problems="bench 虚拟机没在运行($status),Mac mini 可能重启过,需要恢复"
elif [ $rc != 0 ]; then problems="健康检查本身出错:$(tail -1 <<<"$out")"
else problems=$(tail -n +2 <<<"$out"); fi
fi
key=$(tr -d '0-9' <<<"$problems") # 去掉数字,分钟数变了不算新问题
now=$(date +%s); last=$(cat "$D/last_t" 2>/dev/null || echo 0)
if [ -n "$problems" ]; then
if [ "$key" != "$(cat "$D/last_key" 2>/dev/null)" ] || [ $((now - last)) -ge 7200 ]; then
notify "$problems" "采集报警($(wc -l <<<"$problems" | tr -d ' ') 项)"
echo "$key" > "$D/last_key"; echo $now > "$D/last_t"
fi
elif [ -s "$D/last_key" ]; then
notify "之前报的问题都已消失" "采集恢复正常"; rm -f "$D/last_key"
fi
echo "$(date '+%m-%d %H:%M') ${problems:-ok}" | head -1 >> "$D/log"

59
scripts/drain_restart.sh Executable file
View File

@ -0,0 +1,59 @@
#!/bin/bash
# 排空旧调度程序后换新代码重启,不丢在跑的轨迹。在 Mac mini 的 bench 虚拟机里后台跑:
# setsid nohup scripts/drain_restart.sh "<run> <旧pid> <配置名> <新并发> [题单文件]" ... >> logs/drain-restart.log 2>&1 &
# 排空期间:腾出的名额先用新代码逐条续跑"连不上"卡住的轨迹(旧代码永远不会续它们)。
# 旧进程没有子进程、也没有校验中的轨迹满 10 分钟,说明卡在最后的导出上,按 bench-restart-hazards 第 5 条 kill -9。
set -u
B=/mnt/data/ming01.wang/model-compile-hbdk/bench; C=/mnt/data/ming01.wang/configs; L=/mnt/data/ming01.wang/logs
export BENCH_ROOT=$B BENCH_STATE_DIR=$B/state
export PATH=/home/amos.guest/.local/lib/salty-controller/python/bin:/home/amos.guest/.local/lib/salty-controller/runtime/bin:/home/amos.guest/.local/bin:/usr/local/bin:/usr/bin:/bin
log() { echo "$(date '+%m-%d %H:%M:%S') $*"; }
SPECS=("$@"); declare -A idle done_
for s in "${SPECS[@]}"; do set -- $s; kill -TERM "$2" && log "SIGTERM $1 pid $2,开始排空"; done
while [ ${#done_[@]} -lt ${#SPECS[@]} ]; do
for s in "${SPECS[@]}"; do
set -- $s; run=$1 pid=$2 cfg=$3 n=$4 jobs=${5:-}
[ -n "${done_[$run]:-}" ] && continue
if kill -0 "$pid" 2>/dev/null; then
busy=$(python3 -c "import sqlite3,sys; print(sqlite3.connect('file:$B/state/jobs.sqlite?mode=ro',uri=True).execute(\"SELECT COUNT(*) FROM rollouts WHERE run_id=? AND state IN ('VERIFYING','CLEANING')\",(sys.argv[1],)).fetchone()[0])" "$run")
if [ -z "$(pgrep -P "$pid")" ] && [ "$busy" = 0 ]; then idle[$run]=$(( ${idle[$run]:-0} + 1 )); else idle[$run]=0; fi
if [ "${idle[$run]}" -ge 10 ]; then
log "$run 旧进程 $pid 已无在跑轨迹 10 分钟仍未退出(卡在导出),kill -9"
kill -9 "$pid"; sleep 5; rm -rf "$B/datasets/$run-training"/.refresh-*
continue
fi
# 每轮最多续跑一条;内存不够或全局名额满就等下一轮。
pick=$(python3 - "$run" <<'EOF'
import json, os, sqlite3, sys
run = sys.argv[1]; B = os.environ["BENCH_ROOT"]
mem = int(next(l.split()[1] for l in open("/proc/meminfo") if l.startswith("MemAvailable"))) // 1024
c = sqlite3.connect(f"file:{B}/state/jobs.sqlite?mode=ro", uri=True)
if mem < 2500 or c.execute("SELECT COUNT(*) FROM rollouts WHERE state IN ('RUNNING','VERIFYING')").fetchone()[0] >= 40:
sys.exit()
for job, kk in c.execute("SELECT job_id, kk FROM rollouts WHERE run_id=? AND state='PAUSED_INFRA'", (run,)):
if os.path.exists(f"/mnt/data/ming01.wang/logs/resume-{run}-{job}-{kk}.log"):
continue # 每条只试一次
try:
e = json.load(open(f"{B}/runs/{run}/{job}/rollouts/{kk:02d}/execution.json"))
except (OSError, ValueError):
continue
if e.get("stop_reason") in ("CONNECT_FAILED", "REQUEST_DEADLINE_BEFORE_SEND") and e.get("cleanup_state") != "QUARANTINED":
print(job, kk); break
EOF
)
if [ -n "$pick" ]; then
set -- $pick
log "$run 用新代码续跑 $1 #$2"
(cd $B && BENCH_CONFIG=$C/mini/$cfg.yaml setsid nohup bin/bench execution resume --run "$run" --job "$1" --kk "$2" \
> "$L/resume-$run-$1-$2.log" 2>&1 < /dev/null &)
fi
continue
fi
args="--run $run --modes e2e --parallel $n --watch"
[ -n "$jobs" ] && args="$args $(sed 's/^/--job /' "$jobs" | tr '\n' ' ')"
log "$run 旧进程已退出,起新进程:$($C/run-collect.sh "$cfg" $args)"
done_[$run]=1
done
sleep 60
done
log "全部重启完成"

61
scripts/health_check.py Normal file
View File

@ -0,0 +1,61 @@
"""Mac mini 主控的健康检查:有问题每条打印一行,健康时什么都不打印。只读。
用法(bench 虚拟机里):python3 scripts/health_check.py RUN [RUN ...]
由本机 scripts/bench_health_alert.sh 每 10 分钟经 ssh 调用,有输出就弹通知。"""
import json
import os
import shutil
import sqlite3
import subprocess
import sys
import time
B = os.environ.get("BENCH_ROOT", "/mnt/data/ming01.wang/model-compile-hbdk/bench")
QUOTA = os.path.expanduser("~/.local/state/bench-shared/quota.sqlite")
INFRA = ("REMOTE_WORKER_FAILED", "STAGE_EXECUTOR_EXIT", "QUEUE_DEADLINE_EXCEEDED")
def problems(runs, now):
ps = subprocess.run(["ps", "-eo", "args"], capture_output=True, text=True).stdout
for run in runs:
if f"collect --run {run} " not in ps:
yield f"{run} 的调度程序没在跑"
c = sqlite3.connect(f"file:{B}/state/jobs.sqlite?mode=ro", uri=True, timeout=30)
q = ",".join("?" * len(runs))
for run, job, kk in c.execute(f"SELECT run_id, job_id, kk FROM rollouts WHERE state='PAUSED_INFRA' AND run_id IN ({q})", runs):
try:
e = json.load(open(f"{B}/runs/{run}/{job}/rollouts/{kk:02d}/execution.json"))
except (OSError, ValueError):
continue
paused = now - (e.get("paused_at") or 0)
if 3600 < paused < 86400: # 正常的暂停几分钟内就会自动续;超过一天的是老问题,不反复报
yield f"{run[-2:]} {job} #{kk} 暂停 {paused // 60:.0f} 分钟没恢复({e.get('stop_reason')})"
for run in runs:
alive = c.execute("SELECT COUNT(*) FROM rollouts WHERE run_id=? AND state='RUNNING'", (run,)).fetchone()[0]
last = c.execute("SELECT MAX(started_at) FROM stages WHERE run_id=?", (run,)).fetchone()[0] or 0
if alive and now - last > 3 * 3600:
yield f"{run} 有 {alive} 条在跑,但 {(now - last) / 3600:.0f} 小时没有开始任何新步骤,调度程序可能卡住了"
n = c.execute(f"SELECT COUNT(*) FROM stages WHERE run_id IN ({q}) AND state='FAIL' AND started_at>?"
f" AND error_class IN ({','.join('?' * len(INFRA))})", (*runs, now - 3600, *INFRA)).fetchone()[0]
if n >= 8:
yield f"最近 1 小时 Salty 容器失败 {n} 次"
n = c.execute(f"SELECT COUNT(*) FROM rollouts WHERE run_id IN ({q}) AND finished_at>?"
" AND state IN ('INFRA_ERROR','ABANDONED','LOST','CANCELLED')", (*runs, now - 3600)).fetchone()[0]
if n >= 5:
yield f"最近 1 小时有 {n} 条轨迹被系统中断"
qc = sqlite3.connect(f"file:{QUOTA}?mode=ro", uri=True, timeout=30)
for group, state, updated in qc.execute(
"SELECT group_id, state, updated_at FROM quota_groups WHERE state IN ('PAUSED','RECONCILING')"
" AND group_id IN (SELECT group_id FROM attempts WHERE started_at>?)", (now - 86400,)):
if now - updated > 2700: # 请求结果不明时本来就会整组核对约半小时,超过 45 分钟才算卡住
yield f"模型账号 {group} 处于 {state} 已 {(now - updated) / 60:.0f} 分钟,用这个账号的轨迹都发不出请求"
free = shutil.disk_usage(B).free / 2**30
if free < 10:
yield f"虚拟机磁盘只剩 {free:.0f} GB"
mem = int(next(l.split()[1] for l in open("/proc/meminfo") if l.startswith("MemAvailable"))) // 1024
if mem < 1024:
yield f"虚拟机可用内存只剩 {mem} MB,调度程序可能被系统杀掉"
if __name__ == "__main__":
for line in problems(sys.argv[1:], time.time()):
print(line)

View File

@ -19,6 +19,16 @@ class CompileBudget(unittest.TestCase):
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 2)
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 1), 1)
def test_infra_kills_are_free_up_to_a_cap(self):
con = sqlite3.connect(':memory:')
con.executescript(db.SCHEMA)
errors = ['REMOTE_WORKER_FAILED', 'STAGE_EXECUTOR_EXIT', None, 'unknown',
'REMOTE_WORKER_FAILED', 'REMOTE_WORKER_FAILED'] # 4 infra kills, 2 real compiles
for attempt, error in enumerate(errors):
con.execute("INSERT INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, error_class, started_at)"
" VALUES ('r', 'j', 0, '05-compile', ?, 'agent-long', 'FAIL', ?, 0)", (attempt, error))
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 2 + 4 - stagerun.FREE_INFRA_COMPILES)
def test_compile_past_the_cap_is_refused_before_any_work(self):
import json, tempfile
from contextlib import nullcontext

View File

@ -561,6 +561,24 @@ class Quota(unittest.TestCase):
self.assertEqual(hashlib.sha256(self.ledger.read_bytes()).hexdigest(),before)
self.assertEqual(snap['groups'][0]['inflight'],0)
def test_connect_failure_pause_resumes_after_cooldown(self):
p=self.provider();b=self.broker(p,self.policy())
scope=b.issue_scope('blip','teacher','glm-test')
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
def refused(connection):
if connection.port==p.server.server_port:
raise ConnectionRefusedError('injected Wi-Fi blip')
return original(connection)
with patch('harness.quota.http.client.HTTPConnection.connect',new=refused):
self.assertEqual(self.request(scope,'g')[0],503)
with b._db() as con:
cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0]
with patch.object(b,'_now',return_value=max(b._now(),cooldown)+1):
self.assertTrue(b.resume_readiness('blip')['eligible'])
b.resume('blip','g',checkpoint_complete=True,side_effects_known=True)
self.assertEqual(self.request(scope,'g')[0],200)
self.assertEqual(len(p.requests),1)
def test_confirmed_connect_failure_does_not_spend_http_budget(self):
p=self.provider();b=self.broker(p,self.policy(rpm=1,window_s=1))
scope=b.issue_scope('connect-failure','teacher','glm-test')

View File

@ -337,7 +337,7 @@ class StageTests(unittest.TestCase):
connection = MagicMock()
# The final paid attempt still owns its job and must remain recoverable
# when COUNT(*) reaches the cap; it must never create another attempt.
connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5),)
connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5), 0) # (compiles, infra kills)
with patch.object(db, "connect", return_value=nullcontext(connection)), \
patch.object(db, "stage_start"), patch.object(db, "stage_finish"), \
patch.object(stagerun.scheduler, "acquire_stage", return_value=nullcontext(0)), \
@ -369,7 +369,7 @@ class StageTests(unittest.TestCase):
(queue / 'request.json').write_text(json.dumps({'stage': '05-compile', 'config': None}))
original = salty_stage.run
connection = MagicMock()
connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5),)
connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5), 0) # (compiles, infra kills)
with patch.object(db, 'connect', return_value=nullcontext(connection)), \
patch.object(db, 'stage_start'), patch.object(db, 'stage_finish'), \
patch.object(stagerun.scheduler, 'acquire_stage', return_value=nullcontext(0)), \
@ -420,7 +420,7 @@ class StageTests(unittest.TestCase):
queue.mkdir(parents=True)
(queue / 'request.json').write_text(json.dumps({'stage': '05-compile', 'config': None}))
connection = MagicMock()
connection.execute.return_value.fetchone.return_value = (stagerun.MAX_COMPILE_ATTEMPTS,)
connection.execute.return_value.fetchone.return_value = (stagerun.MAX_COMPILE_ATTEMPTS, 0) # (compiles, infra kills)
with patch.object(db, 'connect', return_value=nullcontext(connection)), \
patch.object(stagerun.scheduler, 'acquire_stage', side_effect=AssertionError('budget denies dispatch')), \
patch.object(salty_stage, 'run', side_effect=AssertionError('budget denies first submission')):
@ -466,7 +466,7 @@ class StageTests(unittest.TestCase):
queue.mkdir(parents=True)
(queue / 'request.json').write_text(json.dumps({'stage': '05-compile', 'config': None}))
connection = MagicMock()
connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5),)
connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5), 0) # (compiles, infra kills)
original, recover = salty_stage.run, SaltyClient.recover_recorded_submission
with patch.object(db, 'connect', return_value=nullcontext(connection)), \
patch.object(db, 'stage_start'), patch.object(db, 'stage_finish'), \