fix: 系统失败不占编译次数、连不上可自动续跑、暂停轨迹不被新题抢名额;加健康检查与排空重启脚本
- stagerun: 容器被杀/执行器退出不计入 5 次编译(每条最多免 3 次) - quota: CONNECT_FAILED / REQUEST_DEADLINE_BEFORE_SEND 证明未发送,允许自动续跑 - pipeline: 有暂停轨迹只差额度时,空出的名额留给它 - scripts/health_check.py + bench_health_alert.sh: 每 10 分钟检查,异常弹本机通知 - scripts/drain_restart.sh: 排空旧调度程序后重启,排空期间用新代码续跑卡住的轨迹 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
54c716fabb
commit
12787a289d
@ -809,7 +809,8 @@ def collect_rollouts(run_id: str, job_ids: list[str] | None = None, modes: list[
|
||||
# A fully paused group has no request left to drive its own recovery.
|
||||
collection_runtime._broker.reconcile()
|
||||
resumes, resume_pending = _resume_snapshot(run_id, job_ids, modes)
|
||||
ready = resumes + ready
|
||||
# 有暂停的轨迹只差额度时,空出的名额留给它;否则新轨迹总抢在它就绪之前,它会一直饿到 24h 过期。
|
||||
ready = resumes + ([] if resume_pending and not resumes else ready)
|
||||
pending = pending or resume_pending
|
||||
active_jobs = {r["job_id"] for r in futures.values()}
|
||||
free = n - len(futures)
|
||||
|
||||
@ -670,7 +670,8 @@ class Broker:
|
||||
result['generation_id']=gen['generation_id']
|
||||
policy=self.groups[ex['group_id']]
|
||||
if (ex['cancelled'] or gen['state']!='PAUSED'
|
||||
or gen['reason'] not in ('QUOTA_WAIT_BUDGET_EXHAUSTED','REQUEST_OUTCOME_UNKNOWN')
|
||||
# CONNECT_FAILED / REQUEST_DEADLINE_BEFORE_SEND: _not_sent proved no bytes left, replay is safe.
|
||||
or gen['reason'] not in ('QUOTA_WAIT_BUDGET_EXHAUSTED','REQUEST_OUTCOME_UNKNOWN','CONNECT_FAILED','REQUEST_DEADLINE_BEFORE_SEND')
|
||||
# A generation whose own request end is unknown never replays.
|
||||
or con.execute('SELECT 1 FROM attempts WHERE generation_id=? AND ended_at IS NULL',(gen['generation_id'],)).fetchone()):
|
||||
return result | {'reason':'RESUME_NOT_ELIGIBLE'}
|
||||
|
||||
@ -116,13 +116,19 @@ def _completion_status(ctx, name, out):
|
||||
|
||||
|
||||
MAX_COMPILE_ATTEMPTS = 5 # 每条轨迹的主预算;改这里要同步改 agent/prompts/e2e.md 里的次数
|
||||
# 容器被杀(多是内存超限,下次自动加倍)、执行器退出:系统的错,不扣模型的次数;
|
||||
# 每条轨迹最多免这么多次,防止一个怎么加内存都编不过的模型无限重试。
|
||||
FREE_INFRA_COMPILES = 3
|
||||
|
||||
|
||||
def compile_attempts_used(con, run_id, job_id, rollout) -> int:
|
||||
"""Compiles that got a slot; a compile that timed out in the queue never ran and is free.
|
||||
Read from the stage ledger, so resume/recovery cannot reset it."""
|
||||
return con.execute("SELECT COUNT(*) FROM stages WHERE run_id=? AND job_id=? AND rollout=? "
|
||||
"AND stage='05-compile' AND lane!='queued'", (run_id, job_id, rollout)).fetchone()[0]
|
||||
total, infra = con.execute(
|
||||
"SELECT COUNT(*), COALESCE(SUM(error_class IN ('REMOTE_WORKER_FAILED','STAGE_EXECUTOR_EXIT')),0) FROM stages "
|
||||
"WHERE run_id=? AND job_id=? AND rollout=? AND stage='05-compile' AND lane!='queued'",
|
||||
(run_id, job_id, rollout)).fetchone()
|
||||
return total - min(infra, FREE_INFRA_COMPILES)
|
||||
|
||||
|
||||
def _exec_impl(hd: Path, *, rd=None, publish_status=None, protected=False) -> int:
|
||||
|
||||
34
scripts/bench_health_alert.sh
Executable file
34
scripts/bench_health_alert.sh
Executable file
@ -0,0 +1,34 @@
|
||||
#!/bin/bash
|
||||
# 本机每 10 分钟(launchd)查一次 Mac mini 上的采集;有问题弹系统通知,同样的问题每 2 小时再提醒一次,恢复了也通知。
|
||||
# 安装:cp 到 ~/.bench-health/,plist 放 ~/Library/LaunchAgents/cc.d-robotics.bench-health.plist。记录在 ~/.bench-health/log。
|
||||
MINI=amos@192.168.66.251 # Mac mini 换了地址改这里
|
||||
RUNS="r20260929-04 r20260929-05 r20260929-06" # 加新批次时加到这里
|
||||
D=~/.bench-health; mkdir -p "$D"
|
||||
notify() {
|
||||
osascript -e 'on run argv' -e 'display notification (item 1 of argv) with title (item 2 of argv) sound name "Glass"' -e 'end run' "$1" "$2"
|
||||
printf '%s %s\n%s\n' "$(date '+%m-%d %H:%M')" "$2" "$1" >> "$D/alerts.log"
|
||||
}
|
||||
if ! ssh -o ConnectTimeout=15 -o BatchMode=yes "$MINI" true 2>/dev/null; then
|
||||
fails=$(( $(cat "$D/ssh_fails" 2>/dev/null || echo 0) + 1 )); echo $fails > "$D/ssh_fails"
|
||||
[ $fails -ge 2 ] && problems="Mac mini($MINI)连不上,已 $((fails * 10)) 分钟。Wi-Fi 掉了、换了地址或关机了" # Wi-Fi 抖一下不报
|
||||
else
|
||||
rm -f "$D/ssh_fails"
|
||||
out=$(ssh -o ConnectTimeout=15 -o BatchMode=yes "$MINI" \
|
||||
"export PATH=/opt/homebrew/bin:\$PATH; limactl list --format '{{.Status}}' bench && limactl shell bench -- python3 /mnt/data/ming01.wang/model-compile-hbdk/bench/scripts/health_check.py $RUNS" 2>&1)
|
||||
rc=$?
|
||||
status=$(head -1 <<<"$out")
|
||||
if [ "$status" != Running ]; then problems="bench 虚拟机没在运行($status),Mac mini 可能重启过,需要恢复"
|
||||
elif [ $rc != 0 ]; then problems="健康检查本身出错:$(tail -1 <<<"$out")"
|
||||
else problems=$(tail -n +2 <<<"$out"); fi
|
||||
fi
|
||||
key=$(tr -d '0-9' <<<"$problems") # 去掉数字,分钟数变了不算新问题
|
||||
now=$(date +%s); last=$(cat "$D/last_t" 2>/dev/null || echo 0)
|
||||
if [ -n "$problems" ]; then
|
||||
if [ "$key" != "$(cat "$D/last_key" 2>/dev/null)" ] || [ $((now - last)) -ge 7200 ]; then
|
||||
notify "$problems" "采集报警($(wc -l <<<"$problems" | tr -d ' ') 项)"
|
||||
echo "$key" > "$D/last_key"; echo $now > "$D/last_t"
|
||||
fi
|
||||
elif [ -s "$D/last_key" ]; then
|
||||
notify "之前报的问题都已消失" "采集恢复正常"; rm -f "$D/last_key"
|
||||
fi
|
||||
echo "$(date '+%m-%d %H:%M') ${problems:-ok}" | head -1 >> "$D/log"
|
||||
59
scripts/drain_restart.sh
Executable file
59
scripts/drain_restart.sh
Executable file
@ -0,0 +1,59 @@
|
||||
#!/bin/bash
|
||||
# 排空旧调度程序后换新代码重启,不丢在跑的轨迹。在 Mac mini 的 bench 虚拟机里后台跑:
|
||||
# setsid nohup scripts/drain_restart.sh "<run> <旧pid> <配置名> <新并发> [题单文件]" ... >> logs/drain-restart.log 2>&1 &
|
||||
# 排空期间:腾出的名额先用新代码逐条续跑"连不上"卡住的轨迹(旧代码永远不会续它们)。
|
||||
# 旧进程没有子进程、也没有校验中的轨迹满 10 分钟,说明卡在最后的导出上,按 bench-restart-hazards 第 5 条 kill -9。
|
||||
set -u
|
||||
B=/mnt/data/ming01.wang/model-compile-hbdk/bench; C=/mnt/data/ming01.wang/configs; L=/mnt/data/ming01.wang/logs
|
||||
export BENCH_ROOT=$B BENCH_STATE_DIR=$B/state
|
||||
export PATH=/home/amos.guest/.local/lib/salty-controller/python/bin:/home/amos.guest/.local/lib/salty-controller/runtime/bin:/home/amos.guest/.local/bin:/usr/local/bin:/usr/bin:/bin
|
||||
log() { echo "$(date '+%m-%d %H:%M:%S') $*"; }
|
||||
SPECS=("$@"); declare -A idle done_
|
||||
for s in "${SPECS[@]}"; do set -- $s; kill -TERM "$2" && log "SIGTERM $1 pid $2,开始排空"; done
|
||||
while [ ${#done_[@]} -lt ${#SPECS[@]} ]; do
|
||||
for s in "${SPECS[@]}"; do
|
||||
set -- $s; run=$1 pid=$2 cfg=$3 n=$4 jobs=${5:-}
|
||||
[ -n "${done_[$run]:-}" ] && continue
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
busy=$(python3 -c "import sqlite3,sys; print(sqlite3.connect('file:$B/state/jobs.sqlite?mode=ro',uri=True).execute(\"SELECT COUNT(*) FROM rollouts WHERE run_id=? AND state IN ('VERIFYING','CLEANING')\",(sys.argv[1],)).fetchone()[0])" "$run")
|
||||
if [ -z "$(pgrep -P "$pid")" ] && [ "$busy" = 0 ]; then idle[$run]=$(( ${idle[$run]:-0} + 1 )); else idle[$run]=0; fi
|
||||
if [ "${idle[$run]}" -ge 10 ]; then
|
||||
log "$run 旧进程 $pid 已无在跑轨迹 10 分钟仍未退出(卡在导出),kill -9"
|
||||
kill -9 "$pid"; sleep 5; rm -rf "$B/datasets/$run-training"/.refresh-*
|
||||
continue
|
||||
fi
|
||||
# 每轮最多续跑一条;内存不够或全局名额满就等下一轮。
|
||||
pick=$(python3 - "$run" <<'EOF'
|
||||
import json, os, sqlite3, sys
|
||||
run = sys.argv[1]; B = os.environ["BENCH_ROOT"]
|
||||
mem = int(next(l.split()[1] for l in open("/proc/meminfo") if l.startswith("MemAvailable"))) // 1024
|
||||
c = sqlite3.connect(f"file:{B}/state/jobs.sqlite?mode=ro", uri=True)
|
||||
if mem < 2500 or c.execute("SELECT COUNT(*) FROM rollouts WHERE state IN ('RUNNING','VERIFYING')").fetchone()[0] >= 40:
|
||||
sys.exit()
|
||||
for job, kk in c.execute("SELECT job_id, kk FROM rollouts WHERE run_id=? AND state='PAUSED_INFRA'", (run,)):
|
||||
if os.path.exists(f"/mnt/data/ming01.wang/logs/resume-{run}-{job}-{kk}.log"):
|
||||
continue # 每条只试一次
|
||||
try:
|
||||
e = json.load(open(f"{B}/runs/{run}/{job}/rollouts/{kk:02d}/execution.json"))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if e.get("stop_reason") in ("CONNECT_FAILED", "REQUEST_DEADLINE_BEFORE_SEND") and e.get("cleanup_state") != "QUARANTINED":
|
||||
print(job, kk); break
|
||||
EOF
|
||||
)
|
||||
if [ -n "$pick" ]; then
|
||||
set -- $pick
|
||||
log "$run 用新代码续跑 $1 #$2"
|
||||
(cd $B && BENCH_CONFIG=$C/mini/$cfg.yaml setsid nohup bin/bench execution resume --run "$run" --job "$1" --kk "$2" \
|
||||
> "$L/resume-$run-$1-$2.log" 2>&1 < /dev/null &)
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
args="--run $run --modes e2e --parallel $n --watch"
|
||||
[ -n "$jobs" ] && args="$args $(sed 's/^/--job /' "$jobs" | tr '\n' ' ')"
|
||||
log "$run 旧进程已退出,起新进程:$($C/run-collect.sh "$cfg" $args)"
|
||||
done_[$run]=1
|
||||
done
|
||||
sleep 60
|
||||
done
|
||||
log "全部重启完成"
|
||||
61
scripts/health_check.py
Normal file
61
scripts/health_check.py
Normal file
@ -0,0 +1,61 @@
|
||||
"""Mac mini 主控的健康检查:有问题每条打印一行,健康时什么都不打印。只读。
|
||||
用法(bench 虚拟机里):python3 scripts/health_check.py RUN [RUN ...]
|
||||
由本机 scripts/bench_health_alert.sh 每 10 分钟经 ssh 调用,有输出就弹通知。"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
B = os.environ.get("BENCH_ROOT", "/mnt/data/ming01.wang/model-compile-hbdk/bench")
|
||||
QUOTA = os.path.expanduser("~/.local/state/bench-shared/quota.sqlite")
|
||||
INFRA = ("REMOTE_WORKER_FAILED", "STAGE_EXECUTOR_EXIT", "QUEUE_DEADLINE_EXCEEDED")
|
||||
|
||||
|
||||
def problems(runs, now):
|
||||
ps = subprocess.run(["ps", "-eo", "args"], capture_output=True, text=True).stdout
|
||||
for run in runs:
|
||||
if f"collect --run {run} " not in ps:
|
||||
yield f"{run} 的调度程序没在跑"
|
||||
c = sqlite3.connect(f"file:{B}/state/jobs.sqlite?mode=ro", uri=True, timeout=30)
|
||||
q = ",".join("?" * len(runs))
|
||||
for run, job, kk in c.execute(f"SELECT run_id, job_id, kk FROM rollouts WHERE state='PAUSED_INFRA' AND run_id IN ({q})", runs):
|
||||
try:
|
||||
e = json.load(open(f"{B}/runs/{run}/{job}/rollouts/{kk:02d}/execution.json"))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
paused = now - (e.get("paused_at") or 0)
|
||||
if 3600 < paused < 86400: # 正常的暂停几分钟内就会自动续;超过一天的是老问题,不反复报
|
||||
yield f"{run[-2:]} {job} #{kk} 暂停 {paused // 60:.0f} 分钟没恢复({e.get('stop_reason')})"
|
||||
for run in runs:
|
||||
alive = c.execute("SELECT COUNT(*) FROM rollouts WHERE run_id=? AND state='RUNNING'", (run,)).fetchone()[0]
|
||||
last = c.execute("SELECT MAX(started_at) FROM stages WHERE run_id=?", (run,)).fetchone()[0] or 0
|
||||
if alive and now - last > 3 * 3600:
|
||||
yield f"{run} 有 {alive} 条在跑,但 {(now - last) / 3600:.0f} 小时没有开始任何新步骤,调度程序可能卡住了"
|
||||
n = c.execute(f"SELECT COUNT(*) FROM stages WHERE run_id IN ({q}) AND state='FAIL' AND started_at>?"
|
||||
f" AND error_class IN ({','.join('?' * len(INFRA))})", (*runs, now - 3600, *INFRA)).fetchone()[0]
|
||||
if n >= 8:
|
||||
yield f"最近 1 小时 Salty 容器失败 {n} 次"
|
||||
n = c.execute(f"SELECT COUNT(*) FROM rollouts WHERE run_id IN ({q}) AND finished_at>?"
|
||||
" AND state IN ('INFRA_ERROR','ABANDONED','LOST','CANCELLED')", (*runs, now - 3600)).fetchone()[0]
|
||||
if n >= 5:
|
||||
yield f"最近 1 小时有 {n} 条轨迹被系统中断"
|
||||
qc = sqlite3.connect(f"file:{QUOTA}?mode=ro", uri=True, timeout=30)
|
||||
for group, state, updated in qc.execute(
|
||||
"SELECT group_id, state, updated_at FROM quota_groups WHERE state IN ('PAUSED','RECONCILING')"
|
||||
" AND group_id IN (SELECT group_id FROM attempts WHERE started_at>?)", (now - 86400,)):
|
||||
if now - updated > 2700: # 请求结果不明时本来就会整组核对约半小时,超过 45 分钟才算卡住
|
||||
yield f"模型账号 {group} 处于 {state} 已 {(now - updated) / 60:.0f} 分钟,用这个账号的轨迹都发不出请求"
|
||||
free = shutil.disk_usage(B).free / 2**30
|
||||
if free < 10:
|
||||
yield f"虚拟机磁盘只剩 {free:.0f} GB"
|
||||
mem = int(next(l.split()[1] for l in open("/proc/meminfo") if l.startswith("MemAvailable"))) // 1024
|
||||
if mem < 1024:
|
||||
yield f"虚拟机可用内存只剩 {mem} MB,调度程序可能被系统杀掉"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for line in problems(sys.argv[1:], time.time()):
|
||||
print(line)
|
||||
@ -19,6 +19,16 @@ class CompileBudget(unittest.TestCase):
|
||||
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 2)
|
||||
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 1), 1)
|
||||
|
||||
def test_infra_kills_are_free_up_to_a_cap(self):
|
||||
con = sqlite3.connect(':memory:')
|
||||
con.executescript(db.SCHEMA)
|
||||
errors = ['REMOTE_WORKER_FAILED', 'STAGE_EXECUTOR_EXIT', None, 'unknown',
|
||||
'REMOTE_WORKER_FAILED', 'REMOTE_WORKER_FAILED'] # 4 infra kills, 2 real compiles
|
||||
for attempt, error in enumerate(errors):
|
||||
con.execute("INSERT INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, error_class, started_at)"
|
||||
" VALUES ('r', 'j', 0, '05-compile', ?, 'agent-long', 'FAIL', ?, 0)", (attempt, error))
|
||||
self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 2 + 4 - stagerun.FREE_INFRA_COMPILES)
|
||||
|
||||
def test_compile_past_the_cap_is_refused_before_any_work(self):
|
||||
import json, tempfile
|
||||
from contextlib import nullcontext
|
||||
|
||||
@ -561,6 +561,24 @@ class Quota(unittest.TestCase):
|
||||
self.assertEqual(hashlib.sha256(self.ledger.read_bytes()).hexdigest(),before)
|
||||
self.assertEqual(snap['groups'][0]['inflight'],0)
|
||||
|
||||
def test_connect_failure_pause_resumes_after_cooldown(self):
|
||||
p=self.provider();b=self.broker(p,self.policy())
|
||||
scope=b.issue_scope('blip','teacher','glm-test')
|
||||
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
|
||||
def refused(connection):
|
||||
if connection.port==p.server.server_port:
|
||||
raise ConnectionRefusedError('injected Wi-Fi blip')
|
||||
return original(connection)
|
||||
with patch('harness.quota.http.client.HTTPConnection.connect',new=refused):
|
||||
self.assertEqual(self.request(scope,'g')[0],503)
|
||||
with b._db() as con:
|
||||
cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0]
|
||||
with patch.object(b,'_now',return_value=max(b._now(),cooldown)+1):
|
||||
self.assertTrue(b.resume_readiness('blip')['eligible'])
|
||||
b.resume('blip','g',checkpoint_complete=True,side_effects_known=True)
|
||||
self.assertEqual(self.request(scope,'g')[0],200)
|
||||
self.assertEqual(len(p.requests),1)
|
||||
|
||||
def test_confirmed_connect_failure_does_not_spend_http_budget(self):
|
||||
p=self.provider();b=self.broker(p,self.policy(rpm=1,window_s=1))
|
||||
scope=b.issue_scope('connect-failure','teacher','glm-test')
|
||||
|
||||
@ -337,7 +337,7 @@ class StageTests(unittest.TestCase):
|
||||
connection = MagicMock()
|
||||
# The final paid attempt still owns its job and must remain recoverable
|
||||
# when COUNT(*) reaches the cap; it must never create another attempt.
|
||||
connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5),)
|
||||
connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5), 0) # (compiles, infra kills)
|
||||
with patch.object(db, "connect", return_value=nullcontext(connection)), \
|
||||
patch.object(db, "stage_start"), patch.object(db, "stage_finish"), \
|
||||
patch.object(stagerun.scheduler, "acquire_stage", return_value=nullcontext(0)), \
|
||||
@ -369,7 +369,7 @@ class StageTests(unittest.TestCase):
|
||||
(queue / 'request.json').write_text(json.dumps({'stage': '05-compile', 'config': None}))
|
||||
original = salty_stage.run
|
||||
connection = MagicMock()
|
||||
connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5),)
|
||||
connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5), 0) # (compiles, infra kills)
|
||||
with patch.object(db, 'connect', return_value=nullcontext(connection)), \
|
||||
patch.object(db, 'stage_start'), patch.object(db, 'stage_finish'), \
|
||||
patch.object(stagerun.scheduler, 'acquire_stage', return_value=nullcontext(0)), \
|
||||
@ -420,7 +420,7 @@ class StageTests(unittest.TestCase):
|
||||
queue.mkdir(parents=True)
|
||||
(queue / 'request.json').write_text(json.dumps({'stage': '05-compile', 'config': None}))
|
||||
connection = MagicMock()
|
||||
connection.execute.return_value.fetchone.return_value = (stagerun.MAX_COMPILE_ATTEMPTS,)
|
||||
connection.execute.return_value.fetchone.return_value = (stagerun.MAX_COMPILE_ATTEMPTS, 0) # (compiles, infra kills)
|
||||
with patch.object(db, 'connect', return_value=nullcontext(connection)), \
|
||||
patch.object(stagerun.scheduler, 'acquire_stage', side_effect=AssertionError('budget denies dispatch')), \
|
||||
patch.object(salty_stage, 'run', side_effect=AssertionError('budget denies first submission')):
|
||||
@ -466,7 +466,7 @@ class StageTests(unittest.TestCase):
|
||||
queue.mkdir(parents=True)
|
||||
(queue / 'request.json').write_text(json.dumps({'stage': '05-compile', 'config': None}))
|
||||
connection = MagicMock()
|
||||
connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5),)
|
||||
connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5), 0) # (compiles, infra kills)
|
||||
original, recover = salty_stage.run, SaltyClient.recover_recorded_submission
|
||||
with patch.object(db, 'connect', return_value=nullcontext(connection)), \
|
||||
patch.object(db, 'stage_start'), patch.object(db, 'stage_finish'), \
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user