diff --git a/harness/pipeline.py b/harness/pipeline.py index da2bae3..96bb407 100644 --- a/harness/pipeline.py +++ b/harness/pipeline.py @@ -809,7 +809,8 @@ def collect_rollouts(run_id: str, job_ids: list[str] | None = None, modes: list[ # A fully paused group has no request left to drive its own recovery. collection_runtime._broker.reconcile() resumes, resume_pending = _resume_snapshot(run_id, job_ids, modes) - ready = resumes + ready + # 有暂停的轨迹只差额度时,空出的名额留给它;否则新轨迹总抢在它就绪之前,它会一直饿到 24h 过期。 + ready = resumes + ([] if resume_pending and not resumes else ready) pending = pending or resume_pending active_jobs = {r["job_id"] for r in futures.values()} free = n - len(futures) diff --git a/harness/quota.py b/harness/quota.py index 4450095..2f27bc8 100644 --- a/harness/quota.py +++ b/harness/quota.py @@ -670,7 +670,8 @@ class Broker: result['generation_id']=gen['generation_id'] policy=self.groups[ex['group_id']] if (ex['cancelled'] or gen['state']!='PAUSED' - or gen['reason'] not in ('QUOTA_WAIT_BUDGET_EXHAUSTED','REQUEST_OUTCOME_UNKNOWN') + # CONNECT_FAILED / REQUEST_DEADLINE_BEFORE_SEND: _not_sent proved no bytes left, replay is safe. + or gen['reason'] not in ('QUOTA_WAIT_BUDGET_EXHAUSTED','REQUEST_OUTCOME_UNKNOWN','CONNECT_FAILED','REQUEST_DEADLINE_BEFORE_SEND') # A generation whose own request end is unknown never replays. or con.execute('SELECT 1 FROM attempts WHERE generation_id=? AND ended_at IS NULL',(gen['generation_id'],)).fetchone()): return result | {'reason':'RESUME_NOT_ELIGIBLE'} diff --git a/harness/stagerun.py b/harness/stagerun.py index 0fba37e..1db72df 100644 --- a/harness/stagerun.py +++ b/harness/stagerun.py @@ -116,13 +116,19 @@ def _completion_status(ctx, name, out): MAX_COMPILE_ATTEMPTS = 5 # 每条轨迹的主预算;改这里要同步改 agent/prompts/e2e.md 里的次数 +# 容器被杀(多是内存超限,下次自动加倍)、执行器退出:系统的错,不扣模型的次数; +# 每条轨迹最多免这么多次,防止一个怎么加内存都编不过的模型无限重试。 +FREE_INFRA_COMPILES = 3 def compile_attempts_used(con, run_id, job_id, rollout) -> int: """Compiles that got a slot; a compile that timed out in the queue never ran and is free. Read from the stage ledger, so resume/recovery cannot reset it.""" - return con.execute("SELECT COUNT(*) FROM stages WHERE run_id=? AND job_id=? AND rollout=? " - "AND stage='05-compile' AND lane!='queued'", (run_id, job_id, rollout)).fetchone()[0] + total, infra = con.execute( + "SELECT COUNT(*), COALESCE(SUM(error_class IN ('REMOTE_WORKER_FAILED','STAGE_EXECUTOR_EXIT')),0) FROM stages " + "WHERE run_id=? AND job_id=? AND rollout=? AND stage='05-compile' AND lane!='queued'", + (run_id, job_id, rollout)).fetchone() + return total - min(infra, FREE_INFRA_COMPILES) def _exec_impl(hd: Path, *, rd=None, publish_status=None, protected=False) -> int: diff --git a/scripts/bench_health_alert.sh b/scripts/bench_health_alert.sh new file mode 100755 index 0000000..8b44319 --- /dev/null +++ b/scripts/bench_health_alert.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# 本机每 10 分钟(launchd)查一次 Mac mini 上的采集;有问题弹系统通知,同样的问题每 2 小时再提醒一次,恢复了也通知。 +# 安装:cp 到 ~/.bench-health/,plist 放 ~/Library/LaunchAgents/cc.d-robotics.bench-health.plist。记录在 ~/.bench-health/log。 +MINI=amos@192.168.66.251 # Mac mini 换了地址改这里 +RUNS="r20260929-04 r20260929-05 r20260929-06" # 加新批次时加到这里 +D=~/.bench-health; mkdir -p "$D" +notify() { + osascript -e 'on run argv' -e 'display notification (item 1 of argv) with title (item 2 of argv) sound name "Glass"' -e 'end run' "$1" "$2" + printf '%s %s\n%s\n' "$(date '+%m-%d %H:%M')" "$2" "$1" >> "$D/alerts.log" +} +if ! ssh -o ConnectTimeout=15 -o BatchMode=yes "$MINI" true 2>/dev/null; then + fails=$(( $(cat "$D/ssh_fails" 2>/dev/null || echo 0) + 1 )); echo $fails > "$D/ssh_fails" + [ $fails -ge 2 ] && problems="Mac mini($MINI)连不上,已 $((fails * 10)) 分钟。Wi-Fi 掉了、换了地址或关机了" # Wi-Fi 抖一下不报 +else + rm -f "$D/ssh_fails" + out=$(ssh -o ConnectTimeout=15 -o BatchMode=yes "$MINI" \ + "export PATH=/opt/homebrew/bin:\$PATH; limactl list --format '{{.Status}}' bench && limactl shell bench -- python3 /mnt/data/ming01.wang/model-compile-hbdk/bench/scripts/health_check.py $RUNS" 2>&1) + rc=$? + status=$(head -1 <<<"$out") + if [ "$status" != Running ]; then problems="bench 虚拟机没在运行($status),Mac mini 可能重启过,需要恢复" + elif [ $rc != 0 ]; then problems="健康检查本身出错:$(tail -1 <<<"$out")" + else problems=$(tail -n +2 <<<"$out"); fi +fi +key=$(tr -d '0-9' <<<"$problems") # 去掉数字,分钟数变了不算新问题 +now=$(date +%s); last=$(cat "$D/last_t" 2>/dev/null || echo 0) +if [ -n "$problems" ]; then + if [ "$key" != "$(cat "$D/last_key" 2>/dev/null)" ] || [ $((now - last)) -ge 7200 ]; then + notify "$problems" "采集报警($(wc -l <<<"$problems" | tr -d ' ') 项)" + echo "$key" > "$D/last_key"; echo $now > "$D/last_t" + fi +elif [ -s "$D/last_key" ]; then + notify "之前报的问题都已消失" "采集恢复正常"; rm -f "$D/last_key" +fi +echo "$(date '+%m-%d %H:%M') ${problems:-ok}" | head -1 >> "$D/log" diff --git a/scripts/drain_restart.sh b/scripts/drain_restart.sh new file mode 100755 index 0000000..ff0a893 --- /dev/null +++ b/scripts/drain_restart.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# 排空旧调度程序后换新代码重启,不丢在跑的轨迹。在 Mac mini 的 bench 虚拟机里后台跑: +# setsid nohup scripts/drain_restart.sh " <旧pid> <配置名> <新并发> [题单文件]" ... >> logs/drain-restart.log 2>&1 & +# 排空期间:腾出的名额先用新代码逐条续跑"连不上"卡住的轨迹(旧代码永远不会续它们)。 +# 旧进程没有子进程、也没有校验中的轨迹满 10 分钟,说明卡在最后的导出上,按 bench-restart-hazards 第 5 条 kill -9。 +set -u +B=/mnt/data/ming01.wang/model-compile-hbdk/bench; C=/mnt/data/ming01.wang/configs; L=/mnt/data/ming01.wang/logs +export BENCH_ROOT=$B BENCH_STATE_DIR=$B/state +export PATH=/home/amos.guest/.local/lib/salty-controller/python/bin:/home/amos.guest/.local/lib/salty-controller/runtime/bin:/home/amos.guest/.local/bin:/usr/local/bin:/usr/bin:/bin +log() { echo "$(date '+%m-%d %H:%M:%S') $*"; } +SPECS=("$@"); declare -A idle done_ +for s in "${SPECS[@]}"; do set -- $s; kill -TERM "$2" && log "SIGTERM $1 pid $2,开始排空"; done +while [ ${#done_[@]} -lt ${#SPECS[@]} ]; do + for s in "${SPECS[@]}"; do + set -- $s; run=$1 pid=$2 cfg=$3 n=$4 jobs=${5:-} + [ -n "${done_[$run]:-}" ] && continue + if kill -0 "$pid" 2>/dev/null; then + busy=$(python3 -c "import sqlite3,sys; print(sqlite3.connect('file:$B/state/jobs.sqlite?mode=ro',uri=True).execute(\"SELECT COUNT(*) FROM rollouts WHERE run_id=? AND state IN ('VERIFYING','CLEANING')\",(sys.argv[1],)).fetchone()[0])" "$run") + if [ -z "$(pgrep -P "$pid")" ] && [ "$busy" = 0 ]; then idle[$run]=$(( ${idle[$run]:-0} + 1 )); else idle[$run]=0; fi + if [ "${idle[$run]}" -ge 10 ]; then + log "$run 旧进程 $pid 已无在跑轨迹 10 分钟仍未退出(卡在导出),kill -9" + kill -9 "$pid"; sleep 5; rm -rf "$B/datasets/$run-training"/.refresh-* + continue + fi + # 每轮最多续跑一条;内存不够或全局名额满就等下一轮。 + pick=$(python3 - "$run" <<'EOF' +import json, os, sqlite3, sys +run = sys.argv[1]; B = os.environ["BENCH_ROOT"] +mem = int(next(l.split()[1] for l in open("/proc/meminfo") if l.startswith("MemAvailable"))) // 1024 +c = sqlite3.connect(f"file:{B}/state/jobs.sqlite?mode=ro", uri=True) +if mem < 2500 or c.execute("SELECT COUNT(*) FROM rollouts WHERE state IN ('RUNNING','VERIFYING')").fetchone()[0] >= 40: + sys.exit() +for job, kk in c.execute("SELECT job_id, kk FROM rollouts WHERE run_id=? AND state='PAUSED_INFRA'", (run,)): + if os.path.exists(f"/mnt/data/ming01.wang/logs/resume-{run}-{job}-{kk}.log"): + continue # 每条只试一次 + try: + e = json.load(open(f"{B}/runs/{run}/{job}/rollouts/{kk:02d}/execution.json")) + except (OSError, ValueError): + continue + if e.get("stop_reason") in ("CONNECT_FAILED", "REQUEST_DEADLINE_BEFORE_SEND") and e.get("cleanup_state") != "QUARANTINED": + print(job, kk); break +EOF +) + if [ -n "$pick" ]; then + set -- $pick + log "$run 用新代码续跑 $1 #$2" + (cd $B && BENCH_CONFIG=$C/mini/$cfg.yaml setsid nohup bin/bench execution resume --run "$run" --job "$1" --kk "$2" \ + > "$L/resume-$run-$1-$2.log" 2>&1 < /dev/null &) + fi + continue + fi + args="--run $run --modes e2e --parallel $n --watch" + [ -n "$jobs" ] && args="$args $(sed 's/^/--job /' "$jobs" | tr '\n' ' ')" + log "$run 旧进程已退出,起新进程:$($C/run-collect.sh "$cfg" $args)" + done_[$run]=1 + done + sleep 60 +done +log "全部重启完成" diff --git a/scripts/health_check.py b/scripts/health_check.py new file mode 100644 index 0000000..b82cc06 --- /dev/null +++ b/scripts/health_check.py @@ -0,0 +1,61 @@ +"""Mac mini 主控的健康检查:有问题每条打印一行,健康时什么都不打印。只读。 +用法(bench 虚拟机里):python3 scripts/health_check.py RUN [RUN ...] +由本机 scripts/bench_health_alert.sh 每 10 分钟经 ssh 调用,有输出就弹通知。""" +import json +import os +import shutil +import sqlite3 +import subprocess +import sys +import time + +B = os.environ.get("BENCH_ROOT", "/mnt/data/ming01.wang/model-compile-hbdk/bench") +QUOTA = os.path.expanduser("~/.local/state/bench-shared/quota.sqlite") +INFRA = ("REMOTE_WORKER_FAILED", "STAGE_EXECUTOR_EXIT", "QUEUE_DEADLINE_EXCEEDED") + + +def problems(runs, now): + ps = subprocess.run(["ps", "-eo", "args"], capture_output=True, text=True).stdout + for run in runs: + if f"collect --run {run} " not in ps: + yield f"{run} 的调度程序没在跑" + c = sqlite3.connect(f"file:{B}/state/jobs.sqlite?mode=ro", uri=True, timeout=30) + q = ",".join("?" * len(runs)) + for run, job, kk in c.execute(f"SELECT run_id, job_id, kk FROM rollouts WHERE state='PAUSED_INFRA' AND run_id IN ({q})", runs): + try: + e = json.load(open(f"{B}/runs/{run}/{job}/rollouts/{kk:02d}/execution.json")) + except (OSError, ValueError): + continue + paused = now - (e.get("paused_at") or 0) + if 3600 < paused < 86400: # 正常的暂停几分钟内就会自动续;超过一天的是老问题,不反复报 + yield f"{run[-2:]} {job} #{kk} 暂停 {paused // 60:.0f} 分钟没恢复({e.get('stop_reason')})" + for run in runs: + alive = c.execute("SELECT COUNT(*) FROM rollouts WHERE run_id=? AND state='RUNNING'", (run,)).fetchone()[0] + last = c.execute("SELECT MAX(started_at) FROM stages WHERE run_id=?", (run,)).fetchone()[0] or 0 + if alive and now - last > 3 * 3600: + yield f"{run} 有 {alive} 条在跑,但 {(now - last) / 3600:.0f} 小时没有开始任何新步骤,调度程序可能卡住了" + n = c.execute(f"SELECT COUNT(*) FROM stages WHERE run_id IN ({q}) AND state='FAIL' AND started_at>?" + f" AND error_class IN ({','.join('?' * len(INFRA))})", (*runs, now - 3600, *INFRA)).fetchone()[0] + if n >= 8: + yield f"最近 1 小时 Salty 容器失败 {n} 次" + n = c.execute(f"SELECT COUNT(*) FROM rollouts WHERE run_id IN ({q}) AND finished_at>?" + " AND state IN ('INFRA_ERROR','ABANDONED','LOST','CANCELLED')", (*runs, now - 3600)).fetchone()[0] + if n >= 5: + yield f"最近 1 小时有 {n} 条轨迹被系统中断" + qc = sqlite3.connect(f"file:{QUOTA}?mode=ro", uri=True, timeout=30) + for group, state, updated in qc.execute( + "SELECT group_id, state, updated_at FROM quota_groups WHERE state IN ('PAUSED','RECONCILING')" + " AND group_id IN (SELECT group_id FROM attempts WHERE started_at>?)", (now - 86400,)): + if now - updated > 2700: # 请求结果不明时本来就会整组核对约半小时,超过 45 分钟才算卡住 + yield f"模型账号 {group} 处于 {state} 已 {(now - updated) / 60:.0f} 分钟,用这个账号的轨迹都发不出请求" + free = shutil.disk_usage(B).free / 2**30 + if free < 10: + yield f"虚拟机磁盘只剩 {free:.0f} GB" + mem = int(next(l.split()[1] for l in open("/proc/meminfo") if l.startswith("MemAvailable"))) // 1024 + if mem < 1024: + yield f"虚拟机可用内存只剩 {mem} MB,调度程序可能被系统杀掉" + + +if __name__ == "__main__": + for line in problems(sys.argv[1:], time.time()): + print(line) diff --git a/scripts/test_compile_budget.py b/scripts/test_compile_budget.py index a21f57b..795a247 100644 --- a/scripts/test_compile_budget.py +++ b/scripts/test_compile_budget.py @@ -19,6 +19,16 @@ class CompileBudget(unittest.TestCase): self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 2) self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 1), 1) + def test_infra_kills_are_free_up_to_a_cap(self): + con = sqlite3.connect(':memory:') + con.executescript(db.SCHEMA) + errors = ['REMOTE_WORKER_FAILED', 'STAGE_EXECUTOR_EXIT', None, 'unknown', + 'REMOTE_WORKER_FAILED', 'REMOTE_WORKER_FAILED'] # 4 infra kills, 2 real compiles + for attempt, error in enumerate(errors): + con.execute("INSERT INTO stages (run_id, job_id, rollout, stage, attempt, lane, state, error_class, started_at)" + " VALUES ('r', 'j', 0, '05-compile', ?, 'agent-long', 'FAIL', ?, 0)", (attempt, error)) + self.assertEqual(stagerun.compile_attempts_used(con, 'r', 'j', 0), 2 + 4 - stagerun.FREE_INFRA_COMPILES) + def test_compile_past_the_cap_is_refused_before_any_work(self): import json, tempfile from contextlib import nullcontext diff --git a/scripts/test_quota.py b/scripts/test_quota.py index 239e48e..3a85e4b 100644 --- a/scripts/test_quota.py +++ b/scripts/test_quota.py @@ -561,6 +561,24 @@ class Quota(unittest.TestCase): self.assertEqual(hashlib.sha256(self.ledger.read_bytes()).hexdigest(),before) self.assertEqual(snap['groups'][0]['inflight'],0) + def test_connect_failure_pause_resumes_after_cooldown(self): + p=self.provider();b=self.broker(p,self.policy()) + scope=b.issue_scope('blip','teacher','glm-test') + original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect + def refused(connection): + if connection.port==p.server.server_port: + raise ConnectionRefusedError('injected Wi-Fi blip') + return original(connection) + with patch('harness.quota.http.client.HTTPConnection.connect',new=refused): + self.assertEqual(self.request(scope,'g')[0],503) + with b._db() as con: + cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0] + with patch.object(b,'_now',return_value=max(b._now(),cooldown)+1): + self.assertTrue(b.resume_readiness('blip')['eligible']) + b.resume('blip','g',checkpoint_complete=True,side_effects_known=True) + self.assertEqual(self.request(scope,'g')[0],200) + self.assertEqual(len(p.requests),1) + def test_confirmed_connect_failure_does_not_spend_http_budget(self): p=self.provider();b=self.broker(p,self.policy(rpm=1,window_s=1)) scope=b.issue_scope('connect-failure','teacher','glm-test') diff --git a/scripts/test_salty_stage.py b/scripts/test_salty_stage.py index ea38773..2709d95 100644 --- a/scripts/test_salty_stage.py +++ b/scripts/test_salty_stage.py @@ -337,7 +337,7 @@ class StageTests(unittest.TestCase): connection = MagicMock() # The final paid attempt still owns its job and must remain recoverable # when COUNT(*) reaches the cap; it must never create another attempt. - connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5),) + connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5), 0) # (compiles, infra kills) with patch.object(db, "connect", return_value=nullcontext(connection)), \ patch.object(db, "stage_start"), patch.object(db, "stage_finish"), \ patch.object(stagerun.scheduler, "acquire_stage", return_value=nullcontext(0)), \ @@ -369,7 +369,7 @@ class StageTests(unittest.TestCase): (queue / 'request.json').write_text(json.dumps({'stage': '05-compile', 'config': None})) original = salty_stage.run connection = MagicMock() - connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5),) + connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5), 0) # (compiles, infra kills) with patch.object(db, 'connect', return_value=nullcontext(connection)), \ patch.object(db, 'stage_start'), patch.object(db, 'stage_finish'), \ patch.object(stagerun.scheduler, 'acquire_stage', return_value=nullcontext(0)), \ @@ -420,7 +420,7 @@ class StageTests(unittest.TestCase): queue.mkdir(parents=True) (queue / 'request.json').write_text(json.dumps({'stage': '05-compile', 'config': None})) connection = MagicMock() - connection.execute.return_value.fetchone.return_value = (stagerun.MAX_COMPILE_ATTEMPTS,) + connection.execute.return_value.fetchone.return_value = (stagerun.MAX_COMPILE_ATTEMPTS, 0) # (compiles, infra kills) with patch.object(db, 'connect', return_value=nullcontext(connection)), \ patch.object(stagerun.scheduler, 'acquire_stage', side_effect=AssertionError('budget denies dispatch')), \ patch.object(salty_stage, 'run', side_effect=AssertionError('budget denies first submission')): @@ -466,7 +466,7 @@ class StageTests(unittest.TestCase): queue.mkdir(parents=True) (queue / 'request.json').write_text(json.dumps({'stage': '05-compile', 'config': None})) connection = MagicMock() - connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5),) + connection.execute.return_value.fetchone.return_value = (getattr(stagerun, 'MAX_COMPILE_ATTEMPTS', 5), 0) # (compiles, infra kills) original, recover = salty_stage.run, SaltyClient.recover_recorded_submission with patch.object(db, 'connect', return_value=nullcontext(connection)), \ patch.object(db, 'stage_start'), patch.object(db, 'stage_finish'), \