data-gen-bpu/scripts/health_check.py
bench 12787a289d fix: 系统失败不占编译次数、连不上可自动续跑、暂停轨迹不被新题抢名额;加健康检查与排空重启脚本
- stagerun: 容器被杀/执行器退出不计入 5 次编译(每条最多免 3 次)
- quota: CONNECT_FAILED / REQUEST_DEADLINE_BEFORE_SEND 证明未发送,允许自动续跑
- pipeline: 有暂停轨迹只差额度时,空出的名额留给它
- scripts/health_check.py + bench_health_alert.sh: 每 10 分钟检查,异常弹本机通知
- scripts/drain_restart.sh: 排空旧调度程序后重启,排空期间用新代码续跑卡住的轨迹

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-12 16:38:07 +08:00

62 lines
3.5 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Mac mini 主控的健康检查:有问题每条打印一行,健康时什么都不打印。只读。
用法(bench 虚拟机里):python3 scripts/health_check.py RUN [RUN ...]
由本机 scripts/bench_health_alert.sh 每 10 分钟经 ssh 调用,有输出就弹通知。"""
import json
import os
import shutil
import sqlite3
import subprocess
import sys
import time
B = os.environ.get("BENCH_ROOT", "/mnt/data/ming01.wang/model-compile-hbdk/bench")
QUOTA = os.path.expanduser("~/.local/state/bench-shared/quota.sqlite")
INFRA = ("REMOTE_WORKER_FAILED", "STAGE_EXECUTOR_EXIT", "QUEUE_DEADLINE_EXCEEDED")
def problems(runs, now):
ps = subprocess.run(["ps", "-eo", "args"], capture_output=True, text=True).stdout
for run in runs:
if f"collect --run {run} " not in ps:
yield f"{run} 的调度程序没在跑"
c = sqlite3.connect(f"file:{B}/state/jobs.sqlite?mode=ro", uri=True, timeout=30)
q = ",".join("?" * len(runs))
for run, job, kk in c.execute(f"SELECT run_id, job_id, kk FROM rollouts WHERE state='PAUSED_INFRA' AND run_id IN ({q})", runs):
try:
e = json.load(open(f"{B}/runs/{run}/{job}/rollouts/{kk:02d}/execution.json"))
except (OSError, ValueError):
continue
paused = now - (e.get("paused_at") or 0)
if 3600 < paused < 86400: # 正常的暂停几分钟内就会自动续;超过一天的是老问题,不反复报
yield f"{run[-2:]} {job} #{kk} 暂停 {paused // 60:.0f} 分钟没恢复({e.get('stop_reason')})"
for run in runs:
alive = c.execute("SELECT COUNT(*) FROM rollouts WHERE run_id=? AND state='RUNNING'", (run,)).fetchone()[0]
last = c.execute("SELECT MAX(started_at) FROM stages WHERE run_id=?", (run,)).fetchone()[0] or 0
if alive and now - last > 3 * 3600:
yield f"{run} 有 {alive} 条在跑,但 {(now - last) / 3600:.0f} 小时没有开始任何新步骤,调度程序可能卡住了"
n = c.execute(f"SELECT COUNT(*) FROM stages WHERE run_id IN ({q}) AND state='FAIL' AND started_at>?"
f" AND error_class IN ({','.join('?' * len(INFRA))})", (*runs, now - 3600, *INFRA)).fetchone()[0]
if n >= 8:
yield f"最近 1 小时 Salty 容器失败 {n} 次"
n = c.execute(f"SELECT COUNT(*) FROM rollouts WHERE run_id IN ({q}) AND finished_at>?"
" AND state IN ('INFRA_ERROR','ABANDONED','LOST','CANCELLED')", (*runs, now - 3600)).fetchone()[0]
if n >= 5:
yield f"最近 1 小时有 {n} 条轨迹被系统中断"
qc = sqlite3.connect(f"file:{QUOTA}?mode=ro", uri=True, timeout=30)
for group, state, updated in qc.execute(
"SELECT group_id, state, updated_at FROM quota_groups WHERE state IN ('PAUSED','RECONCILING')"
" AND group_id IN (SELECT group_id FROM attempts WHERE started_at>?)", (now - 86400,)):
if now - updated > 2700: # 请求结果不明时本来就会整组核对约半小时,超过 45 分钟才算卡住
yield f"模型账号 {group} 处于 {state} 已 {(now - updated) / 60:.0f} 分钟,用这个账号的轨迹都发不出请求"
free = shutil.disk_usage(B).free / 2**30
if free < 10:
yield f"虚拟机磁盘只剩 {free:.0f} GB"
mem = int(next(l.split()[1] for l in open("/proc/meminfo") if l.startswith("MemAvailable"))) // 1024
if mem < 1024:
yield f"虚拟机可用内存只剩 {mem} MB,调度程序可能被系统杀掉"
if __name__ == "__main__":
for line in problems(sys.argv[1:], time.time()):
print(line)