data-gen-bpu/scripts/health_check.py
bench a0e86e71ea health_check: alert on board_offline stage failures and on unreachable board ports
10-23 08:00-11:00 all 8 boards behind rdk-sandbox refused connections; board/eval steps failed as
board_offline for 3 hours and nothing in the check covered it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-23 11:19:15 +08:00

94 lines
4.9 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Mac mini 主控的健康检查:有问题每条打印一行,健康时什么都不打印。只读。
用法(bench 虚拟机里):python3 scripts/health_check.py RUN [RUN ...]
由本机 scripts/bench_health_alert.sh 每 10 分钟经 ssh 调用,有输出就弹通知。"""
import json
import os
import shutil
import socket
import sqlite3
import subprocess
import sys
import time
B = os.environ.get("BENCH_ROOT", "/mnt/data/ming01.wang/model-compile-hbdk/bench")
QUOTA = os.path.expanduser("~/.local/state/bench-shared/quota.sqlite")
INFRA = ("REMOTE_WORKER_FAILED", "STAGE_EXECUTOR_EXIT", "QUEUE_DEADLINE_EXCEEDED")
def problems(runs, now):
ps = subprocess.run(["ps", "-eo", "args"], capture_output=True, text=True).stdout
for run in runs:
if f"collect --run {run} " not in ps:
yield f"{run} 的调度程序没在跑"
c = sqlite3.connect(f"file:{B}/state/jobs.sqlite?mode=ro", uri=True, timeout=30)
q = ",".join("?" * len(runs))
for run, job, kk in c.execute(f"SELECT run_id, job_id, kk FROM rollouts WHERE state='PAUSED_INFRA' AND run_id IN ({q})", runs):
try:
e = json.load(open(f"{B}/runs/{run}/{job}/rollouts/{kk:02d}/execution.json"))
except (OSError, ValueError):
continue
paused = now - (e.get("paused_at") or 0)
if 3600 < paused < 86400: # 正常的暂停几分钟内就会自动续;超过一天的是老问题,不反复报
yield f"{run[-2:]} {job} #{kk} 暂停 {paused // 60:.0f} 分钟没恢复({e.get('stop_reason')})"
for run in runs:
alive = c.execute("SELECT COUNT(*) FROM rollouts WHERE run_id=? AND state='RUNNING'", (run,)).fetchone()[0]
last = c.execute("SELECT MAX(started_at) FROM stages WHERE run_id=?", (run,)).fetchone()[0] or 0
if alive and now - last > 3 * 3600:
yield f"{run} 有 {alive} 条在跑,但 {(now - last) / 3600:.0f} 小时没有开始任何新步骤,调度程序可能卡住了"
n = c.execute(f"SELECT COUNT(*) FROM stages WHERE run_id IN ({q}) AND state='FAIL' AND started_at>?"
f" AND error_class IN ({','.join('?' * len(INFRA))})", (*runs, now - 3600, *INFRA)).fetchone()[0]
if n >= 8:
yield f"最近 1 小时 Salty 容器失败 {n} 次"
n = c.execute(f"SELECT COUNT(*) FROM rollouts WHERE run_id IN ({q}) AND finished_at>?"
" AND state IN ('INFRA_ERROR','ABANDONED','LOST','CANCELLED')", (*runs, now - 3600)).fetchone()[0]
if n >= 5:
yield f"最近 1 小时有 {n} 条轨迹被系统中断"
n = c.execute(f"SELECT COUNT(*) FROM stages WHERE run_id IN ({q}) AND state='FAIL' AND started_at>?"
" AND error_class='board_offline'", (*runs, now - 3600)).fetchone()[0]
if n >= 3:
yield f"最近 1 小时上板失败 {n} 次(板卡离线)"
boards = board_endpoints()
down = [f"{h}:{p}" for h, p in boards if not reachable(h, p)]
if boards and len(down) == len(boards):
yield f"{len(boards)} 块板卡全部连不上({', '.join(sorted({d.rsplit(':', 1)[0] for d in down}))}),找板卡网关管理员"
elif down:
yield f"{len(down)}/{len(boards)} 块板卡连不上:端口 {', '.join(d.rsplit(':', 1)[1] for d in down)}"
qc = sqlite3.connect(f"file:{QUOTA}?mode=ro", uri=True, timeout=30)
for group, state, updated in qc.execute(
"SELECT group_id, state, updated_at FROM quota_groups WHERE state IN ('PAUSED','RECONCILING')"
" AND group_id IN (SELECT group_id FROM attempts WHERE started_at>?)", (now - 86400,)):
if now - updated > 2700: # 请求结果不明时本来就会整组核对约半小时,超过 45 分钟才算卡住
yield f"模型账号 {group} 处于 {state} 已 {(now - updated) / 60:.0f} 分钟,用这个账号的轨迹都发不出请求"
free = shutil.disk_usage(B).free / 2**30
if free < 10:
yield f"虚拟机磁盘只剩 {free:.0f} GB"
mem = int(next(l.split()[1] for l in open("/proc/meminfo") if l.startswith("MemAvailable"))) // 1024
if mem < 1024:
yield f"虚拟机可用内存只剩 {mem} MB,调度程序可能被系统杀掉"
def board_endpoints():
"""BOARD*_HOST/PORT pairs from .env.local (host and port only; keys and users are never read out)."""
env = {}
try:
for line in open(f"{B}/.env.local"):
k, _, v = line.strip().partition("=")
if k.startswith("BOARD") and k.endswith(("_HOST", "_PORT")):
env[k] = v.strip().strip('"')
except OSError:
return []
return [(env[k], int(env[k[:-5] + "_PORT"])) for k in env if k.endswith("_HOST") and env.get(k[:-5] + "_PORT", "").isdigit()]
def reachable(host, port):
try:
socket.create_connection((host, port), timeout=5).close()
return True
except OSError:
return False
if __name__ == "__main__":
for line in problems(sys.argv[1:], time.time()):
print(line)