10-23 08:00-11:00 all 8 boards behind rdk-sandbox refused connections; board/eval steps failed as board_offline for 3 hours and nothing in the check covered it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
94 lines
4.9 KiB
Python
94 lines
4.9 KiB
Python
"""Mac mini 主控的健康检查:有问题每条打印一行,健康时什么都不打印。只读。
|
||
用法(bench 虚拟机里):python3 scripts/health_check.py RUN [RUN ...]
|
||
由本机 scripts/bench_health_alert.sh 每 10 分钟经 ssh 调用,有输出就弹通知。"""
|
||
import json
|
||
import os
|
||
import shutil
|
||
import socket
|
||
import sqlite3
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
|
||
B = os.environ.get("BENCH_ROOT", "/mnt/data/ming01.wang/model-compile-hbdk/bench")
|
||
QUOTA = os.path.expanduser("~/.local/state/bench-shared/quota.sqlite")
|
||
INFRA = ("REMOTE_WORKER_FAILED", "STAGE_EXECUTOR_EXIT", "QUEUE_DEADLINE_EXCEEDED")
|
||
|
||
|
||
def problems(runs, now):
|
||
ps = subprocess.run(["ps", "-eo", "args"], capture_output=True, text=True).stdout
|
||
for run in runs:
|
||
if f"collect --run {run} " not in ps:
|
||
yield f"{run} 的调度程序没在跑"
|
||
c = sqlite3.connect(f"file:{B}/state/jobs.sqlite?mode=ro", uri=True, timeout=30)
|
||
q = ",".join("?" * len(runs))
|
||
for run, job, kk in c.execute(f"SELECT run_id, job_id, kk FROM rollouts WHERE state='PAUSED_INFRA' AND run_id IN ({q})", runs):
|
||
try:
|
||
e = json.load(open(f"{B}/runs/{run}/{job}/rollouts/{kk:02d}/execution.json"))
|
||
except (OSError, ValueError):
|
||
continue
|
||
paused = now - (e.get("paused_at") or 0)
|
||
if 3600 < paused < 86400: # 正常的暂停几分钟内就会自动续;超过一天的是老问题,不反复报
|
||
yield f"{run[-2:]} {job} #{kk} 暂停 {paused // 60:.0f} 分钟没恢复({e.get('stop_reason')})"
|
||
for run in runs:
|
||
alive = c.execute("SELECT COUNT(*) FROM rollouts WHERE run_id=? AND state='RUNNING'", (run,)).fetchone()[0]
|
||
last = c.execute("SELECT MAX(started_at) FROM stages WHERE run_id=?", (run,)).fetchone()[0] or 0
|
||
if alive and now - last > 3 * 3600:
|
||
yield f"{run} 有 {alive} 条在跑,但 {(now - last) / 3600:.0f} 小时没有开始任何新步骤,调度程序可能卡住了"
|
||
n = c.execute(f"SELECT COUNT(*) FROM stages WHERE run_id IN ({q}) AND state='FAIL' AND started_at>?"
|
||
f" AND error_class IN ({','.join('?' * len(INFRA))})", (*runs, now - 3600, *INFRA)).fetchone()[0]
|
||
if n >= 8:
|
||
yield f"最近 1 小时 Salty 容器失败 {n} 次"
|
||
n = c.execute(f"SELECT COUNT(*) FROM rollouts WHERE run_id IN ({q}) AND finished_at>?"
|
||
" AND state IN ('INFRA_ERROR','ABANDONED','LOST','CANCELLED')", (*runs, now - 3600)).fetchone()[0]
|
||
if n >= 5:
|
||
yield f"最近 1 小时有 {n} 条轨迹被系统中断"
|
||
n = c.execute(f"SELECT COUNT(*) FROM stages WHERE run_id IN ({q}) AND state='FAIL' AND started_at>?"
|
||
" AND error_class='board_offline'", (*runs, now - 3600)).fetchone()[0]
|
||
if n >= 3:
|
||
yield f"最近 1 小时上板失败 {n} 次(板卡离线)"
|
||
boards = board_endpoints()
|
||
down = [f"{h}:{p}" for h, p in boards if not reachable(h, p)]
|
||
if boards and len(down) == len(boards):
|
||
yield f"{len(boards)} 块板卡全部连不上({', '.join(sorted({d.rsplit(':', 1)[0] for d in down}))}),找板卡网关管理员"
|
||
elif down:
|
||
yield f"{len(down)}/{len(boards)} 块板卡连不上:端口 {', '.join(d.rsplit(':', 1)[1] for d in down)}"
|
||
qc = sqlite3.connect(f"file:{QUOTA}?mode=ro", uri=True, timeout=30)
|
||
for group, state, updated in qc.execute(
|
||
"SELECT group_id, state, updated_at FROM quota_groups WHERE state IN ('PAUSED','RECONCILING')"
|
||
" AND group_id IN (SELECT group_id FROM attempts WHERE started_at>?)", (now - 86400,)):
|
||
if now - updated > 2700: # 请求结果不明时本来就会整组核对约半小时,超过 45 分钟才算卡住
|
||
yield f"模型账号 {group} 处于 {state} 已 {(now - updated) / 60:.0f} 分钟,用这个账号的轨迹都发不出请求"
|
||
free = shutil.disk_usage(B).free / 2**30
|
||
if free < 10:
|
||
yield f"虚拟机磁盘只剩 {free:.0f} GB"
|
||
mem = int(next(l.split()[1] for l in open("/proc/meminfo") if l.startswith("MemAvailable"))) // 1024
|
||
if mem < 1024:
|
||
yield f"虚拟机可用内存只剩 {mem} MB,调度程序可能被系统杀掉"
|
||
|
||
|
||
|
||
def board_endpoints():
|
||
"""BOARD*_HOST/PORT pairs from .env.local (host and port only; keys and users are never read out)."""
|
||
env = {}
|
||
try:
|
||
for line in open(f"{B}/.env.local"):
|
||
k, _, v = line.strip().partition("=")
|
||
if k.startswith("BOARD") and k.endswith(("_HOST", "_PORT")):
|
||
env[k] = v.strip().strip('"')
|
||
except OSError:
|
||
return []
|
||
return [(env[k], int(env[k[:-5] + "_PORT"])) for k in env if k.endswith("_HOST") and env.get(k[:-5] + "_PORT", "").isdigit()]
|
||
|
||
|
||
def reachable(host, port):
|
||
try:
|
||
socket.create_connection((host, port), timeout=5).close()
|
||
return True
|
||
except OSError:
|
||
return False
|
||
|
||
if __name__ == "__main__":
|
||
for line in problems(sys.argv[1:], time.time()):
|
||
print(line)
|