- stagerun: 容器被杀/执行器退出不计入 5 次编译(每条最多免 3 次) - quota: CONNECT_FAILED / REQUEST_DEADLINE_BEFORE_SEND 证明未发送,允许自动续跑 - pipeline: 有暂停轨迹只差额度时,空出的名额留给它 - scripts/health_check.py + bench_health_alert.sh: 每 10 分钟检查,异常弹本机通知 - scripts/drain_restart.sh: 排空旧调度程序后重启,排空期间用新代码续跑卡住的轨迹 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
62 lines
3.5 KiB
Python
62 lines
3.5 KiB
Python
"""Mac mini 主控的健康检查:有问题每条打印一行,健康时什么都不打印。只读。
|
||
用法(bench 虚拟机里):python3 scripts/health_check.py RUN [RUN ...]
|
||
由本机 scripts/bench_health_alert.sh 每 10 分钟经 ssh 调用,有输出就弹通知。"""
|
||
import json
|
||
import os
|
||
import shutil
|
||
import sqlite3
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
|
||
B = os.environ.get("BENCH_ROOT", "/mnt/data/ming01.wang/model-compile-hbdk/bench")
|
||
QUOTA = os.path.expanduser("~/.local/state/bench-shared/quota.sqlite")
|
||
INFRA = ("REMOTE_WORKER_FAILED", "STAGE_EXECUTOR_EXIT", "QUEUE_DEADLINE_EXCEEDED")
|
||
|
||
|
||
def problems(runs, now):
|
||
ps = subprocess.run(["ps", "-eo", "args"], capture_output=True, text=True).stdout
|
||
for run in runs:
|
||
if f"collect --run {run} " not in ps:
|
||
yield f"{run} 的调度程序没在跑"
|
||
c = sqlite3.connect(f"file:{B}/state/jobs.sqlite?mode=ro", uri=True, timeout=30)
|
||
q = ",".join("?" * len(runs))
|
||
for run, job, kk in c.execute(f"SELECT run_id, job_id, kk FROM rollouts WHERE state='PAUSED_INFRA' AND run_id IN ({q})", runs):
|
||
try:
|
||
e = json.load(open(f"{B}/runs/{run}/{job}/rollouts/{kk:02d}/execution.json"))
|
||
except (OSError, ValueError):
|
||
continue
|
||
paused = now - (e.get("paused_at") or 0)
|
||
if 3600 < paused < 86400: # 正常的暂停几分钟内就会自动续;超过一天的是老问题,不反复报
|
||
yield f"{run[-2:]} {job} #{kk} 暂停 {paused // 60:.0f} 分钟没恢复({e.get('stop_reason')})"
|
||
for run in runs:
|
||
alive = c.execute("SELECT COUNT(*) FROM rollouts WHERE run_id=? AND state='RUNNING'", (run,)).fetchone()[0]
|
||
last = c.execute("SELECT MAX(started_at) FROM stages WHERE run_id=?", (run,)).fetchone()[0] or 0
|
||
if alive and now - last > 3 * 3600:
|
||
yield f"{run} 有 {alive} 条在跑,但 {(now - last) / 3600:.0f} 小时没有开始任何新步骤,调度程序可能卡住了"
|
||
n = c.execute(f"SELECT COUNT(*) FROM stages WHERE run_id IN ({q}) AND state='FAIL' AND started_at>?"
|
||
f" AND error_class IN ({','.join('?' * len(INFRA))})", (*runs, now - 3600, *INFRA)).fetchone()[0]
|
||
if n >= 8:
|
||
yield f"最近 1 小时 Salty 容器失败 {n} 次"
|
||
n = c.execute(f"SELECT COUNT(*) FROM rollouts WHERE run_id IN ({q}) AND finished_at>?"
|
||
" AND state IN ('INFRA_ERROR','ABANDONED','LOST','CANCELLED')", (*runs, now - 3600)).fetchone()[0]
|
||
if n >= 5:
|
||
yield f"最近 1 小时有 {n} 条轨迹被系统中断"
|
||
qc = sqlite3.connect(f"file:{QUOTA}?mode=ro", uri=True, timeout=30)
|
||
for group, state, updated in qc.execute(
|
||
"SELECT group_id, state, updated_at FROM quota_groups WHERE state IN ('PAUSED','RECONCILING')"
|
||
" AND group_id IN (SELECT group_id FROM attempts WHERE started_at>?)", (now - 86400,)):
|
||
if now - updated > 2700: # 请求结果不明时本来就会整组核对约半小时,超过 45 分钟才算卡住
|
||
yield f"模型账号 {group} 处于 {state} 已 {(now - updated) / 60:.0f} 分钟,用这个账号的轨迹都发不出请求"
|
||
free = shutil.disk_usage(B).free / 2**30
|
||
if free < 10:
|
||
yield f"虚拟机磁盘只剩 {free:.0f} GB"
|
||
mem = int(next(l.split()[1] for l in open("/proc/meminfo") if l.startswith("MemAvailable"))) // 1024
|
||
if mem < 1024:
|
||
yield f"虚拟机可用内存只剩 {mem} MB,调度程序可能被系统杀掉"
|
||
|
||
|
||
if __name__ == "__main__":
|
||
for line in problems(sys.argv[1:], time.time()):
|
||
print(line)
|