data-gen-bpu/scripts/drain_restart.sh

35 lines
2.3 KiB
Bash
Executable File
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/bin/bash
# 排空旧调度程序后换新代码重启,不丢在跑的轨迹。在 Mac mini 的 bench 虚拟机里后台跑:
# setsid nohup scripts/drain_restart.sh "<run> <旧pid> <配置名> <新并发> [题单文件]" ... >> logs/drain-restart.log 2>&1 &
# 暂停的轨迹归旧进程所有(OWNER_NOT_RELEASABLE),旧进程退出后由新进程自动续跑,不用手动 resume。
# 旧进程没有子进程、也没有校验中的轨迹满 10 分钟,说明卡在最后的导出上,按 bench-restart-hazards 第 5 条 kill -9。
set -u
B=/mnt/data/ming01.wang/model-compile-hbdk/bench; C=/mnt/data/ming01.wang/configs
export BENCH_ROOT=$B BENCH_STATE_DIR=$B/state
export PATH=/home/amos.guest/.local/lib/salty-controller/python/bin:/home/amos.guest/.local/lib/salty-controller/runtime/bin:/home/amos.guest/.local/bin:/usr/local/bin:/usr/bin:/bin
log() { echo "$(date '+%m-%d %H:%M:%S') $*"; }
SPECS=("$@"); declare -A idle=() done_=()
# 已经发过 SIGTERM 时用 NO_TERM=1 重跑本脚本:第二次信号可能让调度程序直接退出、丢掉在跑的轨迹。
[ -z "${NO_TERM:-}" ] && for s in "${SPECS[@]}"; do set -- $s; kill -TERM "$2" && log "SIGTERM $1 pid $2,开始排空"; done
while [ ${#done_[@]} -lt ${#SPECS[@]} ]; do
for s in "${SPECS[@]}"; do
set -- $s; run=$1 pid=$2 cfg=$3 n=$4 jobs=${5:-}
[ -n "${done_[$run]:-}" ] && continue
if kill -0 "$pid" 2>/dev/null; then
busy=$(python3 -c "import sqlite3,sys; print(sqlite3.connect('file:$B/state/jobs.sqlite?mode=ro',uri=True).execute(\"SELECT COUNT(*) FROM rollouts WHERE run_id=? AND state IN ('VERIFYING','CLEANING')\",(sys.argv[1],)).fetchone()[0])" "$run")
if [ -z "$(pgrep -P "$pid")" ] && [ "$busy" = 0 ]; then idle[$run]=$(( ${idle[$run]:-0} + 1 )); else idle[$run]=0; fi
if [ "${idle[$run]}" -ge 10 ]; then
log "$run 旧进程 $pid 已无在跑轨迹 10 分钟仍未退出(卡在导出),kill -9"
kill -9 "$pid"; sleep 5; rm -rf "$B/datasets/$run-training"/.refresh-*
fi
continue
fi
args="--run $run --modes e2e --parallel $n --watch"
[ -n "$jobs" ] && args="$args $(sed 's/^/--job /' "$jobs" | tr '\n' ' ')"
log "$run 旧进程已退出,起新进程:$($C/run-collect.sh "$cfg" $args)"
done_[$run]=1
done
sleep 60
done
log "全部重启完成"