35 lines
2.3 KiB
Bash
Executable File
35 lines
2.3 KiB
Bash
Executable File
#!/bin/bash
|
||
# 排空旧调度程序后换新代码重启,不丢在跑的轨迹。在 Mac mini 的 bench 虚拟机里后台跑:
|
||
# setsid nohup scripts/drain_restart.sh "<run> <旧pid> <配置名> <新并发> [题单文件]" ... >> logs/drain-restart.log 2>&1 &
|
||
# 暂停的轨迹归旧进程所有(OWNER_NOT_RELEASABLE),旧进程退出后由新进程自动续跑,不用手动 resume。
|
||
# 旧进程没有子进程、也没有校验中的轨迹满 10 分钟,说明卡在最后的导出上,按 bench-restart-hazards 第 5 条 kill -9。
|
||
set -u
|
||
B=/mnt/data/ming01.wang/model-compile-hbdk/bench; C=/mnt/data/ming01.wang/configs
|
||
export BENCH_ROOT=$B BENCH_STATE_DIR=$B/state
|
||
export PATH=/home/amos.guest/.local/lib/salty-controller/python/bin:/home/amos.guest/.local/lib/salty-controller/runtime/bin:/home/amos.guest/.local/bin:/usr/local/bin:/usr/bin:/bin
|
||
log() { echo "$(date '+%m-%d %H:%M:%S') $*"; }
|
||
SPECS=("$@"); declare -A idle=() done_=()
|
||
# 已经发过 SIGTERM 时用 NO_TERM=1 重跑本脚本:第二次信号可能让调度程序直接退出、丢掉在跑的轨迹。
|
||
[ -z "${NO_TERM:-}" ] && for s in "${SPECS[@]}"; do set -- $s; kill -TERM "$2" && log "SIGTERM $1 pid $2,开始排空"; done
|
||
while [ ${#done_[@]} -lt ${#SPECS[@]} ]; do
|
||
for s in "${SPECS[@]}"; do
|
||
set -- $s; run=$1 pid=$2 cfg=$3 n=$4 jobs=${5:-}
|
||
[ -n "${done_[$run]:-}" ] && continue
|
||
if kill -0 "$pid" 2>/dev/null; then
|
||
busy=$(python3 -c "import sqlite3,sys; print(sqlite3.connect('file:$B/state/jobs.sqlite?mode=ro',uri=True).execute(\"SELECT COUNT(*) FROM rollouts WHERE run_id=? AND state IN ('VERIFYING','CLEANING')\",(sys.argv[1],)).fetchone()[0])" "$run")
|
||
if [ -z "$(pgrep -P "$pid")" ] && [ "$busy" = 0 ]; then idle[$run]=$(( ${idle[$run]:-0} + 1 )); else idle[$run]=0; fi
|
||
if [ "${idle[$run]}" -ge 10 ]; then
|
||
log "$run 旧进程 $pid 已无在跑轨迹 10 分钟仍未退出(卡在导出),kill -9"
|
||
kill -9 "$pid"; sleep 5; rm -rf "$B/datasets/$run-training"/.refresh-*
|
||
fi
|
||
continue
|
||
fi
|
||
args="--run $run --modes e2e --parallel $n --watch"
|
||
[ -n "$jobs" ] && args="$args $(sed 's/^/--job /' "$jobs" | tr '\n' ' ')"
|
||
log "$run 旧进程已退出,起新进程:$($C/run-collect.sh "$cfg" $args)"
|
||
done_[$run]=1
|
||
done
|
||
sleep 60
|
||
done
|
||
log "全部重启完成"
|