data-gen-bpu/scripts/test_rollout_backfill.py
bench cea5cb5415 watchdog: 按线程而非线程组长判定 D 状态,修复链接期误杀
_proc_tree_stat 只读 /proc/<pid>/stat,拿到的是线程组长的状态。实测 ld.lld 组长是
S(futex_wait),但 17 个线程里有 6~16 个 D 在 FUSE 回写(wait_on_page_bit_common /
balance_dirty_pages / fuse_wait_on_page_writeback)。1001 次观测中 59.7% 属于
"组长口径 0、线程口径 >0",于是正在往 JuiceFS 写盘的链接期被判成挂死:日志不涨 +
CPU 空闲 + d_count=0 三条同时成立,long 车道跑满 3600s 被 SIGTERM。

活体证据:efficientnet.v2.l 日志 511 字节连续 11 次轮询不变、CPU 0.0~0.3%,
旧口径 d_count 恒 0(正在累加 stalled_for),新口径 10~11。对照组 raft.large 在
103% CPU 下新口径恒 0,正常编译不受影响;非链接期 hb_compile 897 次采样 0 次命中,
豁免不会泛化。近期 36 次 05-compile 尝试中 24 次 STALLED、约 49.7 机时。

改动:
- 新增 _d_threads(),遍历 /proc/<pid>/task/*/stat 数 D 线程。task/ 含组长自身,
  是原进程级判定的严格超集。只数 D,不把 futex 等待算豁免——每个 hb_compile 常驻
  40+ 个 futex_wait 线程,算进来会让 stall 判定永久失效。
- 接上一直没接线的 io_stall_s(config 里补键并由 base.py 传入)。它此前是死代码,
  因为 d_count 恒 0;这次保持 21600 不下调,避免把误杀换个理由重演。
- 新增 no_progress_ceiling_s(默认 None,仅 base.py 显式开启):stalled_for 与
  io_blocked_for 会互相清零,d_count 抖动时两个阈值都到不了,仿真显示会塌到 24h
  hard_ceiling。它只看"距上次日志增长或 CPU 活动",免疫 D 抖动。默认关闭是因为
  run() 另有 rollout.py / s06_hbm.py 四个调用点,rollout.py 刻意把 stall_s 设成
  hard_ceiling_s(不 stall-kill agent),改默认值会静默腰斩它。
- _proc_tree_stat 与 _d_threads 的 stat 解析一并移入 try:进程/线程随时消失会给出
  空串或截断串,异常冲出轮询循环会跳过 _kill_tree,把子进程留成孤儿。

heartbeat.py:89/93 复用 _proc_tree_stat 的 d_count,其假 STALLED 告警一并修复。
性能:整树多读约 2.2ms/次,24 路并发折合 0.18% 单核。

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-17 19:51:59 +08:00

143 lines
8.7 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""`bench board-backfill --rollout JOB:KK`:用 rollout 现成 HBM 补跑 07/08 并重新 verify。
复现 2026-09-16 regnet.y.800mf kk0:14 次 eval 全 board_offline → UNVERIFIED;板子恢复后补测一次应得到真实 G3。
"""
from contextlib import ExitStack, nullcontext
from functools import partial
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from harness import db, paths, pipeline, registry, scheduler
from harness.stages import Board, Eval, Baseline, Compile
from harness.stages.base import StageOutcome
from harness.stages.s02_baseline import baseline_dir
from harness.stages import s07_board
class RolloutBackfill(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.root = Path(temp.name)
self.stack = ExitStack()
self.addCleanup(self.stack.close)
self.board = {"name": "sandbox", "march": "nash-m", "board_type": "s100p",
"runtime_dir": "/userdata/bench", "host_env": "TEST_BOARD_HOST"}
self.config = {
"paths": {"venv": {"bench": "/test/venv"}, "harness_python": "python3"},
"perf": {"warmup": 2, "runs": 3},
"watchdog": {"stall_s": 30, "cpu_idle_pct": 1, "poll_s": 1, "hard_ceiling_s": 60,
"io_stall_s": 50, "no_progress_ceiling_s": 55},
"pools": {"board": {"workers": 1, "reserve_for_agent": 1}},
}
for name, value in (("CACHE", self.root / "cache"), ("RUNS", self.root / "runs"),
("BENCH_ROOT", self.root), ("STATE", self.root / "state")):
self.stack.enter_context(patch.object(paths, name, value))
self.stack.enter_context(patch.object(paths, "config", return_value=self.config))
self.stack.enter_context(patch.object(s07_board, "board_for_march", return_value=self.board))
(self.root / "state").mkdir()
self.stack.enter_context(patch.object(db, "connect", partial(db.connect, self.root / "state/jobs.sqlite")))
spec = registry.JobSpec("cls--regnet.y.800mf--nash-m--ptq--int8", "cls", "regnet.y.800mf",
"nash-m", "ptq", "int8", "A", "e2e")
self.job = {
"job_id": spec.job_id, "run_id": "test-run", "task": "cls", "model": "regnet.y.800mf",
"march": "nash-m", "backend": "ptq", "board": None, "evaluator_version": "0.5.0",
"model_spec": {"input_spec": "1x3x224x224", "resize": "256"},
"data": {"dataset": "imagenet1k", "t1": {"path": "/test/t1.tar", "manifest_sha": "tar-sha",
"content_sha": "content-sha"}},
"gates": registry.gate_snapshot(spec),
}
self.jd = paths.RUNS / "test-run" / spec.job_id
self.rd = self.jd / "rollouts" / "00"
for d in ("stages", "artifacts", "agent", "handles", "workspace"):
(self.rd / d).mkdir(parents=True)
(self.jd / "job.json").write_text(json.dumps(self.job))
(self.rd / "rollout.json").write_text(json.dumps({"state": "TIMEOUT", "timeout_reason": "budget", "mode": "e2e",
"policy": "teacher", "max_turns": 60, "max_tokens": 200000}))
(self.rd / "claim.json").write_text(json.dumps({"terminal": "FAIL", "stage": "08-eval", "summary": "板子离线"}))
(self.rd / "agent" / "events.jsonl").write_text("")
(self.jd / "oracle").mkdir()
(self.jd / "oracle" / "result.json").write_text(json.dumps({"terminal": "PASS", "gates": {"G3": {"delta": 1.0, "kind": "rel_pct"}}}))
# oracle 的 02-baseline 已在 cache(rollout 前置条件)
bd = baseline_dir(self.job); bd.mkdir(parents=True, exist_ok=True)
(bd / "metrics.json").write_text(json.dumps({"evaluator_version": "0.5.0", "t1_manifest_sha": "content-sha", "metrics": {"top1": 80}}))
g1 = dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False)
self.meta("06-hbm", 0, "OK", None, {"G1": g1})
self.meta("07-board", 0, "FAIL", "board_offline", {})
for a in range(14):
self.meta("08-eval", a, "FAIL", "board_offline", {})
(self.rd / "artifacts" / f"{spec.job_id}.hbm").write_bytes(b"agent-hbm")
with db.connect() as con:
db.rollout_upsert(con, "test-run", spec.job_id, 0, "e2e", "teacher", state="UNVERIFIED")
def meta(self, name, attempt, state, ec, measurements):
ad = self.rd / "stages" / name / f"a{attempt:02d}"
ad.mkdir(parents=True, exist_ok=True)
m = {"stage": name, "state": state, "error_class": ec, "attempt": attempt, "measurements": measurements, "lane": "agent"}
(ad / "meta.json").write_text(json.dumps(m))
(self.rd / "stages" / name / "meta.json").write_text(json.dumps(m))
def fake_run(self, stage, ctx):
self.assertEqual(ctx.artifacts, self.rd / "artifacts") # 用的是 agent 的 HBM
self.assertEqual(ctx.lane, "backfill")
self.assertEqual(ctx.extra.get("board"), self.board)
measurements = ({"G4": {"latency_ms_mean": 0.31, "contended": False}} if stage.name == "07-board" else
{"G2": {"cos_min": 0.98, "n": 32, "source": "board"},
"G3": {"fp32": 75.0, "quant": 74.0, "source": "board", "tier": "t1", "n": 2000}})
self.meta(stage.name, ctx.attempt, "OK", None, measurements)
return StageOutcome("OK", 0, None, 12.5, measurements=measurements)
def acquire(self, ctx, pool, lane="normal"):
self.assertEqual(pool, "board")
ctx.extra.update(slot=0, board=self.board)
return nullcontext(0)
def test_backfill_turns_unverified_into_measured_verdict(self):
with patch.object(scheduler, "acquire_stage", side_effect=self.acquire), \
patch.object(Board, "run", autospec=True, side_effect=self.fake_run), \
patch.object(Eval, "run", autospec=True, side_effect=self.fake_run), \
patch.object(Baseline, "run", side_effect=AssertionError("baseline must be reused")), \
patch.object(Compile, "run", side_effect=AssertionError("HBM must not be recompiled")):
r = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
self.assertEqual(r["before"], "UNVERIFIED")
self.assertIn(r["after"], ("PASS", "FAIL")) # 有了真实 G3 就不再 UNVERIFIED
self.assertEqual(r["stages"]["07-board"]["attempt"], 1)
self.assertEqual(r["stages"]["08-eval"]["attempt"], 14) # 接在 agent 的 a00–a13 之后
v = json.loads((self.rd / "verdict.json").read_text())
self.assertEqual(v["gates"]["G3"]["source"], "board")
self.assertEqual(v["stages"]["08-eval"]["error_class"], None)
self.assertEqual(v["execution_terminal"], "TIMEOUT") # 执行终止原因照旧,不篡改
self.assertEqual((v.get("claim") or {}).get("terminal"), "FAIL") # claim 一个字不动
hist = json.loads((self.rd / "backfill.json").read_text())
self.assertEqual(sorted(hist[0]["stages"]), ["07-board", "08-eval"])
with db.connect() as con:
lanes = [r[0] for r in con.execute("SELECT lane FROM stages WHERE rollout=0 AND stage='08-eval' AND attempt=14")]
state = con.execute("SELECT state FROM rollouts WHERE kk=0").fetchone()[0]
self.assertEqual(lanes, ["backfill"])
self.assertEqual(state, r["after"])
self.assertEqual((self.rd / "artifacts" / f"{self.job['job_id']}.hbm").read_bytes(), b"agent-hbm")
def test_second_run_is_noop_and_running_rollout_refused(self):
with patch.object(scheduler, "acquire_stage", side_effect=self.acquire), \
patch.object(Board, "run", autospec=True, side_effect=self.fake_run), \
patch.object(Eval, "run", autospec=True, side_effect=self.fake_run):
pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
again = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
self.assertEqual(again["skipped"], "already_measured")
with db.connect() as con:
db.rollout_set(con, "test-run", self.job["job_id"], 0, state="RUNNING")
refused = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
self.assertEqual(refused["skipped"], "rollout_not_finished:RUNNING")
def test_missing_hbm_is_skipped(self):
(self.rd / "artifacts" / f"{self.job['job_id']}.hbm").unlink()
r = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
self.assertEqual(r["skipped"], "no_hbm")
if __name__ == "__main__":
unittest.main()