_proc_tree_stat 只读 /proc/<pid>/stat,拿到的是线程组长的状态。实测 ld.lld 组长是 S(futex_wait),但 17 个线程里有 6~16 个 D 在 FUSE 回写(wait_on_page_bit_common / balance_dirty_pages / fuse_wait_on_page_writeback)。1001 次观测中 59.7% 属于 "组长口径 0、线程口径 >0",于是正在往 JuiceFS 写盘的链接期被判成挂死:日志不涨 + CPU 空闲 + d_count=0 三条同时成立,long 车道跑满 3600s 被 SIGTERM。 活体证据:efficientnet.v2.l 日志 511 字节连续 11 次轮询不变、CPU 0.0~0.3%, 旧口径 d_count 恒 0(正在累加 stalled_for),新口径 10~11。对照组 raft.large 在 103% CPU 下新口径恒 0,正常编译不受影响;非链接期 hb_compile 897 次采样 0 次命中, 豁免不会泛化。近期 36 次 05-compile 尝试中 24 次 STALLED、约 49.7 机时。 改动: - 新增 _d_threads(),遍历 /proc/<pid>/task/*/stat 数 D 线程。task/ 含组长自身, 是原进程级判定的严格超集。只数 D,不把 futex 等待算豁免——每个 hb_compile 常驻 40+ 个 futex_wait 线程,算进来会让 stall 判定永久失效。 - 接上一直没接线的 io_stall_s(config 里补键并由 base.py 传入)。它此前是死代码, 因为 d_count 恒 0;这次保持 21600 不下调,避免把误杀换个理由重演。 - 新增 no_progress_ceiling_s(默认 None,仅 base.py 显式开启):stalled_for 与 io_blocked_for 会互相清零,d_count 抖动时两个阈值都到不了,仿真显示会塌到 24h hard_ceiling。它只看"距上次日志增长或 CPU 活动",免疫 D 抖动。默认关闭是因为 run() 另有 rollout.py / s06_hbm.py 四个调用点,rollout.py 刻意把 stall_s 设成 hard_ceiling_s(不 stall-kill agent),改默认值会静默腰斩它。 - _proc_tree_stat 与 _d_threads 的 stat 解析一并移入 try:进程/线程随时消失会给出 空串或截断串,异常冲出轮询循环会跳过 _kill_tree,把子进程留成孤儿。 heartbeat.py:89/93 复用 _proc_tree_stat 的 d_count,其假 STALLED 告警一并修复。 性能:整树多读约 2.2ms/次,24 路并发折合 0.18% 单核。 Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
143 lines
8.7 KiB
Python
143 lines
8.7 KiB
Python
"""`bench board-backfill --rollout JOB:KK`:用 rollout 现成 HBM 补跑 07/08 并重新 verify。
|
||
|
||
复现 2026-09-16 regnet.y.800mf kk0:14 次 eval 全 board_offline → UNVERIFIED;板子恢复后补测一次应得到真实 G3。
|
||
"""
|
||
from contextlib import ExitStack, nullcontext
|
||
from functools import partial
|
||
import json
|
||
from pathlib import Path
|
||
import tempfile
|
||
import unittest
|
||
from unittest.mock import patch
|
||
|
||
from harness import db, paths, pipeline, registry, scheduler
|
||
from harness.stages import Board, Eval, Baseline, Compile
|
||
from harness.stages.base import StageOutcome
|
||
from harness.stages.s02_baseline import baseline_dir
|
||
from harness.stages import s07_board
|
||
|
||
|
||
class RolloutBackfill(unittest.TestCase):
|
||
def setUp(self):
|
||
temp = tempfile.TemporaryDirectory()
|
||
self.addCleanup(temp.cleanup)
|
||
self.root = Path(temp.name)
|
||
self.stack = ExitStack()
|
||
self.addCleanup(self.stack.close)
|
||
self.board = {"name": "sandbox", "march": "nash-m", "board_type": "s100p",
|
||
"runtime_dir": "/userdata/bench", "host_env": "TEST_BOARD_HOST"}
|
||
self.config = {
|
||
"paths": {"venv": {"bench": "/test/venv"}, "harness_python": "python3"},
|
||
"perf": {"warmup": 2, "runs": 3},
|
||
"watchdog": {"stall_s": 30, "cpu_idle_pct": 1, "poll_s": 1, "hard_ceiling_s": 60,
|
||
"io_stall_s": 50, "no_progress_ceiling_s": 55},
|
||
"pools": {"board": {"workers": 1, "reserve_for_agent": 1}},
|
||
}
|
||
for name, value in (("CACHE", self.root / "cache"), ("RUNS", self.root / "runs"),
|
||
("BENCH_ROOT", self.root), ("STATE", self.root / "state")):
|
||
self.stack.enter_context(patch.object(paths, name, value))
|
||
self.stack.enter_context(patch.object(paths, "config", return_value=self.config))
|
||
self.stack.enter_context(patch.object(s07_board, "board_for_march", return_value=self.board))
|
||
(self.root / "state").mkdir()
|
||
self.stack.enter_context(patch.object(db, "connect", partial(db.connect, self.root / "state/jobs.sqlite")))
|
||
spec = registry.JobSpec("cls--regnet.y.800mf--nash-m--ptq--int8", "cls", "regnet.y.800mf",
|
||
"nash-m", "ptq", "int8", "A", "e2e")
|
||
self.job = {
|
||
"job_id": spec.job_id, "run_id": "test-run", "task": "cls", "model": "regnet.y.800mf",
|
||
"march": "nash-m", "backend": "ptq", "board": None, "evaluator_version": "0.5.0",
|
||
"model_spec": {"input_spec": "1x3x224x224", "resize": "256"},
|
||
"data": {"dataset": "imagenet1k", "t1": {"path": "/test/t1.tar", "manifest_sha": "tar-sha",
|
||
"content_sha": "content-sha"}},
|
||
"gates": registry.gate_snapshot(spec),
|
||
}
|
||
self.jd = paths.RUNS / "test-run" / spec.job_id
|
||
self.rd = self.jd / "rollouts" / "00"
|
||
for d in ("stages", "artifacts", "agent", "handles", "workspace"):
|
||
(self.rd / d).mkdir(parents=True)
|
||
(self.jd / "job.json").write_text(json.dumps(self.job))
|
||
(self.rd / "rollout.json").write_text(json.dumps({"state": "TIMEOUT", "timeout_reason": "budget", "mode": "e2e",
|
||
"policy": "teacher", "max_turns": 60, "max_tokens": 200000}))
|
||
(self.rd / "claim.json").write_text(json.dumps({"terminal": "FAIL", "stage": "08-eval", "summary": "板子离线"}))
|
||
(self.rd / "agent" / "events.jsonl").write_text("")
|
||
(self.jd / "oracle").mkdir()
|
||
(self.jd / "oracle" / "result.json").write_text(json.dumps({"terminal": "PASS", "gates": {"G3": {"delta": 1.0, "kind": "rel_pct"}}}))
|
||
# oracle 的 02-baseline 已在 cache(rollout 前置条件)
|
||
bd = baseline_dir(self.job); bd.mkdir(parents=True, exist_ok=True)
|
||
(bd / "metrics.json").write_text(json.dumps({"evaluator_version": "0.5.0", "t1_manifest_sha": "content-sha", "metrics": {"top1": 80}}))
|
||
g1 = dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False)
|
||
self.meta("06-hbm", 0, "OK", None, {"G1": g1})
|
||
self.meta("07-board", 0, "FAIL", "board_offline", {})
|
||
for a in range(14):
|
||
self.meta("08-eval", a, "FAIL", "board_offline", {})
|
||
(self.rd / "artifacts" / f"{spec.job_id}.hbm").write_bytes(b"agent-hbm")
|
||
with db.connect() as con:
|
||
db.rollout_upsert(con, "test-run", spec.job_id, 0, "e2e", "teacher", state="UNVERIFIED")
|
||
|
||
def meta(self, name, attempt, state, ec, measurements):
|
||
ad = self.rd / "stages" / name / f"a{attempt:02d}"
|
||
ad.mkdir(parents=True, exist_ok=True)
|
||
m = {"stage": name, "state": state, "error_class": ec, "attempt": attempt, "measurements": measurements, "lane": "agent"}
|
||
(ad / "meta.json").write_text(json.dumps(m))
|
||
(self.rd / "stages" / name / "meta.json").write_text(json.dumps(m))
|
||
|
||
def fake_run(self, stage, ctx):
|
||
self.assertEqual(ctx.artifacts, self.rd / "artifacts") # 用的是 agent 的 HBM
|
||
self.assertEqual(ctx.lane, "backfill")
|
||
self.assertEqual(ctx.extra.get("board"), self.board)
|
||
measurements = ({"G4": {"latency_ms_mean": 0.31, "contended": False}} if stage.name == "07-board" else
|
||
{"G2": {"cos_min": 0.98, "n": 32, "source": "board"},
|
||
"G3": {"fp32": 75.0, "quant": 74.0, "source": "board", "tier": "t1", "n": 2000}})
|
||
self.meta(stage.name, ctx.attempt, "OK", None, measurements)
|
||
return StageOutcome("OK", 0, None, 12.5, measurements=measurements)
|
||
|
||
def acquire(self, ctx, pool, lane="normal"):
|
||
self.assertEqual(pool, "board")
|
||
ctx.extra.update(slot=0, board=self.board)
|
||
return nullcontext(0)
|
||
|
||
def test_backfill_turns_unverified_into_measured_verdict(self):
|
||
with patch.object(scheduler, "acquire_stage", side_effect=self.acquire), \
|
||
patch.object(Board, "run", autospec=True, side_effect=self.fake_run), \
|
||
patch.object(Eval, "run", autospec=True, side_effect=self.fake_run), \
|
||
patch.object(Baseline, "run", side_effect=AssertionError("baseline must be reused")), \
|
||
patch.object(Compile, "run", side_effect=AssertionError("HBM must not be recompiled")):
|
||
r = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
|
||
self.assertEqual(r["before"], "UNVERIFIED")
|
||
self.assertIn(r["after"], ("PASS", "FAIL")) # 有了真实 G3 就不再 UNVERIFIED
|
||
self.assertEqual(r["stages"]["07-board"]["attempt"], 1)
|
||
self.assertEqual(r["stages"]["08-eval"]["attempt"], 14) # 接在 agent 的 a00–a13 之后
|
||
v = json.loads((self.rd / "verdict.json").read_text())
|
||
self.assertEqual(v["gates"]["G3"]["source"], "board")
|
||
self.assertEqual(v["stages"]["08-eval"]["error_class"], None)
|
||
self.assertEqual(v["execution_terminal"], "TIMEOUT") # 执行终止原因照旧,不篡改
|
||
self.assertEqual((v.get("claim") or {}).get("terminal"), "FAIL") # claim 一个字不动
|
||
hist = json.loads((self.rd / "backfill.json").read_text())
|
||
self.assertEqual(sorted(hist[0]["stages"]), ["07-board", "08-eval"])
|
||
with db.connect() as con:
|
||
lanes = [r[0] for r in con.execute("SELECT lane FROM stages WHERE rollout=0 AND stage='08-eval' AND attempt=14")]
|
||
state = con.execute("SELECT state FROM rollouts WHERE kk=0").fetchone()[0]
|
||
self.assertEqual(lanes, ["backfill"])
|
||
self.assertEqual(state, r["after"])
|
||
self.assertEqual((self.rd / "artifacts" / f"{self.job['job_id']}.hbm").read_bytes(), b"agent-hbm")
|
||
|
||
def test_second_run_is_noop_and_running_rollout_refused(self):
|
||
with patch.object(scheduler, "acquire_stage", side_effect=self.acquire), \
|
||
patch.object(Board, "run", autospec=True, side_effect=self.fake_run), \
|
||
patch.object(Eval, "run", autospec=True, side_effect=self.fake_run):
|
||
pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
|
||
again = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
|
||
self.assertEqual(again["skipped"], "already_measured")
|
||
with db.connect() as con:
|
||
db.rollout_set(con, "test-run", self.job["job_id"], 0, state="RUNNING")
|
||
refused = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
|
||
self.assertEqual(refused["skipped"], "rollout_not_finished:RUNNING")
|
||
|
||
def test_missing_hbm_is_skipped(self):
|
||
(self.rd / "artifacts" / f"{self.job['job_id']}.hbm").unlink()
|
||
r = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
|
||
self.assertEqual(r["skipped"], "no_hbm")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|