"""`bench board-backfill --rollout JOB:KK`:用 rollout 现成 HBM 补跑 07/08 并重新 verify。 复现 2026-09-16 regnet.y.800mf kk0:14 次 eval 全 board_offline → UNVERIFIED;板子恢复后补测一次应得到真实 G3。 """ from contextlib import ExitStack, nullcontext from functools import partial import json import hashlib from pathlib import Path import tempfile import unittest from unittest.mock import patch from harness import db, paths, pipeline, registry, scheduler from harness.stages import Board, Eval, Baseline, Compile from harness.stages.base import StageOutcome from harness.stages.s02_baseline import baseline_dir from harness.stages import s07_board class RolloutBackfill(unittest.TestCase): def setUp(self): temp = tempfile.TemporaryDirectory() self.addCleanup(temp.cleanup) self.root = Path(temp.name) self.stack = ExitStack() self.addCleanup(self.stack.close) self.board = {"name": "sandbox", "march": "nash-m", "board_type": "s100p", "runtime_dir": "/userdata/bench", "host_env": "TEST_BOARD_HOST"} self.config = { "paths": {"venv": {"bench": "/test/venv"}, "harness_python": "python3"}, "perf": {"warmup": 2, "runs": 3}, "watchdog": {"stall_s": 30, "cpu_idle_pct": 1, "poll_s": 1, "hard_ceiling_s": 60, "io_stall_s": 50, "no_progress_ceiling_s": 55}, "pools": {"board": {"workers": 1, "reserve_for_agent": 1}}, } for name, value in (("CACHE", self.root / "cache"), ("RUNS", self.root / "runs"), ("BENCH_ROOT", self.root), ("STATE", self.root / "state")): self.stack.enter_context(patch.object(paths, name, value)) self.stack.enter_context(patch.object(paths, "config", return_value=self.config)) self.stack.enter_context(patch.object(s07_board, "board_for_march", return_value=self.board)) (self.root / "state").mkdir() self.stack.enter_context(patch.object(db, "connect", partial(db.connect, self.root / "state/jobs.sqlite"))) spec = registry.JobSpec("cls--regnet.y.800mf--nash-m--ptq--int8", "cls", "regnet.y.800mf", "nash-m", "ptq", "int8", "A", "e2e") self.job = { "job_id": spec.job_id, "run_id": "test-run", "task": "cls", "model": "regnet.y.800mf", "march": "nash-m", "backend": "ptq", "board": None, "evaluator_version": "0.5.0", "model_spec": {"input_spec": "1x3x224x224", "resize": "256"}, "data": {"dataset": "imagenet1k", "t1": {"path": "/test/t1.tar", "manifest_sha": "tar-sha", "content_sha": "content-sha"}}, "gates": registry.gate_snapshot(spec), } self.jd = paths.RUNS / "test-run" / spec.job_id self.rd = self.jd / "rollouts" / "00" for d in ("stages", "artifacts", "agent", "handles", "workspace"): (self.rd / d).mkdir(parents=True) (self.jd / "job.json").write_text(json.dumps(self.job)) (self.rd / "rollout.json").write_text(json.dumps({"state": "TIMEOUT", "timeout_reason": "budget", "mode": "e2e", "policy": "teacher", "max_turns": 60, "max_tokens": 200000})) (self.rd / "claim.json").write_text(json.dumps({"terminal": "FAIL", "stage": "08-eval", "summary": "板子离线"})) (self.rd / "agent" / "events.jsonl").write_text("") (self.jd / "oracle").mkdir() (self.jd / "oracle" / "result.json").write_text(json.dumps({"terminal": "PASS", "gates": {"G3": {"delta": 1.0, "kind": "rel_pct"}}})) # oracle 的 02-baseline 已在 cache(rollout 前置条件) bd = baseline_dir(self.job); bd.mkdir(parents=True, exist_ok=True) (bd / "metrics.json").write_text(json.dumps({"evaluator_version": "0.5.0", "t1_manifest_sha": "content-sha", "metrics": {"top1": 80}})) self.hbm_sha = hashlib.sha256(b"agent-hbm").hexdigest() g1 = dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False, hbm_sha256=self.hbm_sha) self.meta("06-hbm", 0, "OK", None, {"G1": g1}) self.meta("07-board", 0, "FAIL", "board_offline", {}) for a in range(14): self.meta("08-eval", a, "FAIL", "board_offline", {}) (self.rd / "artifacts" / f"{spec.job_id}.hbm").write_bytes(b"agent-hbm") with db.connect() as con: db.rollout_upsert(con, "test-run", spec.job_id, 0, "e2e", "teacher", state="UNVERIFIED") def meta(self, name, attempt, state, ec, measurements): ad = self.rd / "stages" / name / f"a{attempt:02d}" ad.mkdir(parents=True, exist_ok=True) m = {"stage": name, "state": state, "error_class": ec, "attempt": attempt, "measurements": measurements, "lane": "agent"} (ad / "meta.json").write_text(json.dumps(m)) (self.rd / "stages" / name / "meta.json").write_text(json.dumps(m)) def fake_run(self, stage, ctx): self.assertEqual(ctx.artifacts, self.rd / "artifacts") # 用的是 agent 的 HBM self.assertEqual(ctx.lane, "backfill") self.assertEqual(ctx.extra.get("board"), self.board) measurements = ({"G4": {"latency_ms_mean": 0.31, "contended": False}} if stage.name == "07-board" else {"G2": {"cos_min": 0.98, "n": 32, "source": "board"}, "G3": {"fp32": 75.0, "quant": 74.0, "source": "board", "tier": "t1", "n": 2000}}) for gate in ("G3", "G4"): if gate in measurements: measurements[gate]["hbm_sha256"] = self.hbm_sha self.meta(stage.name, ctx.attempt, "OK", None, measurements) return StageOutcome("OK", 0, None, 12.5, measurements=measurements) def acquire(self, ctx, pool, lane="normal"): self.assertEqual(pool, "board") ctx.extra.update(slot=0, board=self.board) return nullcontext(0) def test_backfill_turns_unverified_into_measured_verdict(self): with patch.object(scheduler, "acquire_stage", side_effect=self.acquire), \ patch.object(Board, "run", autospec=True, side_effect=self.fake_run), \ patch.object(Eval, "run", autospec=True, side_effect=self.fake_run), \ patch.object(Baseline, "run", side_effect=AssertionError("baseline must be reused")), \ patch.object(Compile, "run", side_effect=AssertionError("HBM must not be recompiled")): r = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None) self.assertEqual(r["before"], "UNVERIFIED") self.assertIn(r["after"], ("PASS", "FAIL")) # 有了真实 G3 就不再 UNVERIFIED self.assertEqual(r["stages"]["07-board"]["attempt"], 1) self.assertEqual(r["stages"]["08-eval"]["attempt"], 14) # 接在 agent 的 a00–a13 之后 v = json.loads((self.rd / "verdict.json").read_text()) self.assertEqual(v["gates"]["G3"]["source"], "board") self.assertEqual(v["stages"]["08-eval"]["error_class"], None) self.assertEqual(v["execution_terminal"], "TIMEOUT") # 执行终止原因照旧,不篡改 self.assertEqual((v.get("claim") or {}).get("terminal"), "FAIL") # claim 一个字不动 hist = json.loads((self.rd / "backfill.json").read_text()) self.assertEqual(sorted(hist[0]["stages"]), ["07-board", "08-eval"]) with db.connect() as con: lanes = [r[0] for r in con.execute("SELECT lane FROM stages WHERE rollout=0 AND stage='08-eval' AND attempt=14")] state = con.execute("SELECT state FROM rollouts WHERE kk=0").fetchone()[0] self.assertEqual(lanes, ["backfill"]) self.assertEqual(state, r["after"]) self.assertEqual((self.rd / "artifacts" / f"{self.job['job_id']}.hbm").read_bytes(), b"agent-hbm") def test_second_run_is_noop_and_running_rollout_refused(self): with patch.object(scheduler, "acquire_stage", side_effect=self.acquire), \ patch.object(Board, "run", autospec=True, side_effect=self.fake_run), \ patch.object(Eval, "run", autospec=True, side_effect=self.fake_run): pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None) again = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None) self.assertEqual(again["skipped"], "already_measured") with db.connect() as con: db.rollout_set(con, "test-run", self.job["job_id"], 0, state="RUNNING") refused = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None) self.assertEqual(refused["skipped"], "rollout_not_finished:RUNNING") def test_missing_hbm_is_skipped(self): (self.rd / "artifacts" / f"{self.job['job_id']}.hbm").unlink() r = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None) self.assertEqual(r["skipped"], "no_hbm") def test_missing_06_hash_requires_revalidation_without_running_or_writing(self): path = self.rd / "stages/06-hbm/meta.json" record = json.loads(path.read_text()) record["measurements"]["G1"].pop("hbm_sha256") path.write_text(json.dumps(record)) before = {p: p.read_bytes() for p in self.rd.rglob("*") if p.is_file()} with patch.object(Board, "run") as board, patch.object(Eval, "run") as evaluate: result = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None) self.assertEqual(result["skipped"], "needs_hbm_revalidation") self.assertIn("06-hbm", result["next_step"]) board.assert_not_called() evaluate.assert_not_called() self.assertEqual(before, {p: p.read_bytes() for p in self.rd.rglob("*") if p.is_file()}) def test_stale_board_accuracy_is_remeasured_despite_source_board(self): self.meta("07-board", 1, "OK", None, {"G4": {"latency_ms_mean": 1, "hbm_sha256": self.hbm_sha}}) self.meta("08-eval", 14, "OK", None, {"G3": {"fp32": 80, "quant": 80, "source": "board", "hbm_sha256": "a" * 64}}) with patch.object(scheduler, "acquire_stage", side_effect=self.acquire), \ patch.object(Board, "run") as board, \ patch.object(Eval, "run", autospec=True, side_effect=self.fake_run): result = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None) board.assert_not_called() self.assertEqual(set(result["stages"]), {"08-eval"}) self.assertEqual(result["stages"]["08-eval"]["attempt"], 15) if __name__ == "__main__": unittest.main()