176 lines
11 KiB
Python
176 lines
11 KiB
Python
"""`bench board-backfill --rollout JOB:KK`:用 rollout 现成 HBM 补跑 07/08 并重新 verify。
|
||
|
||
复现 2026-09-16 regnet.y.800mf kk0:14 次 eval 全 board_offline → UNVERIFIED;板子恢复后补测一次应得到真实 G3。
|
||
"""
|
||
from contextlib import ExitStack, nullcontext
|
||
from functools import partial
|
||
import json
|
||
import hashlib
|
||
from pathlib import Path
|
||
import tempfile
|
||
import unittest
|
||
from unittest.mock import patch
|
||
|
||
from harness import db, paths, pipeline, registry, scheduler
|
||
from harness.stages import Board, Eval, Baseline, Compile
|
||
from harness.stages.base import StageOutcome
|
||
from harness.stages.s02_baseline import baseline_dir
|
||
from harness.stages import s07_board
|
||
|
||
|
||
class RolloutBackfill(unittest.TestCase):
|
||
def setUp(self):
|
||
temp = tempfile.TemporaryDirectory()
|
||
self.addCleanup(temp.cleanup)
|
||
self.root = Path(temp.name)
|
||
self.stack = ExitStack()
|
||
self.addCleanup(self.stack.close)
|
||
self.board = {"name": "sandbox", "march": "nash-m", "board_type": "s100p",
|
||
"runtime_dir": "/userdata/bench", "host_env": "TEST_BOARD_HOST"}
|
||
self.config = {
|
||
"paths": {"venv": {"bench": "/test/venv"}, "harness_python": "python3"},
|
||
"perf": {"warmup": 2, "runs": 3},
|
||
"watchdog": {"stall_s": 30, "cpu_idle_pct": 1, "poll_s": 1, "hard_ceiling_s": 60,
|
||
"io_stall_s": 50, "no_progress_ceiling_s": 55},
|
||
"pools": {"board": {"workers": 1, "reserve_for_agent": 1}},
|
||
}
|
||
for name, value in (("CACHE", self.root / "cache"), ("RUNS", self.root / "runs"),
|
||
("BENCH_ROOT", self.root), ("STATE", self.root / "state")):
|
||
self.stack.enter_context(patch.object(paths, name, value))
|
||
self.stack.enter_context(patch.object(paths, "config", return_value=self.config))
|
||
self.stack.enter_context(patch.object(s07_board, "board_for_march", return_value=self.board))
|
||
(self.root / "state").mkdir()
|
||
self.stack.enter_context(patch.object(db, "connect", partial(db.connect, self.root / "state/jobs.sqlite")))
|
||
spec = registry.JobSpec("cls--regnet.y.800mf--nash-m--ptq--int8", "cls", "regnet.y.800mf",
|
||
"nash-m", "ptq", "int8", "A", "e2e")
|
||
self.job = {
|
||
"job_id": spec.job_id, "run_id": "test-run", "task": "cls", "model": "regnet.y.800mf",
|
||
"march": "nash-m", "backend": "ptq", "board": None, "evaluator_version": "0.5.0",
|
||
"model_spec": {"input_spec": "1x3x224x224", "resize": "256"},
|
||
"data": {"dataset": "imagenet1k", "t1": {"path": "/test/t1.tar", "manifest_sha": "tar-sha",
|
||
"content_sha": "content-sha"}},
|
||
"gates": registry.gate_snapshot(spec),
|
||
}
|
||
self.jd = paths.RUNS / "test-run" / spec.job_id
|
||
self.rd = self.jd / "rollouts" / "00"
|
||
for d in ("stages", "artifacts", "agent", "handles", "workspace"):
|
||
(self.rd / d).mkdir(parents=True)
|
||
(self.jd / "job.json").write_text(json.dumps(self.job))
|
||
(self.rd / "rollout.json").write_text(json.dumps({"state": "TIMEOUT", "timeout_reason": "budget", "mode": "e2e",
|
||
"policy": "teacher", "max_turns": 60, "max_tokens": 200000}))
|
||
(self.rd / "claim.json").write_text(json.dumps({"terminal": "FAIL", "stage": "08-eval", "summary": "板子离线"}))
|
||
(self.rd / "agent" / "events.jsonl").write_text("")
|
||
(self.jd / "oracle").mkdir()
|
||
(self.jd / "oracle" / "result.json").write_text(json.dumps({"terminal": "PASS", "gates": {"G3": {"delta": 1.0, "kind": "rel_pct"}}}))
|
||
# oracle 的 02-baseline 已在 cache(rollout 前置条件)
|
||
bd = baseline_dir(self.job); bd.mkdir(parents=True, exist_ok=True)
|
||
(bd / "metrics.json").write_text(json.dumps({"evaluator_version": "0.5.0", "t1_manifest_sha": "content-sha", "metrics": {"top1": 80}}))
|
||
self.hbm_sha = hashlib.sha256(b"agent-hbm").hexdigest()
|
||
g1 = dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False,
|
||
hbm_sha256=self.hbm_sha)
|
||
self.meta("06-hbm", 0, "OK", None, {"G1": g1})
|
||
self.meta("07-board", 0, "FAIL", "board_offline", {})
|
||
for a in range(14):
|
||
self.meta("08-eval", a, "FAIL", "board_offline", {})
|
||
(self.rd / "artifacts" / f"{spec.job_id}.hbm").write_bytes(b"agent-hbm")
|
||
with db.connect() as con:
|
||
db.rollout_upsert(con, "test-run", spec.job_id, 0, "e2e", "teacher", state="UNVERIFIED")
|
||
|
||
def meta(self, name, attempt, state, ec, measurements):
|
||
ad = self.rd / "stages" / name / f"a{attempt:02d}"
|
||
ad.mkdir(parents=True, exist_ok=True)
|
||
m = {"stage": name, "state": state, "error_class": ec, "attempt": attempt, "measurements": measurements, "lane": "agent"}
|
||
(ad / "meta.json").write_text(json.dumps(m))
|
||
(self.rd / "stages" / name / "meta.json").write_text(json.dumps(m))
|
||
|
||
def fake_run(self, stage, ctx):
|
||
self.assertEqual(ctx.artifacts, self.rd / "artifacts") # 用的是 agent 的 HBM
|
||
self.assertEqual(ctx.lane, "backfill")
|
||
self.assertEqual(ctx.extra.get("board"), self.board)
|
||
measurements = ({"G4": {"latency_ms_mean": 0.31, "contended": False}} if stage.name == "07-board" else
|
||
{"G2": {"cos_min": 0.98, "n": 32, "source": "board"},
|
||
"G3": {"fp32": 75.0, "quant": 74.0, "source": "board", "tier": "t1", "n": 2000}})
|
||
for gate in ("G3", "G4"):
|
||
if gate in measurements:
|
||
measurements[gate]["hbm_sha256"] = self.hbm_sha
|
||
self.meta(stage.name, ctx.attempt, "OK", None, measurements)
|
||
return StageOutcome("OK", 0, None, 12.5, measurements=measurements)
|
||
|
||
def acquire(self, ctx, pool, lane="normal"):
|
||
self.assertEqual(pool, "board")
|
||
ctx.extra.update(slot=0, board=self.board)
|
||
return nullcontext(0)
|
||
|
||
def test_backfill_turns_unverified_into_measured_verdict(self):
|
||
with patch.object(scheduler, "acquire_stage", side_effect=self.acquire), \
|
||
patch.object(Board, "run", autospec=True, side_effect=self.fake_run), \
|
||
patch.object(Eval, "run", autospec=True, side_effect=self.fake_run), \
|
||
patch.object(Baseline, "run", side_effect=AssertionError("baseline must be reused")), \
|
||
patch.object(Compile, "run", side_effect=AssertionError("HBM must not be recompiled")):
|
||
r = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
|
||
self.assertEqual(r["before"], "UNVERIFIED")
|
||
self.assertIn(r["after"], ("PASS", "FAIL")) # 有了真实 G3 就不再 UNVERIFIED
|
||
self.assertEqual(r["stages"]["07-board"]["attempt"], 1)
|
||
self.assertEqual(r["stages"]["08-eval"]["attempt"], 14) # 接在 agent 的 a00–a13 之后
|
||
v = json.loads((self.rd / "verdict.json").read_text())
|
||
self.assertEqual(v["gates"]["G3"]["source"], "board")
|
||
self.assertEqual(v["stages"]["08-eval"]["error_class"], None)
|
||
self.assertEqual(v["execution_terminal"], "TIMEOUT") # 执行终止原因照旧,不篡改
|
||
self.assertEqual((v.get("claim") or {}).get("terminal"), "FAIL") # claim 一个字不动
|
||
hist = json.loads((self.rd / "backfill.json").read_text())
|
||
self.assertEqual(sorted(hist[0]["stages"]), ["07-board", "08-eval"])
|
||
with db.connect() as con:
|
||
lanes = [r[0] for r in con.execute("SELECT lane FROM stages WHERE rollout=0 AND stage='08-eval' AND attempt=14")]
|
||
state = con.execute("SELECT state FROM rollouts WHERE kk=0").fetchone()[0]
|
||
self.assertEqual(lanes, ["backfill"])
|
||
self.assertEqual(state, r["after"])
|
||
self.assertEqual((self.rd / "artifacts" / f"{self.job['job_id']}.hbm").read_bytes(), b"agent-hbm")
|
||
|
||
def test_second_run_is_noop_and_running_rollout_refused(self):
|
||
with patch.object(scheduler, "acquire_stage", side_effect=self.acquire), \
|
||
patch.object(Board, "run", autospec=True, side_effect=self.fake_run), \
|
||
patch.object(Eval, "run", autospec=True, side_effect=self.fake_run):
|
||
pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
|
||
again = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
|
||
self.assertEqual(again["skipped"], "already_measured")
|
||
with db.connect() as con:
|
||
db.rollout_set(con, "test-run", self.job["job_id"], 0, state="RUNNING")
|
||
refused = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
|
||
self.assertEqual(refused["skipped"], "rollout_not_finished:RUNNING")
|
||
|
||
def test_missing_hbm_is_skipped(self):
|
||
(self.rd / "artifacts" / f"{self.job['job_id']}.hbm").unlink()
|
||
r = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
|
||
self.assertEqual(r["skipped"], "no_hbm")
|
||
|
||
def test_missing_06_hash_requires_revalidation_without_running_or_writing(self):
|
||
path = self.rd / "stages/06-hbm/meta.json"
|
||
record = json.loads(path.read_text())
|
||
record["measurements"]["G1"].pop("hbm_sha256")
|
||
path.write_text(json.dumps(record))
|
||
before = {p: p.read_bytes() for p in self.rd.rglob("*") if p.is_file()}
|
||
with patch.object(Board, "run") as board, patch.object(Eval, "run") as evaluate:
|
||
result = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
|
||
self.assertEqual(result["skipped"], "needs_hbm_revalidation")
|
||
self.assertIn("06-hbm", result["next_step"])
|
||
board.assert_not_called()
|
||
evaluate.assert_not_called()
|
||
self.assertEqual(before, {p: p.read_bytes() for p in self.rd.rglob("*") if p.is_file()})
|
||
|
||
def test_stale_board_accuracy_is_remeasured_despite_source_board(self):
|
||
self.meta("07-board", 1, "OK", None,
|
||
{"G4": {"latency_ms_mean": 1, "hbm_sha256": self.hbm_sha}})
|
||
self.meta("08-eval", 14, "OK", None,
|
||
{"G3": {"fp32": 80, "quant": 80, "source": "board", "hbm_sha256": "a" * 64}})
|
||
with patch.object(scheduler, "acquire_stage", side_effect=self.acquire), \
|
||
patch.object(Board, "run") as board, \
|
||
patch.object(Eval, "run", autospec=True, side_effect=self.fake_run):
|
||
result = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
|
||
board.assert_not_called()
|
||
self.assertEqual(set(result["stages"]), {"08-eval"})
|
||
self.assertEqual(result["stages"]["08-eval"]["attempt"], 15)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|