data-gen-bpu/scripts/test_rollout_backfill.py

176 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""`bench board-backfill --rollout JOB:KK`:用 rollout 现成 HBM 补跑 07/08 并重新 verify。
复现 2026-09-16 regnet.y.800mf kk0:14 次 eval 全 board_offline → UNVERIFIED;板子恢复后补测一次应得到真实 G3。
"""
from contextlib import ExitStack, nullcontext
from functools import partial
import json
import hashlib
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from harness import db, paths, pipeline, registry, scheduler
from harness.stages import Board, Eval, Baseline, Compile
from harness.stages.base import StageOutcome
from harness.stages.s02_baseline import baseline_dir
from harness.stages import s07_board
class RolloutBackfill(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.root = Path(temp.name)
self.stack = ExitStack()
self.addCleanup(self.stack.close)
self.board = {"name": "sandbox", "march": "nash-m", "board_type": "s100p",
"runtime_dir": "/userdata/bench", "host_env": "TEST_BOARD_HOST"}
self.config = {
"paths": {"venv": {"bench": "/test/venv"}, "harness_python": "python3"},
"perf": {"warmup": 2, "runs": 3},
"watchdog": {"stall_s": 30, "cpu_idle_pct": 1, "poll_s": 1, "hard_ceiling_s": 60,
"io_stall_s": 50, "no_progress_ceiling_s": 55},
"pools": {"board": {"workers": 1, "reserve_for_agent": 1}},
}
for name, value in (("CACHE", self.root / "cache"), ("RUNS", self.root / "runs"),
("BENCH_ROOT", self.root), ("STATE", self.root / "state")):
self.stack.enter_context(patch.object(paths, name, value))
self.stack.enter_context(patch.object(paths, "config", return_value=self.config))
self.stack.enter_context(patch.object(s07_board, "board_for_march", return_value=self.board))
(self.root / "state").mkdir()
self.stack.enter_context(patch.object(db, "connect", partial(db.connect, self.root / "state/jobs.sqlite")))
spec = registry.JobSpec("cls--regnet.y.800mf--nash-m--ptq--int8", "cls", "regnet.y.800mf",
"nash-m", "ptq", "int8", "A", "e2e")
self.job = {
"job_id": spec.job_id, "run_id": "test-run", "task": "cls", "model": "regnet.y.800mf",
"march": "nash-m", "backend": "ptq", "board": None, "evaluator_version": "0.5.0",
"model_spec": {"input_spec": "1x3x224x224", "resize": "256"},
"data": {"dataset": "imagenet1k", "t1": {"path": "/test/t1.tar", "manifest_sha": "tar-sha",
"content_sha": "content-sha"}},
"gates": registry.gate_snapshot(spec),
}
self.jd = paths.RUNS / "test-run" / spec.job_id
self.rd = self.jd / "rollouts" / "00"
for d in ("stages", "artifacts", "agent", "handles", "workspace"):
(self.rd / d).mkdir(parents=True)
(self.jd / "job.json").write_text(json.dumps(self.job))
(self.rd / "rollout.json").write_text(json.dumps({"state": "TIMEOUT", "timeout_reason": "budget", "mode": "e2e",
"policy": "teacher", "max_turns": 60, "max_tokens": 200000}))
(self.rd / "claim.json").write_text(json.dumps({"terminal": "FAIL", "stage": "08-eval", "summary": "板子离线"}))
(self.rd / "agent" / "events.jsonl").write_text("")
(self.jd / "oracle").mkdir()
(self.jd / "oracle" / "result.json").write_text(json.dumps({"terminal": "PASS", "gates": {"G3": {"delta": 1.0, "kind": "rel_pct"}}}))
# oracle 的 02-baseline 已在 cache(rollout 前置条件)
bd = baseline_dir(self.job); bd.mkdir(parents=True, exist_ok=True)
(bd / "metrics.json").write_text(json.dumps({"evaluator_version": "0.5.0", "t1_manifest_sha": "content-sha", "metrics": {"top1": 80}}))
self.hbm_sha = hashlib.sha256(b"agent-hbm").hexdigest()
g1 = dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False,
hbm_sha256=self.hbm_sha)
self.meta("06-hbm", 0, "OK", None, {"G1": g1})
self.meta("07-board", 0, "FAIL", "board_offline", {})
for a in range(14):
self.meta("08-eval", a, "FAIL", "board_offline", {})
(self.rd / "artifacts" / f"{spec.job_id}.hbm").write_bytes(b"agent-hbm")
with db.connect() as con:
db.rollout_upsert(con, "test-run", spec.job_id, 0, "e2e", "teacher", state="UNVERIFIED")
def meta(self, name, attempt, state, ec, measurements):
ad = self.rd / "stages" / name / f"a{attempt:02d}"
ad.mkdir(parents=True, exist_ok=True)
m = {"stage": name, "state": state, "error_class": ec, "attempt": attempt, "measurements": measurements, "lane": "agent"}
(ad / "meta.json").write_text(json.dumps(m))
(self.rd / "stages" / name / "meta.json").write_text(json.dumps(m))
def fake_run(self, stage, ctx):
self.assertEqual(ctx.artifacts, self.rd / "artifacts") # 用的是 agent 的 HBM
self.assertEqual(ctx.lane, "backfill")
self.assertEqual(ctx.extra.get("board"), self.board)
measurements = ({"G4": {"latency_ms_mean": 0.31, "contended": False}} if stage.name == "07-board" else
{"G2": {"cos_min": 0.98, "n": 32, "source": "board"},
"G3": {"fp32": 75.0, "quant": 74.0, "source": "board", "tier": "t1", "n": 2000}})
for gate in ("G3", "G4"):
if gate in measurements:
measurements[gate]["hbm_sha256"] = self.hbm_sha
self.meta(stage.name, ctx.attempt, "OK", None, measurements)
return StageOutcome("OK", 0, None, 12.5, measurements=measurements)
def acquire(self, ctx, pool, lane="normal"):
self.assertEqual(pool, "board")
ctx.extra.update(slot=0, board=self.board)
return nullcontext(0)
def test_backfill_turns_unverified_into_measured_verdict(self):
with patch.object(scheduler, "acquire_stage", side_effect=self.acquire), \
patch.object(Board, "run", autospec=True, side_effect=self.fake_run), \
patch.object(Eval, "run", autospec=True, side_effect=self.fake_run), \
patch.object(Baseline, "run", side_effect=AssertionError("baseline must be reused")), \
patch.object(Compile, "run", side_effect=AssertionError("HBM must not be recompiled")):
r = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
self.assertEqual(r["before"], "UNVERIFIED")
self.assertIn(r["after"], ("PASS", "FAIL")) # 有了真实 G3 就不再 UNVERIFIED
self.assertEqual(r["stages"]["07-board"]["attempt"], 1)
self.assertEqual(r["stages"]["08-eval"]["attempt"], 14) # 接在 agent 的 a00–a13 之后
v = json.loads((self.rd / "verdict.json").read_text())
self.assertEqual(v["gates"]["G3"]["source"], "board")
self.assertEqual(v["stages"]["08-eval"]["error_class"], None)
self.assertEqual(v["execution_terminal"], "TIMEOUT") # 执行终止原因照旧,不篡改
self.assertEqual((v.get("claim") or {}).get("terminal"), "FAIL") # claim 一个字不动
hist = json.loads((self.rd / "backfill.json").read_text())
self.assertEqual(sorted(hist[0]["stages"]), ["07-board", "08-eval"])
with db.connect() as con:
lanes = [r[0] for r in con.execute("SELECT lane FROM stages WHERE rollout=0 AND stage='08-eval' AND attempt=14")]
state = con.execute("SELECT state FROM rollouts WHERE kk=0").fetchone()[0]
self.assertEqual(lanes, ["backfill"])
self.assertEqual(state, r["after"])
self.assertEqual((self.rd / "artifacts" / f"{self.job['job_id']}.hbm").read_bytes(), b"agent-hbm")
def test_second_run_is_noop_and_running_rollout_refused(self):
with patch.object(scheduler, "acquire_stage", side_effect=self.acquire), \
patch.object(Board, "run", autospec=True, side_effect=self.fake_run), \
patch.object(Eval, "run", autospec=True, side_effect=self.fake_run):
pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
again = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
self.assertEqual(again["skipped"], "already_measured")
with db.connect() as con:
db.rollout_set(con, "test-run", self.job["job_id"], 0, state="RUNNING")
refused = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
self.assertEqual(refused["skipped"], "rollout_not_finished:RUNNING")
def test_missing_hbm_is_skipped(self):
(self.rd / "artifacts" / f"{self.job['job_id']}.hbm").unlink()
r = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
self.assertEqual(r["skipped"], "no_hbm")
def test_missing_06_hash_requires_revalidation_without_running_or_writing(self):
path = self.rd / "stages/06-hbm/meta.json"
record = json.loads(path.read_text())
record["measurements"]["G1"].pop("hbm_sha256")
path.write_text(json.dumps(record))
before = {p: p.read_bytes() for p in self.rd.rglob("*") if p.is_file()}
with patch.object(Board, "run") as board, patch.object(Eval, "run") as evaluate:
result = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
self.assertEqual(result["skipped"], "needs_hbm_revalidation")
self.assertIn("06-hbm", result["next_step"])
board.assert_not_called()
evaluate.assert_not_called()
self.assertEqual(before, {p: p.read_bytes() for p in self.rd.rglob("*") if p.is_file()})
def test_stale_board_accuracy_is_remeasured_despite_source_board(self):
self.meta("07-board", 1, "OK", None,
{"G4": {"latency_ms_mean": 1, "hbm_sha256": self.hbm_sha}})
self.meta("08-eval", 14, "OK", None,
{"G3": {"fp32": 80, "quant": 80, "source": "board", "hbm_sha256": "a" * 64}})
with patch.object(scheduler, "acquire_stage", side_effect=self.acquire), \
patch.object(Board, "run") as board, \
patch.object(Eval, "run", autospec=True, side_effect=self.fake_run):
result = pipeline.board_backfill_rollout("test-run", self.job["job_id"], 0, log=lambda _: None)
board.assert_not_called()
self.assertEqual(set(result["stages"]), {"08-eval"})
self.assertEqual(result["stages"]["08-eval"]["attempt"], 15)
if __name__ == "__main__":
unittest.main()