data-gen-bpu/scripts/test_board_pipeline.py

246 lines
14 KiB
Python

"""Board selection, stage isolation, and HBM-only recovery regression checks."""
from contextlib import ExitStack, nullcontext, redirect_stdout
import io
import hashlib
import json
import os
from pathlib import Path
import sys
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from harness import db, gates, paths, pipeline, registry, scheduler
from harness.stages import Board, Eval, Baseline, Compile
from harness.stages.base import Stage, StageContext, StageOutcome
from harness.stages.s02_baseline import baseline_dir
from harness.stages.s07_board import selected_board
from harness.stages import s07_board
class BoardPipeline(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.root = Path(temp.name)
self.stack = ExitStack()
self.addCleanup(self.stack.close)
self.board = {"name": "sandbox", "march": "nash-m", "board_type": "s100p",
"runtime_dir": "/userdata/bench", "host_env": "TEST_BOARD_HOST"}
self.config = {
"paths": {"venv": {"bench": "/test/venv"}, "harness_python": "python3"},
"perf": {"warmup": 2, "runs": 3},
"watchdog": {"stall_s": 30, "cpu_idle_pct": 1, "poll_s": 1, "hard_ceiling_s": 60,
"io_stall_s": 50, "no_progress_ceiling_s": 55},
}
for name, value in (("CACHE", self.root / "cache"), ("RUNS", self.root / "runs"),
("BENCH_ROOT", self.root), ("STATE", self.root / "state")):
self.stack.enter_context(patch.object(paths, name, value))
self.stack.enter_context(patch.object(paths, "config", return_value=self.config))
self.lookup = self.stack.enter_context(patch.object(s07_board, "board_for_march", return_value=self.board))
spec = registry.JobSpec("cls--resnet18--nash-m--ptq--int8", "cls", "resnet18",
"nash-m", "ptq", "int8", "A", "oracle")
self.job = {
"job_id": spec.job_id, "run_id": "test-run", "task": "cls", "model": "resnet18",
"march": "nash-m", "backend": "ptq", "board": None, "evaluator_version": "0.5.0",
"model_spec": {"input_spec": "1x3x224x224", "resize": "256"},
"data": {"dataset": "imagenet1k", "t1": {"path": "/test/t1.tar", "manifest_sha": "tar-sha",
"content_sha": "content-sha"}},
"gates": registry.gate_snapshot(spec),
}
jd = paths.RUNS / self.job["run_id"] / spec.job_id
self.ctx = StageContext(self.job["run_id"], self.job, jd, jd / "oracle/stages", jd / "oracle/artifacts")
self.ctx.artifacts.mkdir(parents=True)
(jd / "job.json").write_text(json.dumps(self.job))
def write_baseline(self, version="0.4.0", sha="content-sha"):
directory = baseline_dir(self.job)
directory.mkdir(parents=True, exist_ok=True)
(directory / "metrics.json").write_text(json.dumps(
{"evaluator_version": version, "t1_manifest_sha": sha, "metrics": {"top1": 80}}))
return directory
def test_null_board_snapshot_resolves_current_matching_registry(self):
self.assertEqual(selected_board(self.ctx), self.board)
self.lookup.assert_called_once_with("nash-m")
self.assertIsNone(self.job["board"])
self.job["board"] = {"name": "wrong", "march": "nash-e"}
with self.assertRaisesRegex(ValueError, "board march"):
selected_board(self.ctx)
self.job["board"] = None
self.lookup.return_value = {"march": "nash-p"}
with self.assertRaises(ValueError):
selected_board(self.ctx)
def test_commands_use_hbm_and_board_pool(self):
# A BC simulator artifact must not displace the real HBM.
(self.ctx.artifacts / f"{self.job['job_id']}.bc").write_bytes(b"simulator")
for stage in (Board(), Eval()):
command = stage.command(self.ctx)
self.assertEqual(stage.pool, "board")
self.assertEqual(command[command.index("--hbm") + 1],
str(self.ctx.artifacts / f"{self.job['job_id']}.hbm"))
command = Eval().command(self.ctx)
self.assertEqual(command[command.index("--runner") + 1], "board")
self.assertNotIn("--runtime-python", command)
def test_allocated_board_overrides_frozen_snapshot_and_routes_both_stages(self):
self.job["board"] = self.board
second = {**self.board, "name": "sandbox-2", "slot": 1, "runtime_dir": "/test/second",
"host_env": "SECOND_HOST", "port_env": "SECOND_PORT", "ssh_key_env": "SECOND_KEY"}
self.ctx.extra.update(board=second, slot=1)
secrets = {"SECOND_HOST": "second-host", "SECOND_PORT": "2222", "SECOND_KEY": "/test/second-key",
"BOARD_USER": "root"}
with patch.object(paths, "secret", side_effect=secrets.get):
for stage in (Board(), Eval()):
env = stage.env(self.ctx)
self.assertEqual(env["BOARD_HOST"], "second-host")
self.assertEqual(env["BOARD_PORT"], "2222")
self.assertEqual(env["BOARD_SSH_KEY"], "/test/second-key")
self.assertEqual(env["BOARD_NAME"], "sandbox-2")
command = stage.command(self.ctx)
self.assertEqual(command[command.index("--runtime-dir") + 1], "/test/second")
self.assertEqual(self.job["board"], self.board)
self.ctx.extra["board"] = None
self.assertIsNone(selected_board(self.ctx))
self.assertIsNone(Eval().command(self.ctx))
def test_stage_evidence_identifies_allocated_board_without_connection_secrets(self):
self.ctx.extra.update(board={**self.board, "slot": 1}, slot=1)
with patch.object(Board, "command", return_value=None), \
patch.object(Board, "native", return_value=StageOutcome("OK")):
Board().run(self.ctx)
meta = json.loads((self.ctx.stage_root / "07-board/meta.json").read_text())
self.assertEqual(meta["board"], {"name": "sandbox", "slot": 1, "march": "nash-m", "board_type": "s100p"})
def test_only_board_stages_receive_credentials(self):
secrets = {"TEST_BOARD_HOST": "test-host", "BOARD_USER": "test-user", "BOARD_PORT": "22",
"BOARD_SSH_KEY": "/test/key"}
with patch.dict(os.environ, {"BOARD_HOST": "inherited-secret", "TEACHER_API_KEY": "api-secret"}), \
patch.object(paths, "secret", side_effect=secrets.get):
ordinary = Stage().env(self.ctx)
self.assertNotIn("BOARD_HOST", ordinary)
self.assertNotIn("TEACHER_API_KEY", ordinary)
for stage in (Board(), Eval()):
env = stage.env(self.ctx)
self.assertEqual(env["BOARD_HOST"], "test-host")
self.assertEqual(env["BOARD_SSH_KEY"], "/test/key")
self.assertEqual(env["CUDA_VISIBLE_DEVICES"], "")
self.assertNotIn("TEACHER_API_KEY", env)
def test_absent_board_is_unverified_without_simulator(self):
self.lookup.return_value = None
with patch("harness.stages.base.watchdog.run") as run:
outcomes = [Board().run(self.ctx), Eval().run(self.ctx)]
run.assert_not_called()
self.assertTrue(all(out.state == "SKIPPED" for out in outcomes))
self.assertEqual(gates.g3(outcomes[1].measurements["G3"], self.job["gates"]["G3"])["verdict"],
"unverified")
def test_eval_reads_only_current_attempt(self):
stage = Eval()
self.ctx.attempt = 2
current = stage.stage_dir(self.ctx)
stale = current.parent / "eval.json"
stale.write_text(json.dumps({"G3": {"quant": 99, "source": "sim"}}))
command = stage.command(self.ctx)
self.assertEqual(Path(command[command.index("--out") + 1]), current / "eval.json")
self.assertEqual(stage.measure(self.ctx, current, StageOutcome("OK")), {})
(current / "eval.json").write_text(json.dumps({"G3": {"quant": 76, "source": "board"}}))
# A scalar in the current attempt is no longer sufficient evidence:
# the actual evaluated package and evaluator identity must be bound.
with self.assertRaises((OSError, ValueError)):
stage.measure(self.ctx, current, StageOutcome("OK"))
def test_compatible_baseline_reuse_checks_data_and_version(self):
self.write_baseline()
self.assertIsNotNone(Baseline().cached(self.ctx))
self.write_baseline(sha="different-data")
self.assertIsNone(Baseline().cached(self.ctx))
self.write_baseline(version="0.3.0")
self.assertIsNone(Baseline().cached(self.ctx))
def test_backfill_reuses_hbm_then_skips_completed_repeat(self):
self.write_baseline()
hbm = self.ctx.artifacts / f"{self.job['job_id']}.hbm"
hbm.write_bytes(b"existing-hbm")
sha = hashlib.sha256(hbm.read_bytes()).hexdigest()
g1 = dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False, hbm_sha256=sha)
def meta(name, state, measurements):
directory = self.ctx.stage_root / name
directory.mkdir(parents=True, exist_ok=True)
(directory / "meta.json").write_text(json.dumps(
{"state": state, "error_class": None, "attempt": 0, "measurements": measurements}))
meta("06-hbm", "OK", {"G1": g1})
meta("07-board", "SKIPPED", {"G4": None})
meta("08-eval", "FAIL", {})
result_path = self.ctx.job_dir / "oracle/result.json"
result_path.write_text(json.dumps({"terminal": "UNVERIFIED", "stage_failed": "08-eval",
"error_class": "unknown", "stages": {"08-eval": {"state": "FAIL"}}}))
def execute(stage, ctx):
measurements = ({"G4": {"latency_ms_mean": 10}} if stage.name == "07-board" else
{"G3": {"fp32": 80, "quant": 76, "source": "board", "tier": "t1"}})
for value in measurements.values():
value["hbm_sha256"] = sha
meta(stage.name, "OK", measurements)
return StageOutcome("OK", measurements=measurements)
with patch.object(db, "connect", side_effect=lambda: nullcontext(None)), \
patch.object(db, "list_jobs", return_value=[{"job_id": self.job["job_id"], "state": "FAIL"}]), \
patch.object(db, "stage_start") as started, patch.object(db, "stage_finish"), \
patch.object(db, "set_job_state"), \
patch.object(scheduler, "acquire_stage", side_effect=lambda ctx, pool: nullcontext(0)) as acquire, \
patch.object(Board, "run", autospec=True, side_effect=execute), \
patch.object(Eval, "run", autospec=True, side_effect=execute), \
patch.object(Baseline, "run", side_effect=AssertionError("baseline must be reused")), \
patch.object(Compile, "run", side_effect=AssertionError("HBM must not be recompiled")):
result = pipeline.board_backfill(self.job["run_id"], log=lambda _: None)
self.assertEqual(result["backfilled"], 1)
self.assertEqual([call.args[3] for call in started.call_args_list], ["07-board", "08-eval"])
self.assertEqual([call.args[1] for call in acquire.call_args_list], ["board", "board"])
repeated = pipeline.board_backfill(self.job["run_id"], log=lambda _: None)
self.assertEqual(repeated, {"backfilled": 0, "skipped": {"already_measured": 1}})
self.assertEqual(started.call_count, 2)
self.assertEqual(hbm.read_bytes(), b"existing-hbm")
result = json.loads(result_path.read_text())
self.assertEqual(result["terminal"], "PASS")
self.assertIsNone(result["stage_failed"])
self.assertIsNone(result["error_class"])
self.assertEqual(result["stages"]["08-eval"]["state"], "OK")
def test_eval_entrypoint_uses_board_and_propagates_board_failure(self):
from harness.eval import run, board
directory = self.write_baseline()
target = self.root / "eval.json"
args = ["eval", "--mode", "quant", "--task", "cls", "--model", "resnet18",
"--input-spec", "1x3x224x224", "--t1", "unused.tar", "--baseline", str(directory),
"--runner", "board", "--hbm", "model.hbm", "--evaluator-version", "0.4.0", "--out", str(target)]
def quant(runner, t1, spec, baseline, out, **kwargs):
out["G3"] = {"fp32": 80, "quant": 76}
runner = SimpleNamespace(metadata={"hbm_sha256": "test-sha"})
with patch.object(sys, "argv", args), \
patch.object(run.T1, "open", return_value=SimpleNamespace(manifest={"sha256": "content-sha"}, items=[])), \
patch.object(run.importlib, "import_module", return_value=SimpleNamespace(run_quant=quant, run_fp32=lambda *a, **k: None)), \
patch.object(board, "BoardRunner", return_value=nullcontext(runner)) as physical, \
patch.object(run, "HbmRunner") as simulator, redirect_stdout(io.StringIO()):
self.assertEqual(run.main(), 0)
result = json.loads(target.read_text())
self.assertEqual(result["G3"]["source"], "board")
self.assertEqual(result["G2"]["reason"], "G2 reference samples missing")
target.unlink()
physical.side_effect = RuntimeError("board disconnected")
with self.assertRaisesRegex(RuntimeError, "board disconnected"):
run.main()
self.assertFalse(target.exists())
simulator.assert_not_called()
if __name__ == "__main__":
unittest.main()