246 lines
14 KiB
Python
246 lines
14 KiB
Python
"""Board selection, stage isolation, and HBM-only recovery regression checks."""
|
|
from contextlib import ExitStack, nullcontext, redirect_stdout
|
|
import io
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import db, gates, paths, pipeline, registry, scheduler
|
|
from harness.stages import Board, Eval, Baseline, Compile
|
|
from harness.stages.base import Stage, StageContext, StageOutcome
|
|
from harness.stages.s02_baseline import baseline_dir
|
|
from harness.stages.s07_board import selected_board
|
|
from harness.stages import s07_board
|
|
|
|
|
|
class BoardPipeline(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temp.cleanup)
|
|
self.root = Path(temp.name)
|
|
self.stack = ExitStack()
|
|
self.addCleanup(self.stack.close)
|
|
self.board = {"name": "sandbox", "march": "nash-m", "board_type": "s100p",
|
|
"runtime_dir": "/userdata/bench", "host_env": "TEST_BOARD_HOST"}
|
|
self.config = {
|
|
"paths": {"venv": {"bench": "/test/venv"}, "harness_python": "python3"},
|
|
"perf": {"warmup": 2, "runs": 3},
|
|
"watchdog": {"stall_s": 30, "cpu_idle_pct": 1, "poll_s": 1, "hard_ceiling_s": 60,
|
|
"io_stall_s": 50, "no_progress_ceiling_s": 55},
|
|
}
|
|
for name, value in (("CACHE", self.root / "cache"), ("RUNS", self.root / "runs"),
|
|
("BENCH_ROOT", self.root), ("STATE", self.root / "state")):
|
|
self.stack.enter_context(patch.object(paths, name, value))
|
|
self.stack.enter_context(patch.object(paths, "config", return_value=self.config))
|
|
self.lookup = self.stack.enter_context(patch.object(s07_board, "board_for_march", return_value=self.board))
|
|
spec = registry.JobSpec("cls--resnet18--nash-m--ptq--int8", "cls", "resnet18",
|
|
"nash-m", "ptq", "int8", "A", "oracle")
|
|
self.job = {
|
|
"job_id": spec.job_id, "run_id": "test-run", "task": "cls", "model": "resnet18",
|
|
"march": "nash-m", "backend": "ptq", "board": None, "evaluator_version": "0.5.0",
|
|
"model_spec": {"input_spec": "1x3x224x224", "resize": "256"},
|
|
"data": {"dataset": "imagenet1k", "t1": {"path": "/test/t1.tar", "manifest_sha": "tar-sha",
|
|
"content_sha": "content-sha"}},
|
|
"gates": registry.gate_snapshot(spec),
|
|
}
|
|
jd = paths.RUNS / self.job["run_id"] / spec.job_id
|
|
self.ctx = StageContext(self.job["run_id"], self.job, jd, jd / "oracle/stages", jd / "oracle/artifacts")
|
|
self.ctx.artifacts.mkdir(parents=True)
|
|
(jd / "job.json").write_text(json.dumps(self.job))
|
|
|
|
def write_baseline(self, version="0.4.0", sha="content-sha"):
|
|
directory = baseline_dir(self.job)
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
(directory / "metrics.json").write_text(json.dumps(
|
|
{"evaluator_version": version, "t1_manifest_sha": sha, "metrics": {"top1": 80}}))
|
|
return directory
|
|
|
|
def test_null_board_snapshot_resolves_current_matching_registry(self):
|
|
self.assertEqual(selected_board(self.ctx), self.board)
|
|
self.lookup.assert_called_once_with("nash-m")
|
|
self.assertIsNone(self.job["board"])
|
|
self.job["board"] = {"name": "wrong", "march": "nash-e"}
|
|
with self.assertRaisesRegex(ValueError, "board march"):
|
|
selected_board(self.ctx)
|
|
self.job["board"] = None
|
|
self.lookup.return_value = {"march": "nash-p"}
|
|
with self.assertRaises(ValueError):
|
|
selected_board(self.ctx)
|
|
|
|
def test_commands_use_hbm_and_board_pool(self):
|
|
# A BC simulator artifact must not displace the real HBM.
|
|
(self.ctx.artifacts / f"{self.job['job_id']}.bc").write_bytes(b"simulator")
|
|
for stage in (Board(), Eval()):
|
|
command = stage.command(self.ctx)
|
|
self.assertEqual(stage.pool, "board")
|
|
self.assertEqual(command[command.index("--hbm") + 1],
|
|
str(self.ctx.artifacts / f"{self.job['job_id']}.hbm"))
|
|
command = Eval().command(self.ctx)
|
|
self.assertEqual(command[command.index("--runner") + 1], "board")
|
|
self.assertNotIn("--runtime-python", command)
|
|
|
|
def test_allocated_board_overrides_frozen_snapshot_and_routes_both_stages(self):
|
|
self.job["board"] = self.board
|
|
second = {**self.board, "name": "sandbox-2", "slot": 1, "runtime_dir": "/test/second",
|
|
"host_env": "SECOND_HOST", "port_env": "SECOND_PORT", "ssh_key_env": "SECOND_KEY"}
|
|
self.ctx.extra.update(board=second, slot=1)
|
|
secrets = {"SECOND_HOST": "second-host", "SECOND_PORT": "2222", "SECOND_KEY": "/test/second-key",
|
|
"BOARD_USER": "root"}
|
|
with patch.object(paths, "secret", side_effect=secrets.get):
|
|
for stage in (Board(), Eval()):
|
|
env = stage.env(self.ctx)
|
|
self.assertEqual(env["BOARD_HOST"], "second-host")
|
|
self.assertEqual(env["BOARD_PORT"], "2222")
|
|
self.assertEqual(env["BOARD_SSH_KEY"], "/test/second-key")
|
|
self.assertEqual(env["BOARD_NAME"], "sandbox-2")
|
|
command = stage.command(self.ctx)
|
|
self.assertEqual(command[command.index("--runtime-dir") + 1], "/test/second")
|
|
self.assertEqual(self.job["board"], self.board)
|
|
self.ctx.extra["board"] = None
|
|
self.assertIsNone(selected_board(self.ctx))
|
|
self.assertIsNone(Eval().command(self.ctx))
|
|
|
|
def test_stage_evidence_identifies_allocated_board_without_connection_secrets(self):
|
|
self.ctx.extra.update(board={**self.board, "slot": 1}, slot=1)
|
|
with patch.object(Board, "command", return_value=None), \
|
|
patch.object(Board, "native", return_value=StageOutcome("OK")):
|
|
Board().run(self.ctx)
|
|
meta = json.loads((self.ctx.stage_root / "07-board/meta.json").read_text())
|
|
self.assertEqual(meta["board"], {"name": "sandbox", "slot": 1, "march": "nash-m", "board_type": "s100p"})
|
|
|
|
def test_only_board_stages_receive_credentials(self):
|
|
secrets = {"TEST_BOARD_HOST": "test-host", "BOARD_USER": "test-user", "BOARD_PORT": "22",
|
|
"BOARD_SSH_KEY": "/test/key"}
|
|
with patch.dict(os.environ, {"BOARD_HOST": "inherited-secret", "TEACHER_API_KEY": "api-secret"}), \
|
|
patch.object(paths, "secret", side_effect=secrets.get):
|
|
ordinary = Stage().env(self.ctx)
|
|
self.assertNotIn("BOARD_HOST", ordinary)
|
|
self.assertNotIn("TEACHER_API_KEY", ordinary)
|
|
for stage in (Board(), Eval()):
|
|
env = stage.env(self.ctx)
|
|
self.assertEqual(env["BOARD_HOST"], "test-host")
|
|
self.assertEqual(env["BOARD_SSH_KEY"], "/test/key")
|
|
self.assertEqual(env["CUDA_VISIBLE_DEVICES"], "")
|
|
self.assertNotIn("TEACHER_API_KEY", env)
|
|
|
|
def test_absent_board_is_unverified_without_simulator(self):
|
|
self.lookup.return_value = None
|
|
with patch("harness.stages.base.watchdog.run") as run:
|
|
outcomes = [Board().run(self.ctx), Eval().run(self.ctx)]
|
|
run.assert_not_called()
|
|
self.assertTrue(all(out.state == "SKIPPED" for out in outcomes))
|
|
self.assertEqual(gates.g3(outcomes[1].measurements["G3"], self.job["gates"]["G3"])["verdict"],
|
|
"unverified")
|
|
|
|
def test_eval_reads_only_current_attempt(self):
|
|
stage = Eval()
|
|
self.ctx.attempt = 2
|
|
current = stage.stage_dir(self.ctx)
|
|
stale = current.parent / "eval.json"
|
|
stale.write_text(json.dumps({"G3": {"quant": 99, "source": "sim"}}))
|
|
command = stage.command(self.ctx)
|
|
self.assertEqual(Path(command[command.index("--out") + 1]), current / "eval.json")
|
|
self.assertEqual(stage.measure(self.ctx, current, StageOutcome("OK")), {})
|
|
(current / "eval.json").write_text(json.dumps({"G3": {"quant": 76, "source": "board"}}))
|
|
# A scalar in the current attempt is no longer sufficient evidence:
|
|
# the actual evaluated package and evaluator identity must be bound.
|
|
with self.assertRaises((OSError, ValueError)):
|
|
stage.measure(self.ctx, current, StageOutcome("OK"))
|
|
|
|
def test_compatible_baseline_reuse_checks_data_and_version(self):
|
|
self.write_baseline()
|
|
self.assertIsNotNone(Baseline().cached(self.ctx))
|
|
self.write_baseline(sha="different-data")
|
|
self.assertIsNone(Baseline().cached(self.ctx))
|
|
self.write_baseline(version="0.3.0")
|
|
self.assertIsNone(Baseline().cached(self.ctx))
|
|
|
|
def test_backfill_reuses_hbm_then_skips_completed_repeat(self):
|
|
self.write_baseline()
|
|
hbm = self.ctx.artifacts / f"{self.job['job_id']}.hbm"
|
|
hbm.write_bytes(b"existing-hbm")
|
|
sha = hashlib.sha256(hbm.read_bytes()).hexdigest()
|
|
g1 = dict(hbm_exists=True, march_ok=True, io_ok=True, verifier_ok=True, log_has_error=False, hbm_sha256=sha)
|
|
|
|
def meta(name, state, measurements):
|
|
directory = self.ctx.stage_root / name
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
(directory / "meta.json").write_text(json.dumps(
|
|
{"state": state, "error_class": None, "attempt": 0, "measurements": measurements}))
|
|
|
|
meta("06-hbm", "OK", {"G1": g1})
|
|
meta("07-board", "SKIPPED", {"G4": None})
|
|
meta("08-eval", "FAIL", {})
|
|
result_path = self.ctx.job_dir / "oracle/result.json"
|
|
result_path.write_text(json.dumps({"terminal": "UNVERIFIED", "stage_failed": "08-eval",
|
|
"error_class": "unknown", "stages": {"08-eval": {"state": "FAIL"}}}))
|
|
|
|
def execute(stage, ctx):
|
|
measurements = ({"G4": {"latency_ms_mean": 10}} if stage.name == "07-board" else
|
|
{"G3": {"fp32": 80, "quant": 76, "source": "board", "tier": "t1"}})
|
|
for value in measurements.values():
|
|
value["hbm_sha256"] = sha
|
|
meta(stage.name, "OK", measurements)
|
|
return StageOutcome("OK", measurements=measurements)
|
|
|
|
with patch.object(db, "connect", side_effect=lambda: nullcontext(None)), \
|
|
patch.object(db, "list_jobs", return_value=[{"job_id": self.job["job_id"], "state": "FAIL"}]), \
|
|
patch.object(db, "stage_start") as started, patch.object(db, "stage_finish"), \
|
|
patch.object(db, "set_job_state"), \
|
|
patch.object(scheduler, "acquire_stage", side_effect=lambda ctx, pool: nullcontext(0)) as acquire, \
|
|
patch.object(Board, "run", autospec=True, side_effect=execute), \
|
|
patch.object(Eval, "run", autospec=True, side_effect=execute), \
|
|
patch.object(Baseline, "run", side_effect=AssertionError("baseline must be reused")), \
|
|
patch.object(Compile, "run", side_effect=AssertionError("HBM must not be recompiled")):
|
|
result = pipeline.board_backfill(self.job["run_id"], log=lambda _: None)
|
|
self.assertEqual(result["backfilled"], 1)
|
|
self.assertEqual([call.args[3] for call in started.call_args_list], ["07-board", "08-eval"])
|
|
self.assertEqual([call.args[1] for call in acquire.call_args_list], ["board", "board"])
|
|
repeated = pipeline.board_backfill(self.job["run_id"], log=lambda _: None)
|
|
self.assertEqual(repeated, {"backfilled": 0, "skipped": {"already_measured": 1}})
|
|
self.assertEqual(started.call_count, 2)
|
|
self.assertEqual(hbm.read_bytes(), b"existing-hbm")
|
|
result = json.loads(result_path.read_text())
|
|
self.assertEqual(result["terminal"], "PASS")
|
|
self.assertIsNone(result["stage_failed"])
|
|
self.assertIsNone(result["error_class"])
|
|
self.assertEqual(result["stages"]["08-eval"]["state"], "OK")
|
|
|
|
def test_eval_entrypoint_uses_board_and_propagates_board_failure(self):
|
|
from harness.eval import run, board
|
|
directory = self.write_baseline()
|
|
target = self.root / "eval.json"
|
|
args = ["eval", "--mode", "quant", "--task", "cls", "--model", "resnet18",
|
|
"--input-spec", "1x3x224x224", "--t1", "unused.tar", "--baseline", str(directory),
|
|
"--runner", "board", "--hbm", "model.hbm", "--evaluator-version", "0.4.0", "--out", str(target)]
|
|
|
|
def quant(runner, t1, spec, baseline, out, **kwargs):
|
|
out["G3"] = {"fp32": 80, "quant": 76}
|
|
|
|
runner = SimpleNamespace(metadata={"hbm_sha256": "test-sha"})
|
|
with patch.object(sys, "argv", args), \
|
|
patch.object(run.T1, "open", return_value=SimpleNamespace(manifest={"sha256": "content-sha"}, items=[])), \
|
|
patch.object(run.importlib, "import_module", return_value=SimpleNamespace(run_quant=quant, run_fp32=lambda *a, **k: None)), \
|
|
patch.object(board, "BoardRunner", return_value=nullcontext(runner)) as physical, \
|
|
patch.object(run, "HbmRunner") as simulator, redirect_stdout(io.StringIO()):
|
|
self.assertEqual(run.main(), 0)
|
|
result = json.loads(target.read_text())
|
|
self.assertEqual(result["G3"]["source"], "board")
|
|
self.assertEqual(result["G2"]["reason"], "G2 reference samples missing")
|
|
target.unlink()
|
|
physical.side_effect = RuntimeError("board disconnected")
|
|
with self.assertRaisesRegex(RuntimeError, "board disconnected"):
|
|
run.main()
|
|
self.assertFalse(target.exists())
|
|
simulator.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|