"""板端推理协议回归测试:不访问 SSH,不占用真实板卡。""" import hashlib import json import re import shutil import tempfile import unittest from pathlib import Path from contextlib import nullcontext from types import SimpleNamespace from unittest.mock import patch import numpy as np from harness.eval.board import BoardRunner from harness.tools import board_bench, board_perf class Runtime: model_names = ["model"] input_names = {"model": ["data"]} input_shapes = {"model": {"data": [1, 3, 2, 2]}} input_types = {"model": {"data": "FLOAT32"}} output_names = {"model": [f"output{i}" for i in range(12)]} def __init__(self): self.buffers = {name: np.empty(1, np.float32) for name in self.output_names["model"]} def run(self, feed): for i, name in enumerate(self.output_names["model"]): self.buffers[name][:] = feed["model"]["data"].mean() + i # 返回字典顺序与模型输出名顺序相反,同时重用缓冲区。 return {"model": dict(reversed(list(self.buffers.items())))} class Session: instances = [] fail = False corrupt = False mismatch = False def __init__(self, hbm, runtime_dir): self.hbm = Path(hbm) self.local = tempfile.TemporaryDirectory() self.directory = Path(self.local.name) self.closed = False self.uploads = 0 self.instances.append(self) def __enter__(self): return self def path(self, name): return str(self.directory / name) def upload(self, source, name): shutil.copyfile(source, self.path(name)) self.uploads += 1 def download(self, name, target): shutil.copyfile(self.path(name), target) def bench(self, args): if self.fail: raise SystemExit(17) n = board_bench.infer_inputs(Runtime(), "model", args[1], args[3]) if self.corrupt: np.savez(args[3], **{"s0000/out0": np.ones(1)}) return {"samples": n, "outputs": Runtime.output_names["model"], "source": "board", "hbm_sha256": "wrong" if self.mismatch else hashlib.sha256(self.hbm.read_bytes()).hexdigest(), "board_type": "D-Robotics RDK S100P V1P0", "ucp": "3.13.6"} def remove(self, *names): for name in names: Path(self.path(name)).unlink() def close(self): self.closed = True self.local.cleanup() def sample(value=1, dtype=np.float32): return np.full((1, 3, 2, 2), value, dtype=dtype) class BoardRuntimeTests(unittest.TestCase): def setUp(self): Session.instances = [] Session.fail = Session.corrupt = Session.mismatch = False def test_bulk_copies_reused_buffers_in_declared_output_order(self): with tempfile.TemporaryDirectory() as td: fin, fout = Path(td) / "in.npz", Path(td) / "out.npz" np.savez(fin, s0001=sample(3), s0000=sample(1)) self.assertEqual(board_bench.infer_inputs(Runtime(), "model", str(fin), str(fout)), 2) with np.load(fout) as outputs: self.assertEqual(outputs["s0000/out0"].item(), 1) self.assertEqual(outputs["s0000/out10"].item(), 11) self.assertEqual(outputs["s0001/out0"].item(), 3) def test_bulk_rejects_wrong_shape_dtype_and_multiple_inputs(self): with tempfile.TemporaryDirectory() as td: fin, fout = Path(td) / "in.npz", Path(td) / "out.npz" for x in (sample().reshape(1, 3, 1, 4), sample(dtype=np.uint8)): np.savez(fin, s0000=x) with self.assertRaises(ValueError): board_bench.infer_inputs(Runtime(), "model", str(fin), str(fout)) self.assertFalse(fout.exists()) runtime = Runtime() runtime.input_names = {"model": ["first", "second"]} with self.assertRaisesRegex(ValueError, "one input"): board_bench.infer_inputs(runtime, "model", str(fin), str(fout)) def test_bulk_runtime_type_accepts_f32_enum_but_rejects_other_dtypes(self): with tempfile.TemporaryDirectory() as td: fin, fout = Path(td) / "in.npz", Path(td) / "out.npz" np.savez(fin, s0000=sample()) runtime = Runtime() runtime.input_types = {"model": {"data": "TensorType.F32"}} self.assertEqual(board_bench.infer_inputs(runtime, "model", str(fin), str(fout)), 1) for dtype in ("FLOAT16", "INT8", "UINT8"): runtime.input_types = {"model": {"data": dtype}} with self.assertRaisesRegex(ValueError, "FLOAT32"): board_bench.infer_inputs(runtime, "model", str(fin), str(fout)) def test_runner_chunks_numeric_outputs_and_cleanup(self): with tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", Session): hbm = Path(td) / "model.hbm" hbm.write_bytes(b"compiled-model") with BoardRunner(str(hbm)) as runner: runner.CHUNK = 2 outs = runner.batch([sample(v) for v in range(5)]) self.assertEqual([o[0].item() for o in outs], list(range(5))) self.assertEqual(outs[0][10].item(), 10) self.assertEqual(runner(sample(6))[0].item(), 6) self.assertEqual(runner.metadata["source"], "board") self.assertIn("S100P", runner.metadata["board_type"]) self.assertEqual(list(runner.tmp.iterdir()), []) self.assertEqual(len(Session.instances), 1) self.assertEqual(Session.instances[0].uploads, 4) self.assertEqual(list(Session.instances[0].directory.iterdir()), []) self.assertFalse(runner.tmp.exists()) self.assertTrue(Session.instances[0].closed) def test_runner_failure_and_invalid_outputs_cleanup(self): for mode, error in (("fail", SystemExit), ("corrupt", RuntimeError), ("mismatch", RuntimeError)): with self.subTest(mode=mode), tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", Session): hbm = Path(td) / "model.hbm" hbm.write_bytes(b"compiled-model") setattr(Session, mode, True) runner = BoardRunner(str(hbm)) with self.assertRaises(error): runner.batch([sample()]) self.assertFalse(runner.tmp.exists()) self.assertTrue(Session.instances[-1].closed) setattr(Session, mode, False) def test_runner_setup_failure_cleans_local_temp(self): with tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", side_effect=SystemExit(4)): hbm = Path(td) / "model.hbm" hbm.write_bytes(b"compiled-model") runner = BoardRunner(str(hbm)) with self.assertRaises(SystemExit): runner.__enter__() self.assertFalse(runner.tmp.exists()) def test_ssh_error_is_not_hidden_by_valid_json(self): result = SimpleNamespace(returncode=17, stderr="remote inference failed", stdout='{"valid": true}\n') with patch.object(board_perf.subprocess, "run", return_value=result): with self.assertRaises(SystemExit) as raised: board_perf.run(["ssh", "board", "python3 board_bench.py"]) self.assertEqual(raised.exception.code, 17) def test_remote_session_uploads_once_and_quotes_shell_paths(self): calls = [] def fake_run(command, timeout=1800): calls.append(command) return 'runtime startup\n{"samples": 1}\n' if "python3 " in command[-1] else "" with patch.object(board_perf, "ssh_base", return_value=(["-p", "2222"], "root@board")), \ patch.object(board_perf, "run", side_effect=fake_run), \ patch.object(board_perf, "guarded_run", side_effect=fake_run): with board_perf.RemoteSession("/local/model.hbm", "/userdata/bench space") as session: session.__enter__() self.assertEqual(session.bench(["--runs", "1"])["samples"], 1) scps = [c for c in calls if c[0] == "scp"] self.assertEqual(len(scps), 3) self.assertEqual(scps[0][1:3], ["-P", "2222"]) self.assertIn("'/userdata/bench space/", calls[0][-1]) self.assertTrue(calls[-1][-1].startswith("rm -rf -- ")) bench = next(c[-1] for c in calls if c[-1].startswith("python3 ")) self.assertNotIn("|", bench) self.assertNotIn("2>/dev/null", bench) self.assertIn('board_guard.py', bench) def test_remote_setup_failure_keeps_exit_code_and_cleans_owned_directory(self): calls = [] def fake_run(command, timeout=1800): calls.append(command) if command[0] == "scp": raise SystemExit(23) return "" with patch.object(board_perf, "ssh_base", return_value=([], "root@board")), patch.object(board_perf, "run", side_effect=fake_run): session = board_perf.RemoteSession("/local/model.hbm") with self.assertRaises(SystemExit) as raised: session.__enter__() self.assertEqual(raised.exception.code, 23) self.assertEqual(calls[-1][-1], "rm -rf -- " + session.directory) self.assertFalse(session.active) def test_busy_probe_does_not_match_own_shell_and_logs_real_wait(self): session = SimpleNamespace(bench=lambda args: {"latency_ms_mean": 1.0}) with tempfile.TemporaryDirectory() as td: out = Path(td) / "perf.json" argv = ["board_perf", "--hbm", "model.hbm", "--job-id", "job", "--out", str(out)] with patch.object(board_perf.sys, "argv", argv), \ patch.object(board_perf, "ssh_base", return_value=([], "root@board")), \ patch.object(board_perf, "run", side_effect=["1\n10\n", "0\n0\n"]) as run, \ patch.object(board_perf, "RemoteSession", return_value=nullcontext(session)), \ patch.object(board_perf.time, "sleep") as sleep, patch("builtins.print") as log: self.assertEqual(board_perf.main(), 0) command = run.call_args_list[0].args[0][-1] pattern = command.split("'")[1] self.assertIsNone(re.search(pattern, command)) for process in ("python3 board_bench.py", "hrt_model_exec", "hbm_runtime", "dnn"): self.assertIsNotNone(re.search(pattern, process)) sleep.assert_called_once_with(20) self.assertTrue(any(c.kwargs.get("flush") and "board busy" in c.args[0] for c in log.call_args_list)) result = json.loads(out.read_text()) self.assertEqual(result["busy_waited_s"], 20) self.assertFalse(result["contended"]) def test_busy_board_refuses_to_measure_after_wait_budget(self): with tempfile.TemporaryDirectory() as td: out = Path(td) / 'perf.json' argv = ['board_perf', '--hbm', 'm.hbm', '--job-id', 'j', '--out', str(out), '--busy-wait-s', '0'] with patch.object(board_perf.sys, 'argv', argv), \ patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \ patch.object(board_perf, 'run', return_value='1\n0\n'), \ patch.object(board_perf, 'RemoteSession') as session: with self.assertRaisesRegex(RuntimeError, 'refusing contended'): board_perf.main() session.assert_not_called() self.assertFalse(out.exists()) def test_failed_guard_retains_evidence_without_erasing_live_directory(self): with patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \ patch.object(board_perf, 'run', return_value='') as run, \ patch.object(board_perf, 'guarded_run', side_effect=SystemExit(124)): with self.assertRaises(SystemExit): with board_perf.RemoteSession('model.hbm') as session: session.bench(['--runs', '1']) self.assertFalse(any(c.args[0][-1].startswith('rm ') for c in run.call_args_list)) if __name__ == "__main__": unittest.main()