data-gen-bpu/scripts/test_board_runtime.py

236 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""板端推理协议回归测试:不访问 SSH,不占用真实板卡。"""
import hashlib
import json
import re
import shutil
import tempfile
import unittest
from pathlib import Path
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import patch
import numpy as np
from harness.eval.board import BoardRunner
from harness.tools import board_bench, board_perf
class Runtime:
model_names = ["model"]
input_names = {"model": ["data"]}
input_shapes = {"model": {"data": [1, 3, 2, 2]}}
input_types = {"model": {"data": "FLOAT32"}}
output_names = {"model": [f"output{i}" for i in range(12)]}
def __init__(self):
self.buffers = {name: np.empty(1, np.float32) for name in self.output_names["model"]}
def run(self, feed):
for i, name in enumerate(self.output_names["model"]):
self.buffers[name][:] = feed["model"]["data"].mean() + i
# 返回字典顺序与模型输出名顺序相反,同时重用缓冲区。
return {"model": dict(reversed(list(self.buffers.items())))}
class Session:
instances = []
fail = False
corrupt = False
mismatch = False
def __init__(self, hbm, runtime_dir):
self.hbm = Path(hbm)
self.local = tempfile.TemporaryDirectory()
self.directory = Path(self.local.name)
self.closed = False
self.uploads = 0
self.instances.append(self)
def __enter__(self):
return self
def path(self, name):
return str(self.directory / name)
def upload(self, source, name):
shutil.copyfile(source, self.path(name))
self.uploads += 1
def download(self, name, target):
shutil.copyfile(self.path(name), target)
def bench(self, args):
if self.fail:
raise SystemExit(17)
n = board_bench.infer_inputs(Runtime(), "model", args[1], args[3])
if self.corrupt:
np.savez(args[3], **{"s0000/out0": np.ones(1)})
return {"samples": n, "outputs": Runtime.output_names["model"], "source": "board",
"hbm_sha256": "wrong" if self.mismatch else hashlib.sha256(self.hbm.read_bytes()).hexdigest(),
"board_type": "D-Robotics RDK S100P V1P0", "ucp": "3.13.6"}
def remove(self, *names):
for name in names:
Path(self.path(name)).unlink()
def close(self):
self.closed = True
self.local.cleanup()
def sample(value=1, dtype=np.float32):
return np.full((1, 3, 2, 2), value, dtype=dtype)
class BoardRuntimeTests(unittest.TestCase):
def setUp(self):
Session.instances = []
Session.fail = Session.corrupt = Session.mismatch = False
def test_bulk_copies_reused_buffers_in_declared_output_order(self):
with tempfile.TemporaryDirectory() as td:
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
np.savez(fin, s0001=sample(3), s0000=sample(1))
self.assertEqual(board_bench.infer_inputs(Runtime(), "model", str(fin), str(fout)), 2)
with np.load(fout) as outputs:
self.assertEqual(outputs["s0000/out0"].item(), 1)
self.assertEqual(outputs["s0000/out10"].item(), 11)
self.assertEqual(outputs["s0001/out0"].item(), 3)
def test_bulk_rejects_wrong_shape_dtype_and_multiple_inputs(self):
with tempfile.TemporaryDirectory() as td:
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
for x in (sample().reshape(1, 3, 1, 4), sample(dtype=np.uint8)):
np.savez(fin, s0000=x)
with self.assertRaises(ValueError):
board_bench.infer_inputs(Runtime(), "model", str(fin), str(fout))
self.assertFalse(fout.exists())
runtime = Runtime()
runtime.input_names = {"model": ["first", "second"]}
with self.assertRaisesRegex(ValueError, "one input"):
board_bench.infer_inputs(runtime, "model", str(fin), str(fout))
def test_bulk_runtime_type_accepts_f32_enum_but_rejects_other_dtypes(self):
with tempfile.TemporaryDirectory() as td:
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
np.savez(fin, s0000=sample())
runtime = Runtime()
runtime.input_types = {"model": {"data": "TensorType.F32"}}
self.assertEqual(board_bench.infer_inputs(runtime, "model", str(fin), str(fout)), 1)
for dtype in ("FLOAT16", "INT8", "UINT8"):
runtime.input_types = {"model": {"data": dtype}}
with self.assertRaisesRegex(ValueError, "FLOAT32"):
board_bench.infer_inputs(runtime, "model", str(fin), str(fout))
def test_runner_chunks_numeric_outputs_and_cleanup(self):
with tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", Session):
hbm = Path(td) / "model.hbm"
hbm.write_bytes(b"compiled-model")
with BoardRunner(str(hbm)) as runner:
runner.CHUNK = 2
outs = runner.batch([sample(v) for v in range(5)])
self.assertEqual([o[0].item() for o in outs], list(range(5)))
self.assertEqual(outs[0][10].item(), 10)
self.assertEqual(runner(sample(6))[0].item(), 6)
self.assertEqual(runner.metadata["source"], "board")
self.assertIn("S100P", runner.metadata["board_type"])
self.assertEqual(list(runner.tmp.iterdir()), [])
self.assertEqual(len(Session.instances), 1)
self.assertEqual(Session.instances[0].uploads, 4)
self.assertEqual(list(Session.instances[0].directory.iterdir()), [])
self.assertFalse(runner.tmp.exists())
self.assertTrue(Session.instances[0].closed)
def test_runner_failure_and_invalid_outputs_cleanup(self):
for mode, error in (("fail", SystemExit), ("corrupt", RuntimeError), ("mismatch", RuntimeError)):
with self.subTest(mode=mode), tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", Session):
hbm = Path(td) / "model.hbm"
hbm.write_bytes(b"compiled-model")
setattr(Session, mode, True)
runner = BoardRunner(str(hbm))
with self.assertRaises(error):
runner.batch([sample()])
self.assertFalse(runner.tmp.exists())
self.assertTrue(Session.instances[-1].closed)
setattr(Session, mode, False)
def test_runner_setup_failure_cleans_local_temp(self):
with tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", side_effect=SystemExit(4)):
hbm = Path(td) / "model.hbm"
hbm.write_bytes(b"compiled-model")
runner = BoardRunner(str(hbm))
with self.assertRaises(SystemExit):
runner.__enter__()
self.assertFalse(runner.tmp.exists())
def test_ssh_error_is_not_hidden_by_valid_json(self):
result = SimpleNamespace(returncode=17, stderr="remote inference failed", stdout='{"valid": true}\n')
with patch.object(board_perf.subprocess, "run", return_value=result):
with self.assertRaises(SystemExit) as raised:
board_perf.run(["ssh", "board", "python3 board_bench.py"])
self.assertEqual(raised.exception.code, 17)
def test_remote_session_uploads_once_and_quotes_shell_paths(self):
calls = []
def fake_run(command, timeout=1800):
calls.append(command)
return 'runtime startup\n{"samples": 1}\n' if "python3 " in command[-1] else ""
with patch.object(board_perf, "ssh_base", return_value=(["-p", "2222"], "root@board")), patch.object(board_perf, "run", side_effect=fake_run):
with board_perf.RemoteSession("/local/model.hbm", "/userdata/bench space") as session:
session.__enter__()
self.assertEqual(session.bench(["--runs", "1"])["samples"], 1)
scps = [c for c in calls if c[0] == "scp"]
self.assertEqual(len(scps), 2)
self.assertEqual(scps[0][1:3], ["-P", "2222"])
self.assertIn("'/userdata/bench space/", calls[0][-1])
self.assertTrue(calls[-1][-1].startswith("rm -rf -- "))
bench = next(c[-1] for c in calls if c[-1].startswith("python3 "))
self.assertNotIn("|", bench)
self.assertNotIn("2>/dev/null", bench)
def test_remote_setup_failure_keeps_exit_code_and_cleans_owned_directory(self):
calls = []
def fake_run(command, timeout=1800):
calls.append(command)
if command[0] == "scp":
raise SystemExit(23)
return ""
with patch.object(board_perf, "ssh_base", return_value=([], "root@board")), patch.object(board_perf, "run", side_effect=fake_run):
session = board_perf.RemoteSession("/local/model.hbm")
with self.assertRaises(SystemExit) as raised:
session.__enter__()
self.assertEqual(raised.exception.code, 23)
self.assertEqual(calls[-1][-1], "rm -rf -- " + session.directory)
self.assertFalse(session.active)
def test_busy_probe_does_not_match_own_shell_and_logs_real_wait(self):
session = SimpleNamespace(bench=lambda args: {"latency_ms_mean": 1.0})
with tempfile.TemporaryDirectory() as td:
out = Path(td) / "perf.json"
argv = ["board_perf", "--hbm", "model.hbm", "--job-id", "job", "--out", str(out)]
with patch.object(board_perf.sys, "argv", argv), \
patch.object(board_perf, "ssh_base", return_value=([], "root@board")), \
patch.object(board_perf, "run", side_effect=["1\n10\n", "0\n0\n"]) as run, \
patch.object(board_perf, "RemoteSession", return_value=nullcontext(session)), \
patch.object(board_perf.time, "sleep") as sleep, patch("builtins.print") as log:
self.assertEqual(board_perf.main(), 0)
command = run.call_args_list[0].args[0][-1]
pattern = command.split("'")[1]
self.assertIsNone(re.search(pattern, command))
for process in ("python3 board_bench.py", "hrt_model_exec", "hbm_runtime", "dnn"):
self.assertIsNotNone(re.search(pattern, process))
sleep.assert_called_once_with(20)
self.assertTrue(any(c.kwargs.get("flush") and "board busy" in c.args[0] for c in log.call_args_list))
result = json.loads(out.read_text())
self.assertEqual(result["busy_waited_s"], 20)
self.assertFalse(result["contended"])
if __name__ == "__main__":
unittest.main()