236 lines
11 KiB
Python
236 lines
11 KiB
Python
"""板端推理协议回归测试:不访问 SSH,不占用真实板卡。"""
|
||
import hashlib
|
||
import json
|
||
import re
|
||
import shutil
|
||
import tempfile
|
||
import unittest
|
||
from pathlib import Path
|
||
from contextlib import nullcontext
|
||
from types import SimpleNamespace
|
||
from unittest.mock import patch
|
||
|
||
import numpy as np
|
||
|
||
from harness.eval.board import BoardRunner
|
||
from harness.tools import board_bench, board_perf
|
||
|
||
|
||
class Runtime:
|
||
model_names = ["model"]
|
||
input_names = {"model": ["data"]}
|
||
input_shapes = {"model": {"data": [1, 3, 2, 2]}}
|
||
input_types = {"model": {"data": "FLOAT32"}}
|
||
output_names = {"model": [f"output{i}" for i in range(12)]}
|
||
|
||
def __init__(self):
|
||
self.buffers = {name: np.empty(1, np.float32) for name in self.output_names["model"]}
|
||
|
||
def run(self, feed):
|
||
for i, name in enumerate(self.output_names["model"]):
|
||
self.buffers[name][:] = feed["model"]["data"].mean() + i
|
||
# 返回字典顺序与模型输出名顺序相反,同时重用缓冲区。
|
||
return {"model": dict(reversed(list(self.buffers.items())))}
|
||
|
||
|
||
class Session:
|
||
instances = []
|
||
fail = False
|
||
corrupt = False
|
||
mismatch = False
|
||
|
||
def __init__(self, hbm, runtime_dir):
|
||
self.hbm = Path(hbm)
|
||
self.local = tempfile.TemporaryDirectory()
|
||
self.directory = Path(self.local.name)
|
||
self.closed = False
|
||
self.uploads = 0
|
||
self.instances.append(self)
|
||
|
||
def __enter__(self):
|
||
return self
|
||
|
||
def path(self, name):
|
||
return str(self.directory / name)
|
||
|
||
def upload(self, source, name):
|
||
shutil.copyfile(source, self.path(name))
|
||
self.uploads += 1
|
||
|
||
def download(self, name, target):
|
||
shutil.copyfile(self.path(name), target)
|
||
|
||
def bench(self, args):
|
||
if self.fail:
|
||
raise SystemExit(17)
|
||
n = board_bench.infer_inputs(Runtime(), "model", args[1], args[3])
|
||
if self.corrupt:
|
||
np.savez(args[3], **{"s0000/out0": np.ones(1)})
|
||
return {"samples": n, "outputs": Runtime.output_names["model"], "source": "board",
|
||
"hbm_sha256": "wrong" if self.mismatch else hashlib.sha256(self.hbm.read_bytes()).hexdigest(),
|
||
"board_type": "D-Robotics RDK S100P V1P0", "ucp": "3.13.6"}
|
||
|
||
def remove(self, *names):
|
||
for name in names:
|
||
Path(self.path(name)).unlink()
|
||
|
||
def close(self):
|
||
self.closed = True
|
||
self.local.cleanup()
|
||
|
||
|
||
def sample(value=1, dtype=np.float32):
|
||
return np.full((1, 3, 2, 2), value, dtype=dtype)
|
||
|
||
|
||
class BoardRuntimeTests(unittest.TestCase):
|
||
def setUp(self):
|
||
Session.instances = []
|
||
Session.fail = Session.corrupt = Session.mismatch = False
|
||
|
||
def test_bulk_copies_reused_buffers_in_declared_output_order(self):
|
||
with tempfile.TemporaryDirectory() as td:
|
||
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
|
||
np.savez(fin, s0001=sample(3), s0000=sample(1))
|
||
self.assertEqual(board_bench.infer_inputs(Runtime(), "model", str(fin), str(fout)), 2)
|
||
with np.load(fout) as outputs:
|
||
self.assertEqual(outputs["s0000/out0"].item(), 1)
|
||
self.assertEqual(outputs["s0000/out10"].item(), 11)
|
||
self.assertEqual(outputs["s0001/out0"].item(), 3)
|
||
|
||
def test_bulk_rejects_wrong_shape_dtype_and_multiple_inputs(self):
|
||
with tempfile.TemporaryDirectory() as td:
|
||
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
|
||
for x in (sample().reshape(1, 3, 1, 4), sample(dtype=np.uint8)):
|
||
np.savez(fin, s0000=x)
|
||
with self.assertRaises(ValueError):
|
||
board_bench.infer_inputs(Runtime(), "model", str(fin), str(fout))
|
||
self.assertFalse(fout.exists())
|
||
runtime = Runtime()
|
||
runtime.input_names = {"model": ["first", "second"]}
|
||
with self.assertRaisesRegex(ValueError, "one input"):
|
||
board_bench.infer_inputs(runtime, "model", str(fin), str(fout))
|
||
|
||
def test_bulk_runtime_type_accepts_f32_enum_but_rejects_other_dtypes(self):
|
||
with tempfile.TemporaryDirectory() as td:
|
||
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
|
||
np.savez(fin, s0000=sample())
|
||
runtime = Runtime()
|
||
runtime.input_types = {"model": {"data": "TensorType.F32"}}
|
||
self.assertEqual(board_bench.infer_inputs(runtime, "model", str(fin), str(fout)), 1)
|
||
for dtype in ("FLOAT16", "INT8", "UINT8"):
|
||
runtime.input_types = {"model": {"data": dtype}}
|
||
with self.assertRaisesRegex(ValueError, "FLOAT32"):
|
||
board_bench.infer_inputs(runtime, "model", str(fin), str(fout))
|
||
|
||
def test_runner_chunks_numeric_outputs_and_cleanup(self):
|
||
with tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", Session):
|
||
hbm = Path(td) / "model.hbm"
|
||
hbm.write_bytes(b"compiled-model")
|
||
with BoardRunner(str(hbm)) as runner:
|
||
runner.CHUNK = 2
|
||
outs = runner.batch([sample(v) for v in range(5)])
|
||
self.assertEqual([o[0].item() for o in outs], list(range(5)))
|
||
self.assertEqual(outs[0][10].item(), 10)
|
||
self.assertEqual(runner(sample(6))[0].item(), 6)
|
||
self.assertEqual(runner.metadata["source"], "board")
|
||
self.assertIn("S100P", runner.metadata["board_type"])
|
||
self.assertEqual(list(runner.tmp.iterdir()), [])
|
||
self.assertEqual(len(Session.instances), 1)
|
||
self.assertEqual(Session.instances[0].uploads, 4)
|
||
self.assertEqual(list(Session.instances[0].directory.iterdir()), [])
|
||
self.assertFalse(runner.tmp.exists())
|
||
self.assertTrue(Session.instances[0].closed)
|
||
|
||
def test_runner_failure_and_invalid_outputs_cleanup(self):
|
||
for mode, error in (("fail", SystemExit), ("corrupt", RuntimeError), ("mismatch", RuntimeError)):
|
||
with self.subTest(mode=mode), tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", Session):
|
||
hbm = Path(td) / "model.hbm"
|
||
hbm.write_bytes(b"compiled-model")
|
||
setattr(Session, mode, True)
|
||
runner = BoardRunner(str(hbm))
|
||
with self.assertRaises(error):
|
||
runner.batch([sample()])
|
||
self.assertFalse(runner.tmp.exists())
|
||
self.assertTrue(Session.instances[-1].closed)
|
||
setattr(Session, mode, False)
|
||
|
||
def test_runner_setup_failure_cleans_local_temp(self):
|
||
with tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", side_effect=SystemExit(4)):
|
||
hbm = Path(td) / "model.hbm"
|
||
hbm.write_bytes(b"compiled-model")
|
||
runner = BoardRunner(str(hbm))
|
||
with self.assertRaises(SystemExit):
|
||
runner.__enter__()
|
||
self.assertFalse(runner.tmp.exists())
|
||
|
||
def test_ssh_error_is_not_hidden_by_valid_json(self):
|
||
result = SimpleNamespace(returncode=17, stderr="remote inference failed", stdout='{"valid": true}\n')
|
||
with patch.object(board_perf.subprocess, "run", return_value=result):
|
||
with self.assertRaises(SystemExit) as raised:
|
||
board_perf.run(["ssh", "board", "python3 board_bench.py"])
|
||
self.assertEqual(raised.exception.code, 17)
|
||
|
||
def test_remote_session_uploads_once_and_quotes_shell_paths(self):
|
||
calls = []
|
||
|
||
def fake_run(command, timeout=1800):
|
||
calls.append(command)
|
||
return 'runtime startup\n{"samples": 1}\n' if "python3 " in command[-1] else ""
|
||
|
||
with patch.object(board_perf, "ssh_base", return_value=(["-p", "2222"], "root@board")), patch.object(board_perf, "run", side_effect=fake_run):
|
||
with board_perf.RemoteSession("/local/model.hbm", "/userdata/bench space") as session:
|
||
session.__enter__()
|
||
self.assertEqual(session.bench(["--runs", "1"])["samples"], 1)
|
||
scps = [c for c in calls if c[0] == "scp"]
|
||
self.assertEqual(len(scps), 2)
|
||
self.assertEqual(scps[0][1:3], ["-P", "2222"])
|
||
self.assertIn("'/userdata/bench space/", calls[0][-1])
|
||
self.assertTrue(calls[-1][-1].startswith("rm -rf -- "))
|
||
bench = next(c[-1] for c in calls if c[-1].startswith("python3 "))
|
||
self.assertNotIn("|", bench)
|
||
self.assertNotIn("2>/dev/null", bench)
|
||
|
||
def test_remote_setup_failure_keeps_exit_code_and_cleans_owned_directory(self):
|
||
calls = []
|
||
|
||
def fake_run(command, timeout=1800):
|
||
calls.append(command)
|
||
if command[0] == "scp":
|
||
raise SystemExit(23)
|
||
return ""
|
||
|
||
with patch.object(board_perf, "ssh_base", return_value=([], "root@board")), patch.object(board_perf, "run", side_effect=fake_run):
|
||
session = board_perf.RemoteSession("/local/model.hbm")
|
||
with self.assertRaises(SystemExit) as raised:
|
||
session.__enter__()
|
||
self.assertEqual(raised.exception.code, 23)
|
||
self.assertEqual(calls[-1][-1], "rm -rf -- " + session.directory)
|
||
self.assertFalse(session.active)
|
||
|
||
def test_busy_probe_does_not_match_own_shell_and_logs_real_wait(self):
|
||
session = SimpleNamespace(bench=lambda args: {"latency_ms_mean": 1.0})
|
||
with tempfile.TemporaryDirectory() as td:
|
||
out = Path(td) / "perf.json"
|
||
argv = ["board_perf", "--hbm", "model.hbm", "--job-id", "job", "--out", str(out)]
|
||
with patch.object(board_perf.sys, "argv", argv), \
|
||
patch.object(board_perf, "ssh_base", return_value=([], "root@board")), \
|
||
patch.object(board_perf, "run", side_effect=["1\n10\n", "0\n0\n"]) as run, \
|
||
patch.object(board_perf, "RemoteSession", return_value=nullcontext(session)), \
|
||
patch.object(board_perf.time, "sleep") as sleep, patch("builtins.print") as log:
|
||
self.assertEqual(board_perf.main(), 0)
|
||
command = run.call_args_list[0].args[0][-1]
|
||
pattern = command.split("'")[1]
|
||
self.assertIsNone(re.search(pattern, command))
|
||
for process in ("python3 board_bench.py", "hrt_model_exec", "hbm_runtime", "dnn"):
|
||
self.assertIsNotNone(re.search(pattern, process))
|
||
sleep.assert_called_once_with(20)
|
||
self.assertTrue(any(c.kwargs.get("flush") and "board busy" in c.args[0] for c in log.call_args_list))
|
||
result = json.loads(out.read_text())
|
||
self.assertEqual(result["busy_waited_s"], 20)
|
||
self.assertFalse(result["contended"])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|