411 lines
21 KiB
Python
411 lines
21 KiB
Python
"""板端推理协议回归测试:不访问 SSH,不占用真实板卡。"""
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import shutil
|
||
import shlex
|
||
import tempfile
|
||
import unittest
|
||
from pathlib import Path
|
||
from contextlib import nullcontext
|
||
from types import SimpleNamespace
|
||
from unittest.mock import Mock, patch
|
||
|
||
import numpy as np
|
||
|
||
from harness.eval.board import BoardRunner
|
||
from harness.tools import board_bench, board_perf
|
||
|
||
|
||
class Runtime:
|
||
model_names = ["model"]
|
||
input_names = {"model": ["data"]}
|
||
input_shapes = {"model": {"data": [1, 3, 2, 2]}}
|
||
input_types = {"model": {"data": "FLOAT32"}}
|
||
output_names = {"model": [f"output{i}" for i in range(12)]}
|
||
|
||
def __init__(self):
|
||
self.buffers = {name: np.empty(1, np.float32) for name in self.output_names["model"]}
|
||
|
||
def run(self, feed):
|
||
for i, name in enumerate(self.output_names["model"]):
|
||
self.buffers[name][:] = feed["model"]["data"].mean() + i
|
||
# 返回字典顺序与模型输出名顺序相反,同时重用缓冲区。
|
||
return {"model": dict(reversed(list(self.buffers.items())))}
|
||
|
||
|
||
class Session:
|
||
instances = []
|
||
fail = False
|
||
corrupt = False
|
||
mismatch = False
|
||
|
||
def __init__(self, hbm, runtime_dir):
|
||
self.hbm = Path(hbm)
|
||
self.local = tempfile.TemporaryDirectory()
|
||
self.directory = Path(self.local.name)
|
||
self.closed = False
|
||
self.uploads = 0
|
||
self.instances.append(self)
|
||
|
||
def __enter__(self):
|
||
return self
|
||
|
||
def path(self, name):
|
||
return str(self.directory / name)
|
||
|
||
def upload(self, source, name):
|
||
shutil.copyfile(source, self.path(name))
|
||
self.uploads += 1
|
||
|
||
def download(self, name, target):
|
||
shutil.copyfile(self.path(name), target)
|
||
|
||
def bench(self, args):
|
||
if self.fail:
|
||
raise SystemExit(17)
|
||
n = board_bench.infer_inputs(Runtime(), "model", args[1], args[3])
|
||
if self.corrupt:
|
||
np.savez(args[3], **{"s0000/out0": np.ones(1)})
|
||
return {"samples": n, "outputs": Runtime.output_names["model"], "source": "board",
|
||
"hbm_sha256": "wrong" if self.mismatch else hashlib.sha256(self.hbm.read_bytes()).hexdigest(),
|
||
"board_type": "D-Robotics RDK S100P V1P0", "ucp": "3.13.6"}
|
||
|
||
def remove(self, *names):
|
||
for name in names:
|
||
Path(self.path(name)).unlink()
|
||
|
||
def close(self):
|
||
self.closed = True
|
||
self.local.cleanup()
|
||
|
||
|
||
def sample(value=1, dtype=np.float32):
|
||
return np.full((1, 3, 2, 2), value, dtype=dtype)
|
||
|
||
|
||
class BoardRuntimeTests(unittest.TestCase):
|
||
def setUp(self):
|
||
Session.instances = []
|
||
Session.fail = Session.corrupt = Session.mismatch = False
|
||
|
||
def test_bulk_copies_reused_buffers_in_declared_output_order(self):
|
||
with tempfile.TemporaryDirectory() as td:
|
||
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
|
||
np.savez(fin, s0001=sample(3), s0000=sample(1))
|
||
self.assertEqual(board_bench.infer_inputs(Runtime(), "model", str(fin), str(fout)), 2)
|
||
with np.load(fout) as outputs:
|
||
self.assertEqual(outputs["s0000/out0"].item(), 1)
|
||
self.assertEqual(outputs["s0000/out10"].item(), 11)
|
||
self.assertEqual(outputs["s0001/out0"].item(), 3)
|
||
|
||
def test_bulk_rejects_wrong_shape_dtype_and_multiple_inputs(self):
|
||
with tempfile.TemporaryDirectory() as td:
|
||
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
|
||
for x in (sample().reshape(1, 3, 1, 4), sample(dtype=np.uint8)):
|
||
np.savez(fin, s0000=x)
|
||
with self.assertRaises(ValueError):
|
||
board_bench.infer_inputs(Runtime(), "model", str(fin), str(fout))
|
||
self.assertFalse(fout.exists())
|
||
runtime = Runtime()
|
||
runtime.input_names = {"model": ["first", "second"]}
|
||
with self.assertRaisesRegex(ValueError, "one input"):
|
||
board_bench.infer_inputs(runtime, "model", str(fin), str(fout))
|
||
|
||
def test_bulk_runtime_type_accepts_f32_enum_but_rejects_other_dtypes(self):
|
||
with tempfile.TemporaryDirectory() as td:
|
||
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
|
||
np.savez(fin, s0000=sample())
|
||
runtime = Runtime()
|
||
runtime.input_types = {"model": {"data": "TensorType.F32"}}
|
||
self.assertEqual(board_bench.infer_inputs(runtime, "model", str(fin), str(fout)), 1)
|
||
for dtype in ("FLOAT16", "INT8", "UINT8"):
|
||
runtime.input_types = {"model": {"data": dtype}}
|
||
with self.assertRaisesRegex(ValueError, "FLOAT32"):
|
||
board_bench.infer_inputs(runtime, "model", str(fin), str(fout))
|
||
|
||
def test_runner_chunks_numeric_outputs_and_cleanup(self):
|
||
with tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", Session):
|
||
hbm = Path(td) / "model.hbm"
|
||
hbm.write_bytes(b"compiled-model")
|
||
with BoardRunner(str(hbm)) as runner:
|
||
runner.CHUNK = 2
|
||
outs = runner.batch([sample(v) for v in range(5)])
|
||
self.assertEqual([o[0].item() for o in outs], list(range(5)))
|
||
self.assertEqual(outs[0][10].item(), 10)
|
||
self.assertEqual(runner(sample(6))[0].item(), 6)
|
||
self.assertEqual(runner.metadata["source"], "board")
|
||
self.assertIn("S100P", runner.metadata["board_type"])
|
||
self.assertEqual(list(runner.tmp.iterdir()), [])
|
||
self.assertEqual(len(Session.instances), 1)
|
||
self.assertEqual(Session.instances[0].uploads, 4)
|
||
self.assertEqual(list(Session.instances[0].directory.iterdir()), [])
|
||
self.assertFalse(runner.tmp.exists())
|
||
self.assertTrue(Session.instances[0].closed)
|
||
|
||
def test_runner_failure_and_invalid_outputs_cleanup(self):
|
||
for mode, error in (("fail", SystemExit), ("corrupt", RuntimeError), ("mismatch", RuntimeError)):
|
||
with self.subTest(mode=mode), tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", Session):
|
||
hbm = Path(td) / "model.hbm"
|
||
hbm.write_bytes(b"compiled-model")
|
||
setattr(Session, mode, True)
|
||
runner = BoardRunner(str(hbm))
|
||
with self.assertRaises(error):
|
||
runner.batch([sample()])
|
||
self.assertFalse(runner.tmp.exists())
|
||
self.assertTrue(Session.instances[-1].closed)
|
||
setattr(Session, mode, False)
|
||
|
||
def test_runner_setup_failure_cleans_local_temp(self):
|
||
with tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", side_effect=SystemExit(4)):
|
||
hbm = Path(td) / "model.hbm"
|
||
hbm.write_bytes(b"compiled-model")
|
||
runner = BoardRunner(str(hbm))
|
||
with self.assertRaises(SystemExit):
|
||
runner.__enter__()
|
||
self.assertFalse(runner.tmp.exists())
|
||
|
||
def test_ssh_error_is_not_hidden_by_valid_json(self):
|
||
result = SimpleNamespace(returncode=17, stderr="remote inference failed", stdout='{"valid": true}\n')
|
||
with patch.object(board_perf.subprocess, "run", return_value=result):
|
||
with self.assertRaises(SystemExit) as raised:
|
||
board_perf.run(["ssh", "board", "python3 board_bench.py"])
|
||
self.assertEqual(raised.exception.code, 17)
|
||
|
||
def test_remote_session_uploads_once_and_quotes_shell_paths(self):
|
||
calls = []
|
||
receipt = {}
|
||
|
||
def fake_run(command, timeout=1800, **kwargs):
|
||
calls.append(command)
|
||
if "python3 " in command[-1]:
|
||
args = shlex.split(command[-1])
|
||
receipt.update(operation_id=args[args.index('--operation-id') + 1],
|
||
guard_version=4, cleanup_state='VERIFIED', all_children_reaped=True, proof='waitpid:ECHILD')
|
||
return 'runtime startup\n{"samples": 1}\n'
|
||
if command[-1].startswith('cat -- '):
|
||
return json.dumps(receipt)
|
||
return ""
|
||
|
||
with patch.object(board_perf, "ssh_base", return_value=(["-p", "2222"], "root@board")), \
|
||
patch.object(board_perf, "run", side_effect=fake_run), \
|
||
patch.object(board_perf, "guarded_run", side_effect=fake_run):
|
||
with board_perf.RemoteSession("/local/model.hbm", "/userdata/bench space") as session:
|
||
session.__enter__()
|
||
self.assertEqual(session.bench(["--runs", "1"])["samples"], 1)
|
||
scps = [c for c in calls if c[0] == "scp"]
|
||
self.assertEqual(len(scps), 6)
|
||
self.assertTrue(any("board_probe.py" in " ".join(cmd) for cmd in scps))
|
||
self.assertTrue(any("board_identity.py" in " ".join(cmd) for cmd in scps))
|
||
self.assertEqual(scps[0][1:3], ["-P", "2222"])
|
||
self.assertIn("'/userdata/bench space/", calls[0][-1])
|
||
self.assertTrue(calls[-1][-1].startswith("rm -rf -- "))
|
||
bench = next(c[-1] for c in calls if c[-1].startswith("python3 "))
|
||
self.assertNotIn("|", bench)
|
||
self.assertNotIn("2>/dev/null", bench)
|
||
self.assertIn('board_guard.py', bench)
|
||
|
||
def test_success_without_matching_cleanup_receipt_is_rejected(self):
|
||
for receipt in ('', '{"operation_id":"wrong","guard_version":3,"cleanup_state":"VERIFIED"}'):
|
||
with self.subTest(receipt=receipt), \
|
||
patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
|
||
patch.object(board_perf, 'run', return_value=receipt), \
|
||
patch.object(board_perf, 'guarded_run', return_value='{"samples":1}'):
|
||
with self.assertRaisesRegex(RuntimeError, 'BOARD_CLEANUP_UNCONFIRMED'):
|
||
with board_perf.RemoteSession('model.hbm') as session:
|
||
session.bench(['--runs', '1'])
|
||
|
||
def test_remote_setup_failure_keeps_exit_code_and_cleans_owned_directory(self):
|
||
calls = []
|
||
|
||
def fake_run(command, timeout=1800, **kwargs):
|
||
calls.append(command)
|
||
if command[0] == "scp":
|
||
raise SystemExit(23)
|
||
return ""
|
||
|
||
with patch.object(board_perf, "ssh_base", return_value=([], "root@board")), patch.object(board_perf, "run", side_effect=fake_run):
|
||
session = board_perf.RemoteSession("/local/model.hbm")
|
||
with self.assertRaises(SystemExit) as raised:
|
||
session.__enter__()
|
||
self.assertEqual(raised.exception.code, 23)
|
||
self.assertEqual(calls[-1][-1], "rm -rf -- " + session.directory)
|
||
self.assertFalse(session.active)
|
||
|
||
def test_setup_failure_has_not_started_proof_without_dispatch(self):
|
||
for failed_command in ('ssh', 'scp'):
|
||
with self.subTest(command=failed_command), tempfile.TemporaryDirectory() as td:
|
||
def fail_setup(command, **kwargs):
|
||
if command[0] == failed_command and not kwargs.get('cleanup'):
|
||
raise SystemExit(255)
|
||
return ''
|
||
with patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
|
||
patch.object(board_perf, 'run', side_effect=fail_setup), \
|
||
patch.object(board_perf, 'guarded_run') as dispatch, \
|
||
patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td}):
|
||
with self.assertRaises(SystemExit):
|
||
board_perf.RemoteSession('model.hbm').__enter__()
|
||
dispatch.assert_not_called()
|
||
receipts = list(Path(td).glob('*.json'))
|
||
self.assertEqual(len(receipts), 1)
|
||
receipt = json.loads(receipts[0].read_text())
|
||
self.assertEqual(receipt['operation_id'], receipts[0].stem)
|
||
self.assertEqual(receipt['guard_version'], 4)
|
||
self.assertEqual(receipt['cleanup_state'], 'VERIFIED')
|
||
self.assertTrue(receipt['all_children_reaped'])
|
||
self.assertEqual(receipt['proof'], 'not_started')
|
||
|
||
def test_setup_retries_only_known_transport_loss_and_is_bounded(self):
|
||
for diagnostic, successes, expected_calls in (
|
||
('kex_exchange_identification: Connection closed by remote host', True, 2),
|
||
('scp: lost connection', True, 2),
|
||
('Permission denied (publickey).', True, 1),
|
||
('', True, 1),
|
||
('Connection reset by peer', False, 2)):
|
||
with self.subTest(diagnostic=diagnostic, successes=successes), \
|
||
patch('harness.execution.from_environment', return_value=None), \
|
||
patch.object(board_perf.time, 'sleep'):
|
||
failed = SimpleNamespace(returncode=255, stdout='', stderr=diagnostic)
|
||
good = SimpleNamespace(returncode=0, stdout='ready', stderr='')
|
||
with patch.object(board_perf.subprocess, 'run', side_effect=[failed, good if successes else failed]) as invoke:
|
||
action = lambda: board_perf.run(['ssh', 'board', 'mkdir -p /userdata/owned'])
|
||
if successes and expected_calls == 2:
|
||
self.assertEqual(board_perf.RemoteSession.setup(action), 'ready')
|
||
else:
|
||
with self.assertRaises(SystemExit) as error:
|
||
board_perf.RemoteSession.setup(action)
|
||
self.assertEqual(error.exception.code, 255)
|
||
self.assertEqual(invoke.call_count, expected_calls)
|
||
|
||
def test_budget_rejection_preserves_prior_unknown_receipt(self):
|
||
owner = Mock()
|
||
owner.snapshot.return_value = {'deadline_monotonic': 59}
|
||
with tempfile.TemporaryDirectory() as td, \
|
||
patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
|
||
patch.object(board_perf, 'run', return_value=''), \
|
||
patch.object(board_perf, 'guarded_run', side_effect=SystemExit(255)) as dispatch, \
|
||
patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td}), \
|
||
patch('harness.execution.from_environment', return_value=None):
|
||
session = board_perf.RemoteSession('model.hbm')
|
||
with self.assertRaises(SystemExit):
|
||
session.bench([])
|
||
original = list(Path(td).glob('*.json'))[0]
|
||
original_bytes = original.read_bytes()
|
||
self.assertEqual(json.loads(original_bytes)['cleanup_state'], 'UNKNOWN')
|
||
with patch('harness.execution.from_environment', return_value=owner), \
|
||
patch.object(board_perf.time, 'monotonic', return_value=0):
|
||
with self.assertRaisesRegex(RuntimeError, 'CLEANUP_BUDGET'):
|
||
session.bench([])
|
||
dispatch.assert_called_once()
|
||
self.assertEqual(original.read_bytes(), original_bytes)
|
||
receipts = [json.loads(p.read_text()) for p in Path(td).glob('*.json')]
|
||
self.assertEqual(len(receipts), 2)
|
||
self.assertEqual({r['cleanup_state'] for r in receipts}, {'UNKNOWN', 'VERIFIED'})
|
||
self.assertTrue(session.failed_bench)
|
||
|
||
def test_dispatched_failure_requires_remote_proof_even_without_output(self):
|
||
for failure in (SystemExit(255), board_perf.subprocess.TimeoutExpired('ssh', 1), None):
|
||
with self.subTest(failure=failure), tempfile.TemporaryDirectory() as td, \
|
||
patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
|
||
patch.object(board_perf, 'run', return_value=''), \
|
||
patch.object(board_perf, 'guarded_run', side_effect=failure, return_value='') as dispatch, \
|
||
patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td}):
|
||
session = board_perf.RemoteSession('model.hbm')
|
||
with self.assertRaises((SystemExit, RuntimeError, board_perf.subprocess.TimeoutExpired)):
|
||
session.bench([])
|
||
dispatch.assert_called_once()
|
||
receipt = json.loads(next(Path(td).glob('*.json')).read_text())
|
||
self.assertEqual(receipt['cleanup_state'], 'UNKNOWN')
|
||
self.assertNotEqual(receipt.get('proof'), 'not_started')
|
||
self.assertTrue(session.failed_bench)
|
||
|
||
def test_terminal_setup_failure_has_not_started_proof(self):
|
||
from harness import terminal_board
|
||
for step in ('__enter__', 'upload'):
|
||
with self.subTest(step=step), tempfile.TemporaryDirectory() as td:
|
||
hd = Path(td)
|
||
(hd / 'snapshot').mkdir(); (hd / 'input').mkdir()
|
||
current = Mock()
|
||
session = Mock()
|
||
session.__enter__ = Mock()
|
||
session.setup.side_effect = lambda action, *args: action(*args)
|
||
getattr(session, step).side_effect = SystemExit(255)
|
||
ctx = SimpleNamespace(extra={'board': {}})
|
||
# A configured board is required before capability/setup are attempted.
|
||
ctx.extra['board']['runtime_dir'] = '/userdata/fixture'
|
||
with patch.object(terminal_board, 'credentials', return_value=nullcontext()), \
|
||
patch.object(terminal_board.lifecycle, 'current', return_value=current), \
|
||
patch.object(terminal_board, 'TerminalSession', return_value=session), \
|
||
patch.object(terminal_board, 'guarded_run') as dispatch:
|
||
with self.assertRaises(SystemExit):
|
||
terminal_board.execute(ctx, hd, {}, 100)
|
||
dispatch.assert_not_called()
|
||
current.quarantine.assert_not_called()
|
||
session.close.assert_called_once()
|
||
paths = list((hd / 'board-receipts').glob('*.json'))
|
||
self.assertEqual(len(paths), 1)
|
||
receipt = json.loads(paths[0].read_text())
|
||
self.assertEqual(receipt['operation_id'], paths[0].stem)
|
||
self.assertEqual(receipt['cleanup_state'], 'VERIFIED')
|
||
self.assertEqual(receipt['proof'], 'not_started')
|
||
|
||
def test_busy_wait_retries_only_guard_rejection_without_shell_probe(self):
|
||
session = SimpleNamespace(bench=Mock(side_effect=[SystemExit(75), {"latency_ms_mean": 1.0}]))
|
||
with tempfile.TemporaryDirectory() as td:
|
||
out = Path(td) / "perf.json"
|
||
argv = ["board_perf", "--hbm", "model.hbm", "--job-id", "job", "--out", str(out)]
|
||
with patch.object(board_perf.sys, "argv", argv), \
|
||
patch.object(board_perf, "run") as run, \
|
||
patch.object(board_perf, "RemoteSession", return_value=nullcontext(session)), \
|
||
patch.object(board_perf.time, "sleep") as sleep:
|
||
self.assertEqual(board_perf.main(), 0)
|
||
run.assert_not_called()
|
||
sleep.assert_called_once_with(20)
|
||
self.assertEqual(session.bench.call_count, 2)
|
||
result = json.loads(out.read_text())
|
||
self.assertEqual(result["busy_waited_s"], 20)
|
||
self.assertFalse(result["contended"])
|
||
|
||
def test_busy_or_unknown_does_not_produce_performance_result(self):
|
||
for status in (75, 78):
|
||
with self.subTest(status=status), tempfile.TemporaryDirectory() as td:
|
||
out = Path(td) / 'perf.json'
|
||
argv = ['board_perf', '--hbm', 'm.hbm', '--job-id', 'j', '--out', str(out)]
|
||
if status == 75:
|
||
argv += ['--busy-wait-s', '0']
|
||
session = SimpleNamespace(bench=Mock(side_effect=SystemExit(status)))
|
||
with patch.object(board_perf.sys, 'argv', argv), \
|
||
patch.object(board_perf, 'RemoteSession', return_value=nullcontext(session)), \
|
||
patch.object(board_perf.time, 'sleep') as sleep:
|
||
with self.assertRaises(SystemExit) as error:
|
||
board_perf.main()
|
||
self.assertEqual(error.exception.code, status)
|
||
session.bench.assert_called_once()
|
||
sleep.assert_not_called()
|
||
self.assertFalse(out.exists())
|
||
|
||
def test_eval_runner_uploads_same_probe_with_real_remote_session(self):
|
||
with tempfile.TemporaryDirectory() as td:
|
||
hbm = Path(td) / 'model.hbm'
|
||
hbm.write_bytes(b'fixture')
|
||
with patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
|
||
patch.object(board_perf, 'run', return_value='') as remote:
|
||
with BoardRunner(str(hbm)):
|
||
pass
|
||
uploads = [call.args[0] for call in remote.call_args_list if call.args[0][0] == 'scp']
|
||
self.assertTrue(any('board_probe.py' in ' '.join(command) for command in uploads))
|
||
self.assertTrue(any('board_guard.py' in ' '.join(command) for command in uploads))
|
||
|
||
def test_failed_guard_retains_evidence_without_erasing_live_directory(self):
|
||
with patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
|
||
patch.object(board_perf, 'run', return_value='') as run, \
|
||
patch.object(board_perf, 'guarded_run', side_effect=SystemExit(124)):
|
||
with self.assertRaises(SystemExit):
|
||
with board_perf.RemoteSession('model.hbm') as session:
|
||
session.bench(['--runs', '1'])
|
||
self.assertFalse(any(c.args[0][-1].startswith('rm ') for c in run.call_args_list))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|