data-gen-bpu/scripts/test_board_runtime.py
bench f56ac2f3de 板上评测按实测字节分批:分割/深度/光流/特征不再逐张上板,大输出模型不再一批 12 GB
- BoardRunner:第一次只送 1 张,量出每张输入+输出字节,之后每批送到 1 GiB 为止(最多 32 张)。
  10-15 solo.r50 一批 32 张 12 GB,写满了一块 S600。
- common.groups:评测器按同一尺寸分组调 run_batched,一组的输出也放得进内存。
  分割、深度、光流、特征原来逐张上板,每张约 10 秒连接开销,100 张约 20 分钟。
  实例分割/姿态、G2 余弦也按它分组;非板卡 runner 保持原来的组大小。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 11:08:04 +08:00

619 lines
33 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""板端推理协议回归测试:不访问 SSH,不占用真实板卡。"""
import hashlib
import json
import os
import shutil
import shlex
import tempfile
import time
import unittest
from pathlib import Path
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock, patch
import numpy as np
from harness.eval.board import BoardRunner
from harness.tools import board_bench, board_perf
class Runtime:
model_names = ["model"]
input_names = {"model": ["data"]}
input_shapes = {"model": {"data": [1, 3, 2, 2]}}
input_types = {"model": {"data": "FLOAT32"}}
output_names = {"model": [f"output{i}" for i in range(12)]}
def __init__(self):
self.buffers = {name: np.empty(1, np.float32) for name in self.output_names["model"]}
def run(self, feed):
for i, name in enumerate(self.output_names["model"]):
self.buffers[name][:] = feed["model"]["data"].mean() + i
# 返回字典顺序与模型输出名顺序相反,同时重用缓冲区。
return {"model": dict(reversed(list(self.buffers.items())))}
class Session:
instances = []
fail = False
corrupt = False
mismatch = False
def __init__(self, hbm, runtime_dir):
self.hbm = Path(hbm)
self.local = tempfile.TemporaryDirectory()
self.directory = Path(self.local.name)
self.closed = False
self.uploads = 0
self.instances.append(self)
def __enter__(self):
return self
def path(self, name):
return str(self.directory / name)
def upload(self, source, name):
shutil.copyfile(source, self.path(name))
self.uploads += 1
def download(self, name, target):
shutil.copyfile(self.path(name), target)
def bench(self, args):
if self.fail:
raise SystemExit(17)
n = board_bench.infer_inputs(Runtime(), "model", args[1], args[3])
if self.corrupt:
np.savez(args[3], **{"s0000/out0": np.ones(1)})
return {"samples": n, "outputs": Runtime.output_names["model"], "source": "board",
"hbm_sha256": "wrong" if self.mismatch else hashlib.sha256(self.hbm.read_bytes()).hexdigest(),
"board_type": "D-Robotics RDK S100P V1P0", "ucp": "3.13.6"}
def remove(self, *names):
for name in names:
Path(self.path(name)).unlink()
def close(self):
self.closed = True
self.local.cleanup()
def sample(value=1, dtype=np.float32):
return np.full((1, 3, 2, 2), value, dtype=dtype)
class BoardRuntimeTests(unittest.TestCase):
def setUp(self):
Session.instances = []
Session.fail = Session.corrupt = Session.mismatch = False
def test_bulk_copies_reused_buffers_in_declared_output_order(self):
with tempfile.TemporaryDirectory() as td:
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
np.savez(fin, s0001=sample(3), s0000=sample(1))
self.assertEqual(board_bench.infer_inputs(Runtime(), "model", str(fin), str(fout)), 2)
with np.load(fout) as outputs:
self.assertEqual(outputs["s0000/out0"].item(), 1)
self.assertEqual(outputs["s0000/out10"].item(), 11)
self.assertEqual(outputs["s0001/out0"].item(), 3)
def test_bulk_rejects_wrong_shape_dtype_and_multiple_inputs(self):
with tempfile.TemporaryDirectory() as td:
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
for x in (sample().reshape(1, 3, 1, 4), sample(dtype=np.uint8)):
np.savez(fin, s0000=x)
with self.assertRaises(ValueError):
board_bench.infer_inputs(Runtime(), "model", str(fin), str(fout))
self.assertFalse(fout.exists())
runtime = Runtime()
runtime.input_names = {"model": ["first", "second"]}
with self.assertRaisesRegex(ValueError, "one input"):
board_bench.infer_inputs(runtime, "model", str(fin), str(fout))
def test_bulk_runtime_type_accepts_f32_enum_but_rejects_other_dtypes(self):
with tempfile.TemporaryDirectory() as td:
fin, fout = Path(td) / "in.npz", Path(td) / "out.npz"
np.savez(fin, s0000=sample())
runtime = Runtime()
runtime.input_types = {"model": {"data": "TensorType.F32"}}
self.assertEqual(board_bench.infer_inputs(runtime, "model", str(fin), str(fout)), 1)
for dtype in ("FLOAT16", "INT8", "UINT8"):
runtime.input_types = {"model": {"data": dtype}}
with self.assertRaisesRegex(ValueError, "FLOAT32"):
board_bench.infer_inputs(runtime, "model", str(fin), str(fout))
def test_runner_chunks_numeric_outputs_and_cleanup(self):
with tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", Session):
hbm = Path(td) / "model.hbm"
hbm.write_bytes(b"compiled-model")
with BoardRunner(str(hbm)) as runner:
runner.CHUNK = 2
outs = runner.batch([sample(v) for v in range(5)])
self.assertEqual([o[0].item() for o in outs], list(range(5)))
self.assertEqual(outs[0][10].item(), 10)
self.assertEqual(runner(sample(6))[0].item(), 6)
self.assertEqual(runner.metadata["source"], "board")
self.assertIn("S100P", runner.metadata["board_type"])
self.assertEqual(list(runner.tmp.iterdir()), [])
self.assertEqual(len(Session.instances), 1)
self.assertEqual(Session.instances[0].uploads, 4)
self.assertEqual(list(Session.instances[0].directory.iterdir()), [])
self.assertFalse(runner.tmp.exists())
self.assertTrue(Session.instances[0].closed)
def test_runner_sizes_board_calls_and_evaluator_groups_by_measured_bytes(self):
# 10-15: one 32-sample solo.r50 call was 12 GB; segmentation went one image per call.
from harness.eval.common import groups
with tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", Session):
hbm = Path(td) / "model.hbm"
hbm.write_bytes(b"compiled-model")
with BoardRunner(str(hbm)) as runner:
self.assertEqual(runner.group_size(), 1) # unmeasured: one sample first
runner.batch([sample(0)])
runner.MAX_BYTES = 3 * runner._sample_bytes
self.assertEqual(runner.group_size(), 3)
self.assertEqual([len(g) for g in groups(runner, list(range(7)))], [3, 3, 1])
outs = runner.batch([sample(v) for v in range(7)])
self.assertEqual([o[0].item() for o in outs], list(range(7)))
self.assertEqual(Session.instances[0].uploads, 1 + 3)
self.assertEqual([len(g) for g in groups(object(), list(range(7)), 4)], [4, 3])
def test_runner_failure_and_invalid_outputs_cleanup(self):
for mode, error in (("fail", SystemExit), ("corrupt", RuntimeError), ("mismatch", RuntimeError)):
with self.subTest(mode=mode), tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", Session):
hbm = Path(td) / "model.hbm"
hbm.write_bytes(b"compiled-model")
setattr(Session, mode, True)
runner = BoardRunner(str(hbm))
with self.assertRaises(error):
runner.batch([sample()])
self.assertFalse(runner.tmp.exists())
self.assertTrue(Session.instances[-1].closed)
setattr(Session, mode, False)
def test_runner_setup_failure_cleans_local_temp(self):
with tempfile.TemporaryDirectory() as td, patch("harness.eval.board.RemoteSession", side_effect=SystemExit(4)):
hbm = Path(td) / "model.hbm"
hbm.write_bytes(b"compiled-model")
runner = BoardRunner(str(hbm))
with self.assertRaises(SystemExit):
runner.__enter__()
self.assertFalse(runner.tmp.exists())
def test_ssh_error_is_not_hidden_by_valid_json(self):
result = SimpleNamespace(returncode=17, stderr="remote inference failed", stdout='{"valid": true}\n')
with patch.object(board_perf.subprocess, "run", return_value=result):
with self.assertRaises(SystemExit) as raised:
board_perf.run(["ssh", "board", "python3 board_bench.py"])
self.assertEqual(raised.exception.code, 17)
def test_remote_session_uploads_once_and_quotes_shell_paths(self):
calls = []
receipt = {}
def fake_run(command, timeout=1800, **kwargs):
calls.append(command)
if "python3 " in command[-1]:
args = shlex.split(command[-1])
receipt.update(operation_id=args[args.index('--operation-id') + 1],
guard_version=4, cleanup_state='VERIFIED', all_children_reaped=True, proof='waitpid:ECHILD')
return 'runtime startup\n{"samples": 1}\n'
if command[-1].startswith('cat -- '):
return json.dumps(receipt)
return ""
with patch.object(board_perf, "ssh_base", return_value=(["-p", "2222"], "root@board")), \
patch.object(board_perf, "run", side_effect=fake_run), \
patch.object(board_perf, "guarded_run", side_effect=fake_run):
with board_perf.RemoteSession("/local/model.hbm", "/userdata/bench space") as session:
session.__enter__()
self.assertEqual(session.bench(["--runs", "1"])["samples"], 1)
scps = [c for c in calls if c[0] == "scp"]
self.assertEqual(len(scps), 6)
self.assertTrue(any("board_probe.py" in " ".join(cmd) for cmd in scps))
self.assertTrue(any("board_identity.py" in " ".join(cmd) for cmd in scps))
self.assertEqual(scps[0][1:3], ["-P", "2222"])
self.assertTrue(calls[0][-1].startswith("find '/userdata/bench space' -maxdepth 1 -name 'eval-*' -mmin +360 -print "))
self.assertIn("'/userdata/bench space/", calls[1][-1])
self.assertTrue(calls[-1][-1].startswith("rm -rf -- "))
bench = next(c[-1] for c in calls if c[-1].startswith("python3 "))
self.assertNotIn("|", bench)
self.assertNotIn("2>/dev/null", bench)
self.assertIn('board_guard.py', bench)
def test_success_without_matching_cleanup_receipt_is_rejected(self):
for receipt in ('', '{"operation_id":"wrong","guard_version":3,"cleanup_state":"VERIFIED"}'):
with self.subTest(receipt=receipt), \
patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
patch.object(board_perf, 'run', return_value=receipt), \
patch.object(board_perf, 'guarded_run', return_value='{"samples":1}'):
with self.assertRaisesRegex(RuntimeError, 'BOARD_CLEANUP_UNCONFIRMED'):
with board_perf.RemoteSession('model.hbm') as session:
session.bench(['--runs', '1'])
def test_remote_setup_failure_keeps_exit_code_and_cleans_owned_directory(self):
calls = []
def fake_run(command, timeout=1800, **kwargs):
calls.append(command)
if command[0] == "scp":
raise SystemExit(23)
return ""
with patch.object(board_perf, "ssh_base", return_value=([], "root@board")), patch.object(board_perf, "run", side_effect=fake_run):
session = board_perf.RemoteSession("/local/model.hbm")
with self.assertRaises(SystemExit) as raised:
session.__enter__()
self.assertEqual(raised.exception.code, 23)
self.assertEqual(calls[-1][-1], "rm -rf -- " + session.directory)
self.assertFalse(session.active)
def test_setup_failure_has_not_started_proof_without_dispatch(self):
for failed_command in ('ssh', 'scp'):
with self.subTest(command=failed_command), tempfile.TemporaryDirectory() as td:
def fail_setup(command, **kwargs):
if command[0] == failed_command and not kwargs.get('cleanup'):
raise SystemExit(255)
return ''
with patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
patch.object(board_perf, 'run', side_effect=fail_setup), \
patch.object(board_perf, 'guarded_run') as dispatch, \
patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td}):
with self.assertRaises(SystemExit):
board_perf.RemoteSession('model.hbm').__enter__()
dispatch.assert_not_called()
receipts = list(Path(td).glob('*.json'))
self.assertEqual(len(receipts), 1)
receipt = json.loads(receipts[0].read_text())
self.assertEqual(receipt['operation_id'], receipts[0].stem)
self.assertEqual(receipt['guard_version'], 4)
self.assertEqual(receipt['cleanup_state'], 'VERIFIED')
self.assertTrue(receipt['all_children_reaped'])
self.assertEqual(receipt['proof'], 'not_started')
def test_setup_retries_only_known_transport_loss_and_is_bounded(self):
for diagnostic, successes, expected_calls in (
('kex_exchange_identification: Connection closed by remote host', True, 2),
('scp: lost connection', True, 2),
('Permission denied (publickey).', True, 1),
('', True, 1),
('Connection reset by peer', False, 2)):
with self.subTest(diagnostic=diagnostic, successes=successes), \
patch('harness.execution.from_environment', return_value=None), \
patch.object(board_perf.time, 'sleep'):
failed = SimpleNamespace(returncode=255, stdout='', stderr=diagnostic)
good = SimpleNamespace(returncode=0, stdout='ready', stderr='')
with patch.object(board_perf.subprocess, 'run', side_effect=[failed, good if successes else failed]) as invoke:
action = lambda: board_perf.run(['ssh', 'board', 'mkdir -p /userdata/owned'])
if successes and expected_calls == 2:
self.assertEqual(board_perf.RemoteSession.setup(action), 'ready')
else:
with self.assertRaises(SystemExit) as error:
board_perf.RemoteSession.setup(action)
self.assertEqual(error.exception.code, 255)
self.assertEqual(invoke.call_count, expected_calls)
def test_container_caps_board_call_by_the_controller_phase_deadline(self):
# A Salty container has no execution state; without this a board hang outlived the phase (10-12/14).
seen = []
def dispatch(cmd, timeout):
seen.append(timeout)
raise SystemExit(255)
with tempfile.TemporaryDirectory() as td, \
patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
patch.object(board_perf, 'run', return_value=''), \
patch.object(board_perf, 'guarded_run', side_effect=dispatch), \
patch('harness.execution.from_environment', return_value=None):
with patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td, 'BENCH_PHASE_DEADLINE': str(time.time() + 700)}):
with self.assertRaises(SystemExit):
board_perf.RemoteSession('model.hbm').bench([])
self.assertAlmostEqual(seen[0], 100, delta=5) # 700 s left - 540 s hand-back - 60 s cleanup
with patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td, 'BENCH_PHASE_DEADLINE': str(time.time() + 500)}):
with self.assertRaisesRegex(RuntimeError, 'BOARD_CLEANUP_BUDGET_INSUFFICIENT'):
board_perf.RemoteSession('model.hbm').bench([])
self.assertEqual(len(seen), 1) # never dispatched without the budget
def test_budget_rejection_preserves_prior_unknown_receipt(self):
owner = Mock()
owner.snapshot.return_value = {'deadline_monotonic': 59}
with tempfile.TemporaryDirectory() as td, \
patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
patch.object(board_perf, 'run', return_value=''), \
patch.object(board_perf, 'guarded_run', side_effect=SystemExit(255)) as dispatch, \
patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td}), \
patch('harness.execution.from_environment', return_value=None):
session = board_perf.RemoteSession('model.hbm')
with self.assertRaises(SystemExit):
session.bench([])
original = list(Path(td).glob('*.json'))[0]
original_bytes = original.read_bytes()
self.assertEqual(json.loads(original_bytes)['cleanup_state'], 'UNKNOWN')
with patch('harness.execution.from_environment', return_value=owner), \
patch.object(board_perf.time, 'monotonic', return_value=0):
with self.assertRaisesRegex(RuntimeError, 'CLEANUP_BUDGET'):
session.bench([])
dispatch.assert_called_once()
self.assertEqual(original.read_bytes(), original_bytes)
receipts = [json.loads(p.read_text()) for p in Path(td).glob('*.json')]
self.assertEqual(len(receipts), 2)
self.assertEqual({r['cleanup_state'] for r in receipts}, {'UNKNOWN', 'VERIFIED'})
self.assertTrue(session.cleanup_unconfirmed)
def test_dispatched_failure_requires_remote_proof_even_without_output(self):
for failure in (SystemExit(255), board_perf.subprocess.TimeoutExpired('ssh', 1), None):
with self.subTest(failure=failure), tempfile.TemporaryDirectory() as td, \
patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
patch.object(board_perf, 'run', return_value=''), \
patch.object(board_perf, 'guarded_run', side_effect=failure, return_value='') as dispatch, \
patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td}):
session = board_perf.RemoteSession('model.hbm')
with self.assertRaises((SystemExit, RuntimeError, board_perf.subprocess.TimeoutExpired)):
session.bench([])
dispatch.assert_called_once()
receipt = json.loads(next(Path(td).glob('*.json')).read_text())
self.assertEqual(receipt['cleanup_state'], 'UNKNOWN')
self.assertNotEqual(receipt.get('proof'), 'not_started')
self.assertTrue(session.cleanup_unconfirmed)
def test_terminal_setup_failure_has_not_started_proof(self):
from harness import terminal_board
for step in ('__enter__', 'upload'):
with self.subTest(step=step), tempfile.TemporaryDirectory() as td:
hd = Path(td)
(hd / 'snapshot').mkdir(); (hd / 'input').mkdir()
current = Mock()
session = Mock()
session.__enter__ = Mock()
session.setup.side_effect = lambda action, *args: action(*args)
getattr(session, step).side_effect = SystemExit(255)
ctx = SimpleNamespace(extra={'board': {}})
# A configured board is required before capability/setup are attempted.
ctx.extra['board']['runtime_dir'] = '/userdata/fixture'
with patch.object(terminal_board, 'credentials', return_value=nullcontext()), \
patch.object(terminal_board.lifecycle, 'current', return_value=current), \
patch.object(terminal_board, 'TerminalSession', return_value=session), \
patch.object(terminal_board, 'guarded_run') as dispatch:
with self.assertRaises(SystemExit):
terminal_board.execute(ctx, hd, {}, 100)
dispatch.assert_not_called()
current.quarantine.assert_not_called()
session.close.assert_called_once()
paths = list((hd / 'board-receipts').glob('*.json'))
self.assertEqual(len(paths), 1)
receipt = json.loads(paths[0].read_text())
self.assertEqual(receipt['operation_id'], paths[0].stem)
self.assertEqual(receipt['cleanup_state'], 'VERIFIED')
self.assertEqual(receipt['proof'], 'not_started')
def test_busy_wait_retries_only_guard_rejection_without_shell_probe(self):
session = SimpleNamespace(bench=Mock(side_effect=[SystemExit(75), {"latency_ms_mean": 1.0}]))
with tempfile.TemporaryDirectory() as td:
out = Path(td) / "perf.json"
argv = ["board_perf", "--hbm", "model.hbm", "--job-id", "job", "--out", str(out)]
with patch.object(board_perf.sys, "argv", argv), \
patch.object(board_perf, "run") as run, \
patch.object(board_perf, "RemoteSession", return_value=nullcontext(session)), \
patch.object(board_perf.time, "sleep") as sleep:
self.assertEqual(board_perf.main(), 0)
run.assert_not_called()
sleep.assert_called_once_with(20)
self.assertEqual(session.bench.call_count, 2)
result = json.loads(out.read_text())
self.assertEqual(result["busy_waited_s"], 20)
self.assertFalse(result["contended"])
def test_busy_or_unknown_does_not_produce_performance_result(self):
for status in (75, 78):
with self.subTest(status=status), tempfile.TemporaryDirectory() as td:
out = Path(td) / 'perf.json'
argv = ['board_perf', '--hbm', 'm.hbm', '--job-id', 'j', '--out', str(out)]
if status == 75:
argv += ['--busy-wait-s', '0']
session = SimpleNamespace(bench=Mock(side_effect=SystemExit(status)))
with patch.object(board_perf.sys, 'argv', argv), \
patch.object(board_perf, 'RemoteSession', return_value=nullcontext(session)), \
patch.object(board_perf.time, 'sleep') as sleep:
with self.assertRaises(SystemExit) as error:
board_perf.main()
self.assertEqual(error.exception.code, status)
session.bench.assert_called_once()
sleep.assert_not_called()
self.assertFalse(out.exists())
def test_eval_runner_uploads_same_probe_with_real_remote_session(self):
with tempfile.TemporaryDirectory() as td:
hbm = Path(td) / 'model.hbm'
hbm.write_bytes(b'fixture')
with patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
patch.object(board_perf, 'run', return_value='') as remote:
with BoardRunner(str(hbm)):
pass
uploads = [call.args[0] for call in remote.call_args_list if call.args[0][0] == 'scp']
self.assertTrue(any('board_probe.py' in ' '.join(command) for command in uploads))
self.assertTrue(any('board_guard.py' in ' '.join(command) for command in uploads))
def test_failed_guard_retains_evidence_without_erasing_live_directory(self):
with patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
patch.object(board_perf, 'run', return_value='') as run, \
patch.object(board_perf, 'guarded_run', side_effect=SystemExit(124)):
with self.assertRaises(SystemExit):
with board_perf.RemoteSession('model.hbm') as session:
session.bench(['--runs', '1'])
self.assertFalse(any(c.args[0][-1].startswith('rm ') for c in run.call_args_list))
# The directory and receipt stay for the possibly live guard; the bulky files go.
emptied = [c.args[0][-1] for c in run.call_args_list if c.args[0][-1].startswith(f'find {session.directory} ')]
self.assertEqual(len(emptied), 1)
self.assertIn("'!' -name '*guard-result.json'", emptied[0])
def test_failed_inference_with_confirmed_cleanup_removes_directory(self):
# 10-15: such sessions were kept too and filled the S600s' /userdata (one solo.r50 chunk = 12 GB).
calls, receipt = [], {}
def fake_run(command, timeout=1800, **kwargs):
calls.append(command)
return json.dumps(receipt) if command[-1].startswith('cat -- ') else ''
def failing_inference(command, timeout):
args = shlex.split(command[-1])
receipt.update(operation_id=args[args.index('--operation-id') + 1], guard_version=4,
cleanup_state='VERIFIED', all_children_reaped=True, proof='waitpid:ECHILD')
raise SystemExit(1)
with patch.object(board_perf, 'ssh_base', return_value=([], 'root@board')), \
patch.object(board_perf, 'run', side_effect=fake_run), \
patch.object(board_perf, 'guarded_run', side_effect=failing_inference):
with self.assertRaises(SystemExit):
with board_perf.RemoteSession('model.hbm') as session:
session.bench(['--runs', '1'])
self.assertEqual(calls[-1][-1], f'rm -rf -- {session.directory}')
class PredispatchProofTests(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.root = Path(temp.name)
self.rd = self.root / 'rollout'
self.receipts = self.rd / 'stages/_queue/op/board-receipts'
self.receipts.mkdir(parents=True)
self.env = patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': str(self.receipts)})
self.env.start(); self.addCleanup(self.env.stop)
def rows(self):
return [json.loads(p.read_text()) for p in self.receipts.glob('*.json')]
def resource_state(self):
# Exercise the existing release gate with a settled, empty local owner.
from harness import execution, paths, scheduler
locks = self.root / 'state/locks'
locks.mkdir(parents=True, exist_ok=True)
identity = {'execution_id': 'fixture', 'epoch': 1}
execution.atomic_json(self.rd / 'execution.json', dict(identity,
execution_state='TERMINAL', cleanup_state='VERIFIED', processes=[]))
execution.atomic_json(locks / 'board-0.lease.json', dict(identity,
pool='board', slot=0, holder={}, operation_id='op', execution_dir=str(self.rd)))
with patch.object(paths, 'STATE', locks.parent), \
patch.object(execution, 'process_status', return_value='DEAD'), \
patch.object(scheduler, '_board_idle_ps', return_value=None):
return scheduler.reconcile_leases()[0]['state']
def eval_failure(self, case):
from harness.eval import run as evaluate
baseline = self.root / 'baseline'
baseline.mkdir(exist_ok=True)
metrics = {'t1_manifest_sha': 'fixture', 'evaluator_version': evaluate.EVALUATOR_VERSION}
if case == 'baseline_mismatch':
metrics['t1_manifest_sha'] = 'different'
if case != 'missing_baseline':
(baseline / 'metrics.json').write_text(json.dumps(metrics))
argv = ['eval', '--mode', 'quant', '--runner', 'board', '--task', 'cls',
'--model', 'fixture', '--input-spec', '1x3x2x2', '--t1', 'missing.tar',
'--hbm', str(self.root / 'missing.hbm'), '--baseline', str(baseline),
'--out', str(self.root / 'eval.json')]
if case == 'version':
argv += ['--evaluator-version', 'invalid']
t1 = SimpleNamespace(manifest={'sha256': 'fixture'}, items=[])
with patch.object(evaluate.sys, 'argv', argv), \
patch.object(evaluate.T1, 'open', side_effect=FileNotFoundError('missing T1')
if case == 'missing_t1' else None, return_value=t1), \
patch.object(board_perf, 'run') as transport, \
patch.object(board_perf, 'guarded_run') as dispatch:
if case in ('version', 'baseline_mismatch'):
self.assertEqual(evaluate.main(), 66 if case == 'version' else 67)
else:
with self.assertRaises(FileNotFoundError):
evaluate.main()
transport.assert_not_called(); dispatch.assert_not_called()
self.assertFalse((self.root / 'eval.json').exists())
def test_local_eval_failures_keep_failure_and_release_without_ssh(self):
for case in ('missing_hbm', 'missing_baseline', 'missing_t1', 'baseline_mismatch', 'version'):
with self.subTest(case=case):
for path in self.receipts.glob('*.json'):
path.unlink()
(self.root / 'baseline/metrics.json').unlink(missing_ok=True)
self.eval_failure(case)
self.assertEqual(len(self.rows()), 1)
receipt = self.rows()[0]
self.assertEqual((receipt['cleanup_state'], receipt['proof']), ('VERIFIED', 'not_started'))
self.assertTrue(receipt['all_children_reaped'])
self.assertEqual(self.resource_state(), 'RELEASED')
def test_perf_configuration_failure_has_proof_without_ssh(self):
argv = ['perf', '--hbm', 'missing.hbm', '--job-id', 'fixture', '--out', str(self.root / 'perf.json')]
with patch.object(board_perf.sys, 'argv', argv), \
patch.object(board_perf, 'ssh_base', side_effect=SystemExit(4)), \
patch.object(board_perf, 'run') as transport, \
patch.object(board_perf, 'guarded_run') as dispatch:
with self.assertRaises(SystemExit) as error:
board_perf.main()
self.assertEqual(error.exception.code, 4)
transport.assert_not_called(); dispatch.assert_not_called()
self.assertEqual(self.resource_state(), 'RELEASED')
def test_prior_unknown_survives_new_local_validation_failure(self):
prior = self.receipts / 'prior.json'
prior.write_text('{"operation_id":"prior","cleanup_state":"UNKNOWN"}\n')
before = prior.read_bytes()
self.eval_failure('missing_hbm')
self.assertEqual(prior.read_bytes(), before)
self.assertEqual(len(self.rows()), 2)
self.assertEqual(self.resource_state(), 'QUARANTINED')
def test_process_exit_during_dispatch_keeps_unknown_before_finally(self):
import subprocess, sys
code = """import os
from unittest.mock import patch
from harness.tools import board_perf
with patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), patch.object(board_perf, 'guarded_run', side_effect=lambda *a: os._exit(77)):
session = board_perf.RemoteSession('fixture')
session.not_started('fixture')
session.bench([])
"""
result = subprocess.run([sys.executable, '-c', code], capture_output=True, text=True, timeout=10)
self.assertEqual(result.returncode, 77, result.stderr)
self.assertEqual({r['cleanup_state'] for r in self.rows()}, {'VERIFIED', 'UNKNOWN'})
self.assertEqual(self.resource_state(), 'QUARANTINED')
def test_dispatch_receipt_must_persist_before_transport(self):
from harness import execution
def fail_write(path, value):
self.assertEqual(value['cleanup_state'], 'UNKNOWN')
raise OSError('fsync failed')
with patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), \
patch.object(board_perf, 'guarded_run') as dispatch, \
patch.object(board_perf, 'run') as transport:
session = board_perf.RemoteSession('fixture')
session.not_started('fixture')
with patch.object(execution, 'atomic_json', side_effect=fail_write):
with self.assertRaisesRegex(OSError, 'fsync failed'):
session.bench([])
dispatch.assert_not_called(); transport.assert_not_called()
def test_real_receipt_replaces_same_pending_operation(self):
actual = {}
def dispatch(command, timeout):
args = shlex.split(command[-1]); op = args[args.index('--operation-id') + 1]
pending = json.loads((self.receipts / (op + '.json')).read_text())
self.assertEqual(pending['cleanup_state'], 'UNKNOWN')
self.assertEqual(pending['operation_id'], op)
actual.update(operation_id=op, guard_version=4, cleanup_state='VERIFIED',
all_children_reaped=True, proof='waitpid:ECHILD')
return '{"samples":1}'
with patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), \
patch.object(board_perf, 'guarded_run', side_effect=dispatch), \
patch.object(board_perf.RemoteSession, 'execute', side_effect=lambda *a, **kw: json.dumps(actual)):
session = board_perf.RemoteSession('fixture')
session.not_started('fixture')
self.assertEqual(session.bench([])['samples'], 1)
self.assertEqual(len(self.rows()), 2)
self.assertEqual({r['cleanup_state'] for r in self.rows()}, {'VERIFIED'})
self.assertEqual(self.resource_state(), 'RELEASED')
if __name__ == "__main__":
unittest.main()