151 lines
7.3 KiB
Python
151 lines
7.3 KiB
Python
"""Occupancy uses one per-core fact reader; unavailable telemetry cannot admit work."""
|
|
from io import StringIO
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness.tools import board_bench, board_guard, board_probe
|
|
|
|
|
|
class BoardProbeTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temp.cleanup)
|
|
self.root = Path(temp.name)
|
|
self.sysfs = self.root / 'sys'
|
|
self.proc = self.root / 'proc'
|
|
self.proc.mkdir()
|
|
|
|
def ratio(self, path, value):
|
|
target = self.sysfs / path
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_text(str(value))
|
|
return target
|
|
|
|
def occupancy(self):
|
|
return board_probe.occupancy(self.sysfs, self.proc)
|
|
|
|
def test_each_legacy_layout_is_read(self):
|
|
for path in ('class/bpu/bpu_core0/ratio', 'class/bpu/bpu/ratio'):
|
|
with self.subTest(path=path):
|
|
target = self.ratio(path, 2)
|
|
result = self.occupancy()
|
|
self.assertEqual(result['status'], 'idle')
|
|
self.assertEqual(result['bpu']['cores'][0]['path'], str(target))
|
|
self.assertEqual(result['bpu']['mean_pct'], 2)
|
|
target.unlink()
|
|
target.parent.rmdir()
|
|
|
|
def test_four_core_values_and_aliases_preserve_one_busy_core(self):
|
|
targets = [self.ratio(f'devices/system/bpu/bpu{i}/ratio', value)
|
|
for i, value in enumerate((0, 0, 20, 0))]
|
|
old = self.sysfs / 'class/bpu'
|
|
old.mkdir(parents=True)
|
|
(old / 'bpu_core0').symlink_to(targets[0].parent, target_is_directory=True)
|
|
self.ratio('class/bpu/bpu/ratio', 0) # aggregate fallback must not mask cores
|
|
result = self.occupancy()
|
|
self.assertEqual(result['status'], 'busy')
|
|
self.assertEqual(result['bpu']['mean_pct'], 5)
|
|
self.assertEqual(result['bpu']['max_pct'], 20)
|
|
self.assertEqual([c['ratio_pct'] for c in result['bpu']['cores']], [0, 0, 20, 0])
|
|
|
|
def test_absent_missing_malformed_and_nonfinite_are_unknown(self):
|
|
self.assertEqual(self.occupancy()['status'], 'unknown')
|
|
self.ratio('devices/system/bpu/bpu0/ratio', 0)
|
|
other = self.sysfs / 'devices/system/bpu/bpu1/ratio'
|
|
other.parent.mkdir()
|
|
for value in (None, '', 'broken', 'nan', 'inf', -1, 101):
|
|
with self.subTest(value=value):
|
|
if value is not None:
|
|
other.write_text(str(value))
|
|
result = self.occupancy()
|
|
self.assertEqual(result['status'], 'unknown')
|
|
self.assertEqual(result['bpu']['status'], 'partial')
|
|
self.assertIsNone(result['bpu']['mean_pct'])
|
|
self.assertIsNotNone(result['bpu']['cores'][1]['error'])
|
|
|
|
def test_new_core_family_wins_over_empty_legacy_placeholders(self):
|
|
for i in range(4):
|
|
self.ratio(f'devices/system/bpu/bpu{i}/ratio', 0)
|
|
(self.sysfs / f'class/bpu/bpu_core{i}').mkdir(parents=True)
|
|
result = self.occupancy()
|
|
self.assertEqual(result['status'], 'idle')
|
|
self.assertEqual(len(result['bpu']['cores']), 4)
|
|
# A genuinely missing reading in the selected family still cannot use old data.
|
|
(self.sysfs / 'devices/system/bpu/bpu3/ratio').unlink()
|
|
self.ratio('class/bpu/bpu_core3/ratio', 0)
|
|
self.assertEqual(self.occupancy()['status'], 'unknown')
|
|
|
|
def test_process_probe_skips_self_and_does_not_kill_competitors(self):
|
|
self.ratio('class/bpu/bpu/ratio', 0)
|
|
for pid, args in ((os.getpid(), ['python', 'board_bench.py']),
|
|
(987654, ['python', '/test/board_bench.py']),
|
|
(987656, ['python', '-c', 'import hbm_runtime; hbm_runtime.HB_HBMRuntime("x")']),
|
|
(987655, ['ssh', 'python3 /test/board_bench.py --runs 2'])):
|
|
directory = self.proc / str(pid)
|
|
directory.mkdir()
|
|
(directory / 'cmdline').write_bytes('\0'.join(args).encode())
|
|
with patch.object(os, 'kill') as kill:
|
|
result = self.occupancy()
|
|
self.assertEqual(result['pids'], [987654, 987656])
|
|
self.assertEqual(result['status'], 'busy')
|
|
kill.assert_not_called()
|
|
|
|
def test_recognizes_execution_without_matching_readonly_arguments(self):
|
|
cases = [
|
|
(['tail', '-f', '/tmp/dnn'], False),
|
|
(['sh', '-c', 'tail -f /tmp/hbm_runtime.log'], False),
|
|
(['python3', '-c', 'import hbm_runtime; print("HB_HBMRuntime(x)")'], False),
|
|
(['python3', '/tmp/read.py', 'board_bench.py'], False),
|
|
(['python3.10', '-u', '/tmp/board_bench.py'], True),
|
|
(['python3', '-m', 'harness.tools.board_bench'], True),
|
|
(['python3', '-c', 'from hbm_runtime import HB_HBMRuntime; m=HB_HBMRuntime("x")'], True),
|
|
(['python3', '-c', 'import dnn; dnn.load("x")'], True),
|
|
(['/usr/bin/hrt_model_exec', '--model', 'x.hbm'], True),
|
|
]
|
|
for args, expected in cases:
|
|
with self.subTest(args=args):
|
|
self.assertEqual(board_probe._inference_command(args), expected)
|
|
|
|
def test_performance_scalar_and_bounded_raw_sample_contract(self):
|
|
self.ratio('class/bpu/bpu/ratio', 20)
|
|
raw = board_probe.read_bpu_ratios(self.sysfs)
|
|
result = board_bench.ratio_summary([20, 40], [raw])
|
|
self.assertAlmostEqual(result['bpu_util'], .3)
|
|
self.assertEqual(result['bpu_ratios'], [raw])
|
|
self.assertEqual(result['bpu_ratio_samples'], 2)
|
|
self.assertIsNone(board_bench.ratio_summary([20, None], [raw])['bpu_util'])
|
|
self.assertIsNone(board_bench.ratio_summary([], [])['bpu_util'])
|
|
|
|
def test_guard_probes_under_lock_and_never_spawns_busy_or_unknown(self):
|
|
directory = self.root / 'eval'
|
|
directory.mkdir()
|
|
for status, code in (('busy', 75), ('unknown', 78)):
|
|
with self.subTest(status=status):
|
|
def probe():
|
|
with (self.root / '.board-inference.lock').open('a') as contender:
|
|
with self.assertRaises(BlockingIOError):
|
|
fcntl.flock(contender, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
return {'status': status, 'pids': [], 'bpu': {'status': 'unknown'}}
|
|
with patch.object(board_guard.subprocess, 'Popen') as child, \
|
|
patch('sys.stderr', new_callable=StringIO) as evidence:
|
|
self.assertEqual(board_guard.supervise(['not-started'], directory, 1, probe=probe), code)
|
|
child.assert_not_called()
|
|
records = [json.loads(line) for line in evidence.getvalue().splitlines()]
|
|
self.assertNotIn('started', [row['event'] for row in records])
|
|
receipt = json.loads((directory / 'guard-result.json').read_text())
|
|
self.assertEqual(receipt['proof'], 'not_started')
|
|
self.assertEqual(receipt['cleanup_state'], 'VERIFIED')
|
|
self.assertTrue(receipt['all_children_reaped'])
|
|
self.assertIsNone(receipt['pid'])
|
|
self.assertIsNone(receipt['child_exit_code'])
|
|
self.assertFalse((self.root / '.board-inference.quarantine.json').exists())
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|