data-gen-bpu/scripts/test_board_guard.py

206 lines
8.9 KiB
Python

"""Real process fault injection. No SSH or BPU used by these tests."""
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import tempfile
import time
import unittest
from unittest.mock import patch
from io import StringIO
from harness.tools.board_perf import guarded_run
from harness import execution
GUARD = Path(__file__).resolve().parents[1] / 'harness/tools/board_guard.py'
class TransportTests(unittest.TestCase):
def test_lease_and_streamed_evidence_and_json(self):
code = "import sys; assert sys.stdin.readline()=='lease\\n'; print('live-progress',file=sys.stderr,flush=True); print('{\"ok\":true}')"
with patch('sys.stderr', new_callable=StringIO) as err:
self.assertEqual(json.loads(guarded_run([sys.executable, '-u', '-c', code], 2)), {'ok': True})
self.assertIn('live-progress', err.getvalue())
def test_nonzero_not_hidden_by_json(self):
with self.assertRaises(SystemExit) as raised:
guarded_run([sys.executable, '-c', "print('{}'); raise SystemExit(17)"], 2)
self.assertEqual(raised.exception.code, 17)
def test_failed_native_stdout_is_retained_as_bounded_diagnostic(self):
code = "print('x'*8000); print('native-model-load-error'); raise SystemExit(17)"
with patch('sys.stderr', new_callable=StringIO) as err:
with self.assertRaises(SystemExit) as raised:
guarded_run([sys.executable, '-c', code], 2)
self.assertEqual(raised.exception.code, 17)
self.assertIn('native-model-load-error', err.getvalue())
self.assertLessEqual(len(err.getvalue()), 4000)
@unittest.skipUnless(sys.platform == 'linux', 'Linux process ownership and /proc required')
class GuardTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name)
self.directory = self.root / 'eval-test'
self.directory.mkdir()
self.sysfs = self.root / 'sys'
ratio = self.sysfs / 'class/bpu/bpu/ratio'
ratio.parent.mkdir(parents=True)
ratio.write_text('0')
self.processes = []
self.children = []
def tearDown(self):
for proc in self.processes:
if proc.poll() is None:
proc.kill()
proc.communicate(timeout=3)
for identity in self.children:
execution.signal_process(identity, signal.SIGKILL)
self.tmp.cleanup()
def start(self, timeout=3, lease=.6, command=None):
code = ("import os,signal,time; from pathlib import Path; "
"signal.signal(signal.SIGTERM,signal.SIG_IGN); "
f"Path({str(self.directory/'pid')!r}).write_text(str(os.getpid())); "
"time.sleep(30)")
# Inject only this test's sysfs; production CLI has no telemetry bypass.
wrapper = ("import sys;from pathlib import Path;"
f"sys.path.insert(0,{str(GUARD.parents[2])!r});"
"from harness.tools import board_guard,board_probe;"
f"raise SystemExit(board_guard.supervise({command or [sys.executable, '-u', '-c', code]!r},"
f"Path({str(self.directory)!r}),{timeout!r},{lease!r},.15,"
f"probe=lambda:board_probe.occupancy(Path({str(self.sysfs)!r}))))")
argv = [sys.executable, '-u', '-c', wrapper]
proc = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
self.processes.append(proc)
return proc
def child(self):
for _ in range(100):
p = self.directory / 'pid'
if p.exists():
pid = int(p.read_text())
self.children.append(execution.process_identity(pid))
return pid
time.sleep(.02)
self.fail('child did not start')
def assert_dead(self, pid):
for _ in range(100):
try:
st = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()[0]
if st == 'Z': return
except (FileNotFoundError, ProcessLookupError): return
time.sleep(.02)
self.fail(f'child {pid} survived')
def test_eof_reaps_term_ignoring_child(self):
proc = self.start(); pid = self.child()
proc.stdin.close(); proc.stdin = None
proc.communicate(timeout=3)
self.assertEqual(proc.returncode, 125); self.assert_dead(pid)
self.assertEqual(json.loads((self.directory/'guard-result.json').read_text())['reason'], 'host_disconnected')
def test_missing_lease_expires_without_eof(self):
proc = self.start(); pid = self.child()
proc.wait(timeout=3)
self.assertEqual(proc.returncode, 125); self.assert_dead(pid)
self.assertEqual(json.loads((self.directory/'guard-result.json').read_text())['reason'], 'lease_expired')
def test_deadline_reaps_even_with_valid_lease(self):
proc = self.start(timeout=.35, lease=2); pid = self.child()
proc.stdin.write(b'lease\n'); proc.stdin.flush()
proc.wait(timeout=3)
self.assertEqual(proc.returncode, 124); self.assert_dead(pid)
def test_sigterm_owner_reaps_child(self):
proc = self.start(); pid = self.child()
proc.terminate(); proc.wait(timeout=3); self.assert_dead(pid)
def test_sigkill_owner_kernel_kills_child(self):
proc = self.start(); pid = self.child()
proc.kill(); proc.wait(timeout=3); self.assert_dead(pid)
def test_exited_group_leader_does_not_leave_grandchild(self):
leaf = "import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);time.sleep(30)"
code = ("import subprocess,sys,time;from pathlib import Path;"
f"p=subprocess.Popen([sys.executable,'-c',{leaf!r}]);"
f"Path({str(self.directory/'pid')!r}).write_text(str(p.pid));time.sleep(.1)")
proc = self.start(command=[sys.executable, '-c', code]); pid = self.child()
proc.wait(timeout=3); self.assert_dead(pid)
def test_double_fork_cleared_env_and_session_cannot_escape_guard(self):
code = f"""import os,signal,time
r,w=os.pipe()
if os.fork():
os.close(w)
open({str(self.directory/'pid')!r},'wb').write(os.read(r,100))
os._exit(0)
os.close(r); os.environ.clear(); os.setsid()
if os.fork(): os._exit(0)
signal.signal(signal.SIGTERM,signal.SIG_IGN)
os.write(w,str(os.getpid()).encode()); os.close(w)
time.sleep(30)
"""
proc = self.start(command=[sys.executable, '-c', code]); pid = self.child()
proc.wait(timeout=3); self.assert_dead(pid)
receipt = json.loads((self.directory/'guard-result.json').read_text())
self.assertEqual(receipt['proof'], 'waitpid:ECHILD')
self.assertTrue(receipt['all_children_reaped'])
self.assertEqual(receipt['cleanup_state'], 'VERIFIED')
def test_sigkill_missing_receipt_blocks_next_board_admission(self):
first = self.start(); pid = self.child()
first.kill(); first.wait(timeout=3); self.assert_dead(pid)
self.assertFalse((self.directory/'guard-result.json').exists())
second = self.start()
second.wait(timeout=3)
self.assertEqual(second.returncode, 78)
self.assertTrue((self.directory.parent/'.board-inference.quarantine.json').exists())
def test_unmanaged_inference_is_rejected(self):
script = self.root / 'board_bench.py'
script.write_text('import time; time.sleep(30)')
unmanaged = subprocess.Popen([sys.executable, str(script)])
self.processes.append(unmanaged)
proc = self.start()
proc.wait(timeout=3)
self.assertEqual(proc.returncode, 75)
self.assertIsNone(unmanaged.poll())
def test_board_mutex_rejects_second_owner(self):
first = self.start(lease=3); pid = self.child()
second = self.start()
second.wait(timeout=2)
self.assertEqual(second.returncode, 75)
self.assertIsNone(first.poll())
def test_success_preserves_output_and_unlocks(self):
command = [sys.executable, '-c', "print('{\"ok\":true}')"]
for _ in range(2):
proc = self.start(command=command)
proc.wait(timeout=3)
out, err = proc.communicate()
self.assertEqual(proc.returncode, 0, err)
self.assertEqual(json.loads(out), {'ok': True})
def test_next_chunk_rechecks_probe_after_previous_child_reaped(self):
command = [sys.executable, '-c', "print('{\"chunk\":true}')"]
ratio = self.sysfs / 'class/bpu/bpu/ratio'
for value, expected in ((0, 0), (30, 75), (0, 0)):
ratio.write_text(str(value))
proc = self.start(command=command)
proc.wait(timeout=3)
out, err = proc.communicate()
self.assertEqual(proc.returncode, expected, err)
events = [json.loads(line) for line in err.splitlines()]
self.assertEqual(any(e['event'] == 'started' for e in events), expected == 0)
if __name__ == '__main__':
unittest.main()