data-gen-bpu/scripts/test_board_guard.py

145 lines
5.9 KiB
Python

"""Real process fault injection. No SSH or BPU used by these tests."""
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import tempfile
import time
import unittest
from unittest.mock import patch
from io import StringIO
from harness.tools.board_perf import guarded_run
GUARD = Path(__file__).resolve().parents[1] / 'harness/tools/board_guard.py'
class TransportTests(unittest.TestCase):
def test_lease_and_streamed_evidence_and_json(self):
code = "import sys; assert sys.stdin.readline()=='lease\\n'; print('live-progress',file=sys.stderr,flush=True); print('{\"ok\":true}')"
with patch('sys.stderr', new_callable=StringIO) as err:
self.assertEqual(json.loads(guarded_run([sys.executable, '-u', '-c', code], 2)), {'ok': True})
self.assertIn('live-progress', err.getvalue())
def test_nonzero_not_hidden_by_json(self):
with self.assertRaises(SystemExit) as raised:
guarded_run([sys.executable, '-c', "print('{}'); raise SystemExit(17)"], 2)
self.assertEqual(raised.exception.code, 17)
@unittest.skipUnless(sys.platform == 'linux', 'Linux process ownership and /proc required')
class GuardTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name)
self.directory = self.root / 'eval-test'
self.directory.mkdir()
self.processes = []
self.children = []
def tearDown(self):
for proc in self.processes:
if proc.poll() is None:
proc.kill()
proc.communicate(timeout=3)
for pid in self.children:
try: os.killpg(pid, signal.SIGKILL)
except ProcessLookupError: pass
self.tmp.cleanup()
def start(self, timeout=3, lease=.6, command=None):
code = ("import os,signal,time; from pathlib import Path; "
"signal.signal(signal.SIGTERM,signal.SIG_IGN); "
f"Path({str(self.directory/'pid')!r}).write_text(str(os.getpid())); "
"time.sleep(30)")
argv = [sys.executable, '-u', str(GUARD), '--directory', str(self.directory),
'--timeout', str(timeout), '--lease', str(lease), '--grace', '.15', '--',
*(command or [sys.executable, '-u', '-c', code])]
proc = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
self.processes.append(proc)
return proc
def child(self):
for _ in range(100):
p = self.directory / 'pid'
if p.exists():
pid = int(p.read_text()); self.children.append(pid); return pid
time.sleep(.02)
self.fail('child did not start')
def assert_dead(self, pid):
for _ in range(100):
try:
st = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()[0]
if st == 'Z': return
except FileNotFoundError: return
time.sleep(.02)
self.fail(f'child {pid} survived')
def test_eof_reaps_term_ignoring_child(self):
proc = self.start(); pid = self.child()
proc.stdin.close(); proc.stdin = None
proc.communicate(timeout=3)
self.assertEqual(proc.returncode, 125); self.assert_dead(pid)
self.assertEqual(json.loads((self.directory/'guard-result.json').read_text())['reason'], 'host_disconnected')
def test_missing_lease_expires_without_eof(self):
proc = self.start(); pid = self.child()
proc.wait(timeout=3)
self.assertEqual(proc.returncode, 125); self.assert_dead(pid)
self.assertEqual(json.loads((self.directory/'guard-result.json').read_text())['reason'], 'lease_expired')
def test_deadline_reaps_even_with_valid_lease(self):
proc = self.start(timeout=.35, lease=2); pid = self.child()
proc.stdin.write(b'lease\n'); proc.stdin.flush()
proc.wait(timeout=3)
self.assertEqual(proc.returncode, 124); self.assert_dead(pid)
def test_sigterm_owner_reaps_child(self):
proc = self.start(); pid = self.child()
proc.terminate(); proc.wait(timeout=3); self.assert_dead(pid)
def test_sigkill_owner_kernel_kills_child(self):
proc = self.start(); pid = self.child()
proc.kill(); proc.wait(timeout=3); self.assert_dead(pid)
def test_exited_group_leader_does_not_leave_grandchild(self):
leaf = "import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);time.sleep(30)"
code = ("import subprocess,sys,time;from pathlib import Path;"
f"p=subprocess.Popen([sys.executable,'-c',{leaf!r}]);"
f"Path({str(self.directory/'pid')!r}).write_text(str(p.pid));time.sleep(.1)")
proc = self.start(command=[sys.executable, '-c', code]); pid = self.child()
proc.wait(timeout=3); self.assert_dead(pid)
def test_unmanaged_inference_is_rejected(self):
script = self.root / 'board_bench.py'
script.write_text('import time; time.sleep(30)')
unmanaged = subprocess.Popen([sys.executable, str(script)])
self.processes.append(unmanaged)
proc = self.start()
proc.wait(timeout=3)
self.assertEqual(proc.returncode, 75)
self.assertIsNone(unmanaged.poll())
def test_board_mutex_rejects_second_owner(self):
first = self.start(lease=3); pid = self.child()
second = self.start()
second.wait(timeout=2)
self.assertEqual(second.returncode, 75)
self.assertIsNone(first.poll())
def test_success_preserves_output_and_unlocks(self):
command = [sys.executable, '-c', "print('{\"ok\":true}')"]
for _ in range(2):
proc = self.start(command=command)
proc.wait(timeout=3)
out, err = proc.communicate()
self.assertEqual(proc.returncode, 0, err)
self.assertEqual(json.loads(out), {'ok': True})
if __name__ == '__main__':
unittest.main()