145 lines
5.9 KiB
Python
145 lines
5.9 KiB
Python
"""Real process fault injection. No SSH or BPU used by these tests."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
from io import StringIO
|
|
|
|
from harness.tools.board_perf import guarded_run
|
|
|
|
GUARD = Path(__file__).resolve().parents[1] / 'harness/tools/board_guard.py'
|
|
|
|
|
|
class TransportTests(unittest.TestCase):
|
|
def test_lease_and_streamed_evidence_and_json(self):
|
|
code = "import sys; assert sys.stdin.readline()=='lease\\n'; print('live-progress',file=sys.stderr,flush=True); print('{\"ok\":true}')"
|
|
with patch('sys.stderr', new_callable=StringIO) as err:
|
|
self.assertEqual(json.loads(guarded_run([sys.executable, '-u', '-c', code], 2)), {'ok': True})
|
|
self.assertIn('live-progress', err.getvalue())
|
|
|
|
def test_nonzero_not_hidden_by_json(self):
|
|
with self.assertRaises(SystemExit) as raised:
|
|
guarded_run([sys.executable, '-c', "print('{}'); raise SystemExit(17)"], 2)
|
|
self.assertEqual(raised.exception.code, 17)
|
|
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'Linux process ownership and /proc required')
|
|
class GuardTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.tmp.name)
|
|
self.directory = self.root / 'eval-test'
|
|
self.directory.mkdir()
|
|
self.processes = []
|
|
self.children = []
|
|
|
|
def tearDown(self):
|
|
for proc in self.processes:
|
|
if proc.poll() is None:
|
|
proc.kill()
|
|
proc.communicate(timeout=3)
|
|
for pid in self.children:
|
|
try: os.killpg(pid, signal.SIGKILL)
|
|
except ProcessLookupError: pass
|
|
self.tmp.cleanup()
|
|
|
|
def start(self, timeout=3, lease=.6, command=None):
|
|
code = ("import os,signal,time; from pathlib import Path; "
|
|
"signal.signal(signal.SIGTERM,signal.SIG_IGN); "
|
|
f"Path({str(self.directory/'pid')!r}).write_text(str(os.getpid())); "
|
|
"time.sleep(30)")
|
|
argv = [sys.executable, '-u', str(GUARD), '--directory', str(self.directory),
|
|
'--timeout', str(timeout), '--lease', str(lease), '--grace', '.15', '--',
|
|
*(command or [sys.executable, '-u', '-c', code])]
|
|
proc = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
self.processes.append(proc)
|
|
return proc
|
|
|
|
def child(self):
|
|
for _ in range(100):
|
|
p = self.directory / 'pid'
|
|
if p.exists():
|
|
pid = int(p.read_text()); self.children.append(pid); return pid
|
|
time.sleep(.02)
|
|
self.fail('child did not start')
|
|
|
|
def assert_dead(self, pid):
|
|
for _ in range(100):
|
|
try:
|
|
st = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()[0]
|
|
if st == 'Z': return
|
|
except FileNotFoundError: return
|
|
time.sleep(.02)
|
|
self.fail(f'child {pid} survived')
|
|
|
|
def test_eof_reaps_term_ignoring_child(self):
|
|
proc = self.start(); pid = self.child()
|
|
proc.stdin.close(); proc.stdin = None
|
|
proc.communicate(timeout=3)
|
|
self.assertEqual(proc.returncode, 125); self.assert_dead(pid)
|
|
self.assertEqual(json.loads((self.directory/'guard-result.json').read_text())['reason'], 'host_disconnected')
|
|
|
|
def test_missing_lease_expires_without_eof(self):
|
|
proc = self.start(); pid = self.child()
|
|
proc.wait(timeout=3)
|
|
self.assertEqual(proc.returncode, 125); self.assert_dead(pid)
|
|
self.assertEqual(json.loads((self.directory/'guard-result.json').read_text())['reason'], 'lease_expired')
|
|
|
|
def test_deadline_reaps_even_with_valid_lease(self):
|
|
proc = self.start(timeout=.35, lease=2); pid = self.child()
|
|
proc.stdin.write(b'lease\n'); proc.stdin.flush()
|
|
proc.wait(timeout=3)
|
|
self.assertEqual(proc.returncode, 124); self.assert_dead(pid)
|
|
|
|
def test_sigterm_owner_reaps_child(self):
|
|
proc = self.start(); pid = self.child()
|
|
proc.terminate(); proc.wait(timeout=3); self.assert_dead(pid)
|
|
|
|
def test_sigkill_owner_kernel_kills_child(self):
|
|
proc = self.start(); pid = self.child()
|
|
proc.kill(); proc.wait(timeout=3); self.assert_dead(pid)
|
|
|
|
def test_exited_group_leader_does_not_leave_grandchild(self):
|
|
leaf = "import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);time.sleep(30)"
|
|
code = ("import subprocess,sys,time;from pathlib import Path;"
|
|
f"p=subprocess.Popen([sys.executable,'-c',{leaf!r}]);"
|
|
f"Path({str(self.directory/'pid')!r}).write_text(str(p.pid));time.sleep(.1)")
|
|
proc = self.start(command=[sys.executable, '-c', code]); pid = self.child()
|
|
proc.wait(timeout=3); self.assert_dead(pid)
|
|
|
|
def test_unmanaged_inference_is_rejected(self):
|
|
script = self.root / 'board_bench.py'
|
|
script.write_text('import time; time.sleep(30)')
|
|
unmanaged = subprocess.Popen([sys.executable, str(script)])
|
|
self.processes.append(unmanaged)
|
|
proc = self.start()
|
|
proc.wait(timeout=3)
|
|
self.assertEqual(proc.returncode, 75)
|
|
self.assertIsNone(unmanaged.poll())
|
|
|
|
def test_board_mutex_rejects_second_owner(self):
|
|
first = self.start(lease=3); pid = self.child()
|
|
second = self.start()
|
|
second.wait(timeout=2)
|
|
self.assertEqual(second.returncode, 75)
|
|
self.assertIsNone(first.poll())
|
|
|
|
def test_success_preserves_output_and_unlocks(self):
|
|
command = [sys.executable, '-c', "print('{\"ok\":true}')"]
|
|
for _ in range(2):
|
|
proc = self.start(command=command)
|
|
proc.wait(timeout=3)
|
|
out, err = proc.communicate()
|
|
self.assertEqual(proc.returncode, 0, err)
|
|
self.assertEqual(json.loads(out), {'ok': True})
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|