206 lines
8.9 KiB
Python
206 lines
8.9 KiB
Python
"""Real process fault injection. No SSH or BPU used by these tests."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
from io import StringIO
|
|
|
|
from harness.tools.board_perf import guarded_run
|
|
from harness import execution
|
|
|
|
GUARD = Path(__file__).resolve().parents[1] / 'harness/tools/board_guard.py'
|
|
|
|
|
|
class TransportTests(unittest.TestCase):
|
|
def test_lease_and_streamed_evidence_and_json(self):
|
|
code = "import sys; assert sys.stdin.readline()=='lease\\n'; print('live-progress',file=sys.stderr,flush=True); print('{\"ok\":true}')"
|
|
with patch('sys.stderr', new_callable=StringIO) as err:
|
|
self.assertEqual(json.loads(guarded_run([sys.executable, '-u', '-c', code], 2)), {'ok': True})
|
|
self.assertIn('live-progress', err.getvalue())
|
|
|
|
def test_nonzero_not_hidden_by_json(self):
|
|
with self.assertRaises(SystemExit) as raised:
|
|
guarded_run([sys.executable, '-c', "print('{}'); raise SystemExit(17)"], 2)
|
|
self.assertEqual(raised.exception.code, 17)
|
|
|
|
def test_failed_native_stdout_is_retained_as_bounded_diagnostic(self):
|
|
code = "print('x'*8000); print('native-model-load-error'); raise SystemExit(17)"
|
|
with patch('sys.stderr', new_callable=StringIO) as err:
|
|
with self.assertRaises(SystemExit) as raised:
|
|
guarded_run([sys.executable, '-c', code], 2)
|
|
self.assertEqual(raised.exception.code, 17)
|
|
self.assertIn('native-model-load-error', err.getvalue())
|
|
self.assertLessEqual(len(err.getvalue()), 4000)
|
|
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'Linux process ownership and /proc required')
|
|
class GuardTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.tmp.name)
|
|
self.directory = self.root / 'eval-test'
|
|
self.directory.mkdir()
|
|
self.sysfs = self.root / 'sys'
|
|
ratio = self.sysfs / 'class/bpu/bpu/ratio'
|
|
ratio.parent.mkdir(parents=True)
|
|
ratio.write_text('0')
|
|
self.processes = []
|
|
self.children = []
|
|
|
|
def tearDown(self):
|
|
for proc in self.processes:
|
|
if proc.poll() is None:
|
|
proc.kill()
|
|
proc.communicate(timeout=3)
|
|
for identity in self.children:
|
|
execution.signal_process(identity, signal.SIGKILL)
|
|
self.tmp.cleanup()
|
|
|
|
def start(self, timeout=3, lease=.6, command=None):
|
|
code = ("import os,signal,time; from pathlib import Path; "
|
|
"signal.signal(signal.SIGTERM,signal.SIG_IGN); "
|
|
f"Path({str(self.directory/'pid')!r}).write_text(str(os.getpid())); "
|
|
"time.sleep(30)")
|
|
# Inject only this test's sysfs; production CLI has no telemetry bypass.
|
|
wrapper = ("import sys;from pathlib import Path;"
|
|
f"sys.path.insert(0,{str(GUARD.parents[2])!r});"
|
|
"from harness.tools import board_guard,board_probe;"
|
|
f"raise SystemExit(board_guard.supervise({command or [sys.executable, '-u', '-c', code]!r},"
|
|
f"Path({str(self.directory)!r}),{timeout!r},{lease!r},.15,"
|
|
f"probe=lambda:board_probe.occupancy(Path({str(self.sysfs)!r}))))")
|
|
argv = [sys.executable, '-u', '-c', wrapper]
|
|
proc = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
self.processes.append(proc)
|
|
return proc
|
|
|
|
def child(self):
|
|
for _ in range(100):
|
|
p = self.directory / 'pid'
|
|
if p.exists():
|
|
pid = int(p.read_text())
|
|
self.children.append(execution.process_identity(pid))
|
|
return pid
|
|
time.sleep(.02)
|
|
self.fail('child did not start')
|
|
|
|
def assert_dead(self, pid):
|
|
for _ in range(100):
|
|
try:
|
|
st = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()[0]
|
|
if st == 'Z': return
|
|
except (FileNotFoundError, ProcessLookupError): return
|
|
time.sleep(.02)
|
|
self.fail(f'child {pid} survived')
|
|
|
|
def test_eof_reaps_term_ignoring_child(self):
|
|
proc = self.start(); pid = self.child()
|
|
proc.stdin.close(); proc.stdin = None
|
|
proc.communicate(timeout=3)
|
|
self.assertEqual(proc.returncode, 125); self.assert_dead(pid)
|
|
self.assertEqual(json.loads((self.directory/'guard-result.json').read_text())['reason'], 'host_disconnected')
|
|
|
|
def test_missing_lease_expires_without_eof(self):
|
|
proc = self.start(); pid = self.child()
|
|
proc.wait(timeout=3)
|
|
self.assertEqual(proc.returncode, 125); self.assert_dead(pid)
|
|
self.assertEqual(json.loads((self.directory/'guard-result.json').read_text())['reason'], 'lease_expired')
|
|
|
|
def test_deadline_reaps_even_with_valid_lease(self):
|
|
proc = self.start(timeout=.35, lease=2); pid = self.child()
|
|
proc.stdin.write(b'lease\n'); proc.stdin.flush()
|
|
proc.wait(timeout=3)
|
|
self.assertEqual(proc.returncode, 124); self.assert_dead(pid)
|
|
|
|
def test_sigterm_owner_reaps_child(self):
|
|
proc = self.start(); pid = self.child()
|
|
proc.terminate(); proc.wait(timeout=3); self.assert_dead(pid)
|
|
|
|
def test_sigkill_owner_kernel_kills_child(self):
|
|
proc = self.start(); pid = self.child()
|
|
proc.kill(); proc.wait(timeout=3); self.assert_dead(pid)
|
|
|
|
def test_exited_group_leader_does_not_leave_grandchild(self):
|
|
leaf = "import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);time.sleep(30)"
|
|
code = ("import subprocess,sys,time;from pathlib import Path;"
|
|
f"p=subprocess.Popen([sys.executable,'-c',{leaf!r}]);"
|
|
f"Path({str(self.directory/'pid')!r}).write_text(str(p.pid));time.sleep(.1)")
|
|
proc = self.start(command=[sys.executable, '-c', code]); pid = self.child()
|
|
proc.wait(timeout=3); self.assert_dead(pid)
|
|
|
|
def test_double_fork_cleared_env_and_session_cannot_escape_guard(self):
|
|
code = f"""import os,signal,time
|
|
r,w=os.pipe()
|
|
if os.fork():
|
|
os.close(w)
|
|
open({str(self.directory/'pid')!r},'wb').write(os.read(r,100))
|
|
os._exit(0)
|
|
os.close(r); os.environ.clear(); os.setsid()
|
|
if os.fork(): os._exit(0)
|
|
signal.signal(signal.SIGTERM,signal.SIG_IGN)
|
|
os.write(w,str(os.getpid()).encode()); os.close(w)
|
|
time.sleep(30)
|
|
"""
|
|
proc = self.start(command=[sys.executable, '-c', code]); pid = self.child()
|
|
proc.wait(timeout=3); self.assert_dead(pid)
|
|
receipt = json.loads((self.directory/'guard-result.json').read_text())
|
|
self.assertEqual(receipt['proof'], 'waitpid:ECHILD')
|
|
self.assertTrue(receipt['all_children_reaped'])
|
|
self.assertEqual(receipt['cleanup_state'], 'VERIFIED')
|
|
|
|
def test_sigkill_missing_receipt_blocks_next_board_admission(self):
|
|
first = self.start(); pid = self.child()
|
|
first.kill(); first.wait(timeout=3); self.assert_dead(pid)
|
|
self.assertFalse((self.directory/'guard-result.json').exists())
|
|
second = self.start()
|
|
second.wait(timeout=3)
|
|
self.assertEqual(second.returncode, 78)
|
|
self.assertTrue((self.directory.parent/'.board-inference.quarantine.json').exists())
|
|
|
|
def test_unmanaged_inference_is_rejected(self):
|
|
script = self.root / 'board_bench.py'
|
|
script.write_text('import time; time.sleep(30)')
|
|
unmanaged = subprocess.Popen([sys.executable, str(script)])
|
|
self.processes.append(unmanaged)
|
|
proc = self.start()
|
|
proc.wait(timeout=3)
|
|
self.assertEqual(proc.returncode, 75)
|
|
self.assertIsNone(unmanaged.poll())
|
|
|
|
def test_board_mutex_rejects_second_owner(self):
|
|
first = self.start(lease=3); pid = self.child()
|
|
second = self.start()
|
|
second.wait(timeout=2)
|
|
self.assertEqual(second.returncode, 75)
|
|
self.assertIsNone(first.poll())
|
|
|
|
def test_success_preserves_output_and_unlocks(self):
|
|
command = [sys.executable, '-c', "print('{\"ok\":true}')"]
|
|
for _ in range(2):
|
|
proc = self.start(command=command)
|
|
proc.wait(timeout=3)
|
|
out, err = proc.communicate()
|
|
self.assertEqual(proc.returncode, 0, err)
|
|
self.assertEqual(json.loads(out), {'ok': True})
|
|
|
|
def test_next_chunk_rechecks_probe_after_previous_child_reaped(self):
|
|
command = [sys.executable, '-c', "print('{\"chunk\":true}')"]
|
|
ratio = self.sysfs / 'class/bpu/bpu/ratio'
|
|
for value, expected in ((0, 0), (30, 75), (0, 0)):
|
|
ratio.write_text(str(value))
|
|
proc = self.start(command=command)
|
|
proc.wait(timeout=3)
|
|
out, err = proc.communicate()
|
|
self.assertEqual(proc.returncode, expected, err)
|
|
events = [json.loads(line) for line in err.splitlines()]
|
|
self.assertEqual(any(e['event'] == 'started' for e in events), expected == 0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|