"""Real process fault injection. No SSH or BPU used by these tests.""" import json import os from pathlib import Path import signal import subprocess import sys import tempfile import time import unittest from unittest.mock import patch from io import StringIO from harness.tools.board_perf import guarded_run, run from harness import execution GUARD = Path(__file__).resolve().parents[1] / 'harness/tools/board_guard.py' class TransportTests(unittest.TestCase): def test_lease_and_streamed_evidence_and_json(self): code = "import sys; assert sys.stdin.readline()=='lease\\n'; print('live-progress',file=sys.stderr,flush=True); print('{\"ok\":true}')" with patch('sys.stderr', new_callable=StringIO) as err: self.assertEqual(json.loads(guarded_run([sys.executable, '-u', '-c', code], 2)), {'ok': True}) self.assertIn('live-progress', err.getvalue()) def test_pre_auth_ssh_drop_is_replayed_but_real_failures_are_not(self): with tempfile.TemporaryDirectory() as tmp: marker = Path(tmp) / 'dropped' code = (f"import pathlib,sys; m=pathlib.Path({str(marker)!r})\n" "if not m.exists(): m.touch(); print('kex_exchange_identification: Connection closed by remote host', file=sys.stderr); raise SystemExit(255)\n" "print('{}')") with patch('sys.stderr', new_callable=StringIO), patch('time.sleep'): self.assertEqual(json.loads(guarded_run([sys.executable, '-c', code], 2)), {}) marker.unlink() self.assertEqual(json.loads(run([sys.executable, '-c', code])), {}) with self.assertRaises(SystemExit) as raised: run([sys.executable, '-c', "import sys; print('kex_exchange_identification', file=sys.stderr); raise SystemExit(255)"]) self.assertEqual(raised.exception.code, 255) # bounded: gives up after three drops def test_nonzero_not_hidden_by_json(self): with self.assertRaises(SystemExit) as raised: guarded_run([sys.executable, '-c', "print('{}'); raise SystemExit(17)"], 2) self.assertEqual(raised.exception.code, 17) def test_failed_native_stdout_is_retained_as_bounded_diagnostic(self): code = "print('x'*8000); print('native-model-load-error'); raise SystemExit(17)" with patch('sys.stderr', new_callable=StringIO) as err: with self.assertRaises(SystemExit) as raised: guarded_run([sys.executable, '-c', code], 2) self.assertEqual(raised.exception.code, 17) self.assertIn('native-model-load-error', err.getvalue()) self.assertLessEqual(len(err.getvalue()), 4000) @unittest.skipUnless(sys.platform == 'linux', 'Linux process ownership and /proc required') class GuardTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.root = Path(self.tmp.name) self.directory = self.root / 'eval-test' self.directory.mkdir() self.sysfs = self.root / 'sys' ratio = self.sysfs / 'class/bpu/bpu/ratio' ratio.parent.mkdir(parents=True) ratio.write_text('0') self.processes = [] self.children = [] def tearDown(self): for proc in self.processes: if proc.poll() is None: proc.kill() proc.communicate(timeout=3) for identity in self.children: execution.signal_process(identity, signal.SIGKILL) self.tmp.cleanup() def start(self, timeout=3, lease=.6, command=None): code = ("import os,signal,time; from pathlib import Path; " "signal.signal(signal.SIGTERM,signal.SIG_IGN); " f"Path({str(self.directory/'pid')!r}).write_text(str(os.getpid())); " "time.sleep(30)") # Inject only this test's sysfs; production CLI has no telemetry bypass. wrapper = ("import sys;from pathlib import Path;" f"sys.path.insert(0,{str(GUARD.parents[2])!r});" "from harness.tools import board_guard,board_probe;" f"raise SystemExit(board_guard.supervise({command or [sys.executable, '-u', '-c', code]!r}," f"Path({str(self.directory)!r}),{timeout!r},{lease!r},.15," f"probe=lambda:board_probe.occupancy(Path({str(self.sysfs)!r}))))") argv = [sys.executable, '-u', '-c', wrapper] proc = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.processes.append(proc) return proc def child(self): for _ in range(100): p = self.directory / 'pid' if p.exists(): pid = int(p.read_text()) self.children.append(execution.process_identity(pid)) return pid time.sleep(.02) self.fail('child did not start') def assert_dead(self, pid): for _ in range(100): try: st = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()[0] if st == 'Z': return except (FileNotFoundError, ProcessLookupError): return time.sleep(.02) self.fail(f'child {pid} survived') def test_eof_reaps_term_ignoring_child(self): proc = self.start(); pid = self.child() proc.stdin.close(); proc.stdin = None proc.communicate(timeout=3) self.assertEqual(proc.returncode, 125); self.assert_dead(pid) self.assertEqual(json.loads((self.directory/'guard-result.json').read_text())['reason'], 'host_disconnected') def test_missing_lease_expires_without_eof(self): proc = self.start(); pid = self.child() proc.wait(timeout=3) self.assertEqual(proc.returncode, 125); self.assert_dead(pid) self.assertEqual(json.loads((self.directory/'guard-result.json').read_text())['reason'], 'lease_expired') def test_deadline_reaps_even_with_valid_lease(self): proc = self.start(timeout=.35, lease=2); pid = self.child() proc.stdin.write(b'lease\n'); proc.stdin.flush() proc.wait(timeout=3) self.assertEqual(proc.returncode, 124); self.assert_dead(pid) def test_sigterm_owner_reaps_child(self): proc = self.start(); pid = self.child() proc.terminate(); proc.wait(timeout=3); self.assert_dead(pid) def test_sigkill_owner_kernel_kills_child(self): proc = self.start(); pid = self.child() proc.kill(); proc.wait(timeout=3); self.assert_dead(pid) def test_exited_group_leader_does_not_leave_grandchild(self): leaf = "import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);time.sleep(30)" code = ("import subprocess,sys,time;from pathlib import Path;" f"p=subprocess.Popen([sys.executable,'-c',{leaf!r}]);" f"Path({str(self.directory/'pid')!r}).write_text(str(p.pid));time.sleep(.1)") proc = self.start(command=[sys.executable, '-c', code]); pid = self.child() proc.wait(timeout=3); self.assert_dead(pid) def test_double_fork_cleared_env_and_session_cannot_escape_guard(self): code = f"""import os,signal,time r,w=os.pipe() if os.fork(): os.close(w) open({str(self.directory/'pid')!r},'wb').write(os.read(r,100)) os._exit(0) os.close(r); os.environ.clear(); os.setsid() if os.fork(): os._exit(0) signal.signal(signal.SIGTERM,signal.SIG_IGN) os.write(w,str(os.getpid()).encode()); os.close(w) time.sleep(30) """ proc = self.start(command=[sys.executable, '-c', code]); pid = self.child() proc.wait(timeout=3); self.assert_dead(pid) receipt = json.loads((self.directory/'guard-result.json').read_text()) self.assertEqual(receipt['proof'], 'waitpid:ECHILD') self.assertTrue(receipt['all_children_reaped']) self.assertEqual(receipt['cleanup_state'], 'VERIFIED') def test_sigkill_missing_receipt_blocks_only_while_operation_survives(self): quarantine = self.directory.parent/'.board-inference.quarantine.json' first = self.start(); pid = self.child() first.kill(); first.wait(timeout=3); self.assert_dead(pid) self.assertFalse((self.directory/'guard-result.json').exists()) token = json.loads(quarantine.read_text())['token'] survivor = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'], env={**os.environ, 'BENCH_BOARD_OPERATION': token}) self.processes.append(survivor) second = self.start() second.wait(timeout=3) self.assertEqual(second.returncode, 78) self.assertTrue(quarantine.exists()) survivor.kill(); survivor.wait(timeout=3) (self.directory/'pid').unlink() third = self.start(); self.child() # admitted: the stale record no longer blocks the board self.assertIn(b'stale_quarantine_cleared', third.stderr.read1(4096) + third.stderr.read1(4096)) def test_unmanaged_inference_is_rejected(self): script = self.root / 'board_bench.py' script.write_text('import time; time.sleep(30)') unmanaged = subprocess.Popen([sys.executable, str(script)]) self.processes.append(unmanaged) proc = self.start() proc.wait(timeout=3) self.assertEqual(proc.returncode, 75) self.assertIsNone(unmanaged.poll()) def test_board_mutex_rejects_second_owner(self): first = self.start(lease=3); pid = self.child() second = self.start() second.wait(timeout=2) self.assertEqual(second.returncode, 75) self.assertIsNone(first.poll()) def test_success_preserves_output_and_unlocks(self): command = [sys.executable, '-c', "print('{\"ok\":true}')"] for _ in range(2): proc = self.start(command=command) proc.wait(timeout=3) out, err = proc.communicate() self.assertEqual(proc.returncode, 0, err) self.assertEqual(json.loads(out), {'ok': True}) def test_next_chunk_rechecks_probe_after_previous_child_reaped(self): command = [sys.executable, '-c', "print('{\"chunk\":true}')"] ratio = self.sysfs / 'class/bpu/bpu/ratio' for value, expected in ((0, 0), (30, 75), (0, 0)): ratio.write_text(str(value)) proc = self.start(command=command) proc.wait(timeout=3) out, err = proc.communicate() self.assertEqual(proc.returncode, expected, err) events = [json.loads(line) for line in err.splitlines()] self.assertEqual(any(e['event'] == 'started' for e in events), expected == 0) if __name__ == '__main__': unittest.main()