data-gen-bpu/scripts/test_board_guard.py
ming01.wang f61d2efebb fix: replay board ssh calls dropped before authentication
Board ssh ports are public and constantly hold 15-35 unauthenticated
scanner connections; whenever that exceeds sshd MaxStartups, our handshake
is closed with kex_exchange_identification. Only upload steps retried, so
a single drop failed the bench call itself (exit 255) or lost the cleanup
receipt fetch, which then quarantined the board.

run() and guarded_run() now replay a call up to twice when ssh reports
kex_exchange_identification: the connection closed before auth, so the
board never saw the command and replay cannot duplicate work. Any other
failure is returned unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 09:18:48 +08:00

229 lines
10 KiB
Python

"""Real process fault injection. No SSH or BPU used by these tests."""
import json
import os
from pathlib import Path
import signal
import subprocess
import sys
import tempfile
import time
import unittest
from unittest.mock import patch
from io import StringIO
from harness.tools.board_perf import guarded_run, run
from harness import execution
GUARD = Path(__file__).resolve().parents[1] / 'harness/tools/board_guard.py'
class TransportTests(unittest.TestCase):
def test_lease_and_streamed_evidence_and_json(self):
code = "import sys; assert sys.stdin.readline()=='lease\\n'; print('live-progress',file=sys.stderr,flush=True); print('{\"ok\":true}')"
with patch('sys.stderr', new_callable=StringIO) as err:
self.assertEqual(json.loads(guarded_run([sys.executable, '-u', '-c', code], 2)), {'ok': True})
self.assertIn('live-progress', err.getvalue())
def test_pre_auth_ssh_drop_is_replayed_but_real_failures_are_not(self):
with tempfile.TemporaryDirectory() as tmp:
marker = Path(tmp) / 'dropped'
code = (f"import pathlib,sys; m=pathlib.Path({str(marker)!r})\n"
"if not m.exists(): m.touch(); print('kex_exchange_identification: Connection closed by remote host', file=sys.stderr); raise SystemExit(255)\n"
"print('{}')")
with patch('sys.stderr', new_callable=StringIO), patch('time.sleep'):
self.assertEqual(json.loads(guarded_run([sys.executable, '-c', code], 2)), {})
marker.unlink()
self.assertEqual(json.loads(run([sys.executable, '-c', code])), {})
with self.assertRaises(SystemExit) as raised:
run([sys.executable, '-c', "import sys; print('kex_exchange_identification', file=sys.stderr); raise SystemExit(255)"])
self.assertEqual(raised.exception.code, 255) # bounded: gives up after three drops
def test_nonzero_not_hidden_by_json(self):
with self.assertRaises(SystemExit) as raised:
guarded_run([sys.executable, '-c', "print('{}'); raise SystemExit(17)"], 2)
self.assertEqual(raised.exception.code, 17)
def test_failed_native_stdout_is_retained_as_bounded_diagnostic(self):
code = "print('x'*8000); print('native-model-load-error'); raise SystemExit(17)"
with patch('sys.stderr', new_callable=StringIO) as err:
with self.assertRaises(SystemExit) as raised:
guarded_run([sys.executable, '-c', code], 2)
self.assertEqual(raised.exception.code, 17)
self.assertIn('native-model-load-error', err.getvalue())
self.assertLessEqual(len(err.getvalue()), 4000)
@unittest.skipUnless(sys.platform == 'linux', 'Linux process ownership and /proc required')
class GuardTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name)
self.directory = self.root / 'eval-test'
self.directory.mkdir()
self.sysfs = self.root / 'sys'
ratio = self.sysfs / 'class/bpu/bpu/ratio'
ratio.parent.mkdir(parents=True)
ratio.write_text('0')
self.processes = []
self.children = []
def tearDown(self):
for proc in self.processes:
if proc.poll() is None:
proc.kill()
proc.communicate(timeout=3)
for identity in self.children:
execution.signal_process(identity, signal.SIGKILL)
self.tmp.cleanup()
def start(self, timeout=3, lease=.6, command=None):
code = ("import os,signal,time; from pathlib import Path; "
"signal.signal(signal.SIGTERM,signal.SIG_IGN); "
f"Path({str(self.directory/'pid')!r}).write_text(str(os.getpid())); "
"time.sleep(30)")
# Inject only this test's sysfs; production CLI has no telemetry bypass.
wrapper = ("import sys;from pathlib import Path;"
f"sys.path.insert(0,{str(GUARD.parents[2])!r});"
"from harness.tools import board_guard,board_probe;"
f"raise SystemExit(board_guard.supervise({command or [sys.executable, '-u', '-c', code]!r},"
f"Path({str(self.directory)!r}),{timeout!r},{lease!r},.15,"
f"probe=lambda:board_probe.occupancy(Path({str(self.sysfs)!r}))))")
argv = [sys.executable, '-u', '-c', wrapper]
proc = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
self.processes.append(proc)
return proc
def child(self):
for _ in range(100):
p = self.directory / 'pid'
if p.exists():
pid = int(p.read_text())
self.children.append(execution.process_identity(pid))
return pid
time.sleep(.02)
self.fail('child did not start')
def assert_dead(self, pid):
for _ in range(100):
try:
st = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()[0]
if st == 'Z': return
except (FileNotFoundError, ProcessLookupError): return
time.sleep(.02)
self.fail(f'child {pid} survived')
def test_eof_reaps_term_ignoring_child(self):
proc = self.start(); pid = self.child()
proc.stdin.close(); proc.stdin = None
proc.communicate(timeout=3)
self.assertEqual(proc.returncode, 125); self.assert_dead(pid)
self.assertEqual(json.loads((self.directory/'guard-result.json').read_text())['reason'], 'host_disconnected')
def test_missing_lease_expires_without_eof(self):
proc = self.start(); pid = self.child()
proc.wait(timeout=3)
self.assertEqual(proc.returncode, 125); self.assert_dead(pid)
self.assertEqual(json.loads((self.directory/'guard-result.json').read_text())['reason'], 'lease_expired')
def test_deadline_reaps_even_with_valid_lease(self):
proc = self.start(timeout=.35, lease=2); pid = self.child()
proc.stdin.write(b'lease\n'); proc.stdin.flush()
proc.wait(timeout=3)
self.assertEqual(proc.returncode, 124); self.assert_dead(pid)
def test_sigterm_owner_reaps_child(self):
proc = self.start(); pid = self.child()
proc.terminate(); proc.wait(timeout=3); self.assert_dead(pid)
def test_sigkill_owner_kernel_kills_child(self):
proc = self.start(); pid = self.child()
proc.kill(); proc.wait(timeout=3); self.assert_dead(pid)
def test_exited_group_leader_does_not_leave_grandchild(self):
leaf = "import signal,time;signal.signal(signal.SIGTERM,signal.SIG_IGN);time.sleep(30)"
code = ("import subprocess,sys,time;from pathlib import Path;"
f"p=subprocess.Popen([sys.executable,'-c',{leaf!r}]);"
f"Path({str(self.directory/'pid')!r}).write_text(str(p.pid));time.sleep(.1)")
proc = self.start(command=[sys.executable, '-c', code]); pid = self.child()
proc.wait(timeout=3); self.assert_dead(pid)
def test_double_fork_cleared_env_and_session_cannot_escape_guard(self):
code = f"""import os,signal,time
r,w=os.pipe()
if os.fork():
os.close(w)
open({str(self.directory/'pid')!r},'wb').write(os.read(r,100))
os._exit(0)
os.close(r); os.environ.clear(); os.setsid()
if os.fork(): os._exit(0)
signal.signal(signal.SIGTERM,signal.SIG_IGN)
os.write(w,str(os.getpid()).encode()); os.close(w)
time.sleep(30)
"""
proc = self.start(command=[sys.executable, '-c', code]); pid = self.child()
proc.wait(timeout=3); self.assert_dead(pid)
receipt = json.loads((self.directory/'guard-result.json').read_text())
self.assertEqual(receipt['proof'], 'waitpid:ECHILD')
self.assertTrue(receipt['all_children_reaped'])
self.assertEqual(receipt['cleanup_state'], 'VERIFIED')
def test_sigkill_missing_receipt_blocks_only_while_operation_survives(self):
quarantine = self.directory.parent/'.board-inference.quarantine.json'
first = self.start(); pid = self.child()
first.kill(); first.wait(timeout=3); self.assert_dead(pid)
self.assertFalse((self.directory/'guard-result.json').exists())
token = json.loads(quarantine.read_text())['token']
survivor = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'],
env={**os.environ, 'BENCH_BOARD_OPERATION': token})
self.processes.append(survivor)
second = self.start()
second.wait(timeout=3)
self.assertEqual(second.returncode, 78)
self.assertTrue(quarantine.exists())
survivor.kill(); survivor.wait(timeout=3)
(self.directory/'pid').unlink()
third = self.start(); self.child() # admitted: the stale record no longer blocks the board
self.assertIn(b'stale_quarantine_cleared', third.stderr.read1(4096) + third.stderr.read1(4096))
def test_unmanaged_inference_is_rejected(self):
script = self.root / 'board_bench.py'
script.write_text('import time; time.sleep(30)')
unmanaged = subprocess.Popen([sys.executable, str(script)])
self.processes.append(unmanaged)
proc = self.start()
proc.wait(timeout=3)
self.assertEqual(proc.returncode, 75)
self.assertIsNone(unmanaged.poll())
def test_board_mutex_rejects_second_owner(self):
first = self.start(lease=3); pid = self.child()
second = self.start()
second.wait(timeout=2)
self.assertEqual(second.returncode, 75)
self.assertIsNone(first.poll())
def test_success_preserves_output_and_unlocks(self):
command = [sys.executable, '-c', "print('{\"ok\":true}')"]
for _ in range(2):
proc = self.start(command=command)
proc.wait(timeout=3)
out, err = proc.communicate()
self.assertEqual(proc.returncode, 0, err)
self.assertEqual(json.loads(out), {'ok': True})
def test_next_chunk_rechecks_probe_after_previous_child_reaped(self):
command = [sys.executable, '-c', "print('{\"chunk\":true}')"]
ratio = self.sysfs / 'class/bpu/bpu/ratio'
for value, expected in ((0, 0), (30, 75), (0, 0)):
ratio.write_text(str(value))
proc = self.start(command=command)
proc.wait(timeout=3)
out, err = proc.communicate()
self.assertEqual(proc.returncode, expected, err)
events = [json.loads(line) for line in err.splitlines()]
self.assertEqual(any(e['event'] == 'started' for e in events), expected == 0)
if __name__ == '__main__':
unittest.main()