diff --git a/harness/tools/board_perf.py b/harness/tools/board_perf.py index ca7ce54..fd77f9d 100644 --- a/harness/tools/board_perf.py +++ b/harness/tools/board_perf.py @@ -33,7 +33,21 @@ def ssh_base() -> tuple[list[str], str]: "-i", key, "-p", port or "22"], f"{user}@{host}" +PRE_AUTH_DROP = "kex_exchange_identification" # ssh closed before auth: the board never saw the command + + def run(cmd: list[str], timeout: int = 1800, *, cleanup=False) -> str: + """Replay only pre-auth drops; public board ports are flooded by scanners and drop handshakes.""" + for attempt in range(3): + try: + return _run_once(cmd, timeout, cleanup=cleanup) + except SystemExit as exc: + if attempt == 2 or exc.code not in (1, 255) or PRE_AUTH_DROP not in getattr(exc, "stderr", ""): + raise + time.sleep(2) + + +def _run_once(cmd: list[str], timeout: int, *, cleanup: bool) -> str: from harness.execution import from_environment execution = from_environment() if execution is None or cleanup: @@ -67,7 +81,21 @@ def result_json(text: str) -> dict: raise RuntimeError("board_bench produced no JSON:\n" + text[-2000:]) from e +class _PreAuthDrop(Exception): + pass + + def guarded_run(cmd: list[str], timeout: float) -> str: + for attempt in range(3): + try: + return _guarded_once(cmd, timeout) + except _PreAuthDrop: + if attempt == 2: + raise SystemExit(255) + time.sleep(2) + + +def _guarded_once(cmd: list[str], timeout: float) -> str: """Stream board evidence to the host watchdog and renew a separate stdin lease. A killed host/SSH cannot leave the native process running indefinitely: the @@ -77,7 +105,7 @@ def guarded_run(cmd: list[str], timeout: float) -> str: execution = from_environment() with execution.admission() if execution else nullcontext(): proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output = bytearray() + output, errors = bytearray(), bytearray() deadline = time.monotonic() + timeout + 15 # board deadline includes its cleanup acknowledgement next_lease = 0.0 try: @@ -100,12 +128,15 @@ def guarded_run(cmd: list[str], timeout: float) -> str: selector.unregister(key.fileobj) elif key.data == 'stderr': sys.stderr.write(data.decode(errors='replace')); sys.stderr.flush() + errors.extend(data); del errors[:-4096] else: output.extend(data) # Only the final JSON and a bounded diagnostic tail are needed. if len(output) > 1024 * 1024: del output[:-1024 * 1024] rc = proc.wait(timeout=5) + if rc == 255 and not output and PRE_AUTH_DROP.encode() in errors: + raise _PreAuthDrop() if rc: # Native runtimes also log failures to stdout; keep a bounded tail. sys.stderr.write(output.decode(errors='replace')[-4000:]); sys.stderr.flush() diff --git a/scripts/test_board_guard.py b/scripts/test_board_guard.py index 3e07b5f..f33c9a2 100644 --- a/scripts/test_board_guard.py +++ b/scripts/test_board_guard.py @@ -11,7 +11,7 @@ import unittest from unittest.mock import patch from io import StringIO -from harness.tools.board_perf import guarded_run +from harness.tools.board_perf import guarded_run, run from harness import execution GUARD = Path(__file__).resolve().parents[1] / 'harness/tools/board_guard.py' @@ -24,6 +24,20 @@ class TransportTests(unittest.TestCase): self.assertEqual(json.loads(guarded_run([sys.executable, '-u', '-c', code], 2)), {'ok': True}) self.assertIn('live-progress', err.getvalue()) + def test_pre_auth_ssh_drop_is_replayed_but_real_failures_are_not(self): + with tempfile.TemporaryDirectory() as tmp: + marker = Path(tmp) / 'dropped' + code = (f"import pathlib,sys; m=pathlib.Path({str(marker)!r})\n" + "if not m.exists(): m.touch(); print('kex_exchange_identification: Connection closed by remote host', file=sys.stderr); raise SystemExit(255)\n" + "print('{}')") + with patch('sys.stderr', new_callable=StringIO), patch('time.sleep'): + self.assertEqual(json.loads(guarded_run([sys.executable, '-c', code], 2)), {}) + marker.unlink() + self.assertEqual(json.loads(run([sys.executable, '-c', code])), {}) + with self.assertRaises(SystemExit) as raised: + run([sys.executable, '-c', "import sys; print('kex_exchange_identification', file=sys.stderr); raise SystemExit(255)"]) + self.assertEqual(raised.exception.code, 255) # bounded: gives up after three drops + def test_nonzero_not_hidden_by_json(self): with self.assertRaises(SystemExit) as raised: guarded_run([sys.executable, '-c', "print('{}'); raise SystemExit(17)"], 2)