fix: replay board ssh calls dropped before authentication

Board ssh ports are public and constantly hold 15-35 unauthenticated
scanner connections; whenever that exceeds sshd MaxStartups, our handshake
is closed with kex_exchange_identification. Only upload steps retried, so
a single drop failed the bench call itself (exit 255) or lost the cleanup
receipt fetch, which then quarantined the board.

run() and guarded_run() now replay a call up to twice when ssh reports
kex_exchange_identification: the connection closed before auth, so the
board never saw the command and replay cannot duplicate work. Any other
failure is returned unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ming01.wang 2026-10-04 09:18:48 +08:00
parent f315496d18
commit f61d2efebb
2 changed files with 47 additions and 2 deletions

View File

@ -33,7 +33,21 @@ def ssh_base() -> tuple[list[str], str]:
"-i", key, "-p", port or "22"], f"{user}@{host}"
PRE_AUTH_DROP = "kex_exchange_identification" # ssh closed before auth: the board never saw the command
def run(cmd: list[str], timeout: int = 1800, *, cleanup=False) -> str:
"""Replay only pre-auth drops; public board ports are flooded by scanners and drop handshakes."""
for attempt in range(3):
try:
return _run_once(cmd, timeout, cleanup=cleanup)
except SystemExit as exc:
if attempt == 2 or exc.code not in (1, 255) or PRE_AUTH_DROP not in getattr(exc, "stderr", ""):
raise
time.sleep(2)
def _run_once(cmd: list[str], timeout: int, *, cleanup: bool) -> str:
from harness.execution import from_environment
execution = from_environment()
if execution is None or cleanup:
@ -67,7 +81,21 @@ def result_json(text: str) -> dict:
raise RuntimeError("board_bench produced no JSON:\n" + text[-2000:]) from e
class _PreAuthDrop(Exception):
pass
def guarded_run(cmd: list[str], timeout: float) -> str:
for attempt in range(3):
try:
return _guarded_once(cmd, timeout)
except _PreAuthDrop:
if attempt == 2:
raise SystemExit(255)
time.sleep(2)
def _guarded_once(cmd: list[str], timeout: float) -> str:
"""Stream board evidence to the host watchdog and renew a separate stdin lease.
A killed host/SSH cannot leave the native process running indefinitely: the
@ -77,7 +105,7 @@ def guarded_run(cmd: list[str], timeout: float) -> str:
execution = from_environment()
with execution.admission() if execution else nullcontext():
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output = bytearray()
output, errors = bytearray(), bytearray()
deadline = time.monotonic() + timeout + 15 # board deadline includes its cleanup acknowledgement
next_lease = 0.0
try:
@ -100,12 +128,15 @@ def guarded_run(cmd: list[str], timeout: float) -> str:
selector.unregister(key.fileobj)
elif key.data == 'stderr':
sys.stderr.write(data.decode(errors='replace')); sys.stderr.flush()
errors.extend(data); del errors[:-4096]
else:
output.extend(data)
# Only the final JSON and a bounded diagnostic tail are needed.
if len(output) > 1024 * 1024:
del output[:-1024 * 1024]
rc = proc.wait(timeout=5)
if rc == 255 and not output and PRE_AUTH_DROP.encode() in errors:
raise _PreAuthDrop()
if rc:
# Native runtimes also log failures to stdout; keep a bounded tail.
sys.stderr.write(output.decode(errors='replace')[-4000:]); sys.stderr.flush()

View File

@ -11,7 +11,7 @@ import unittest
from unittest.mock import patch
from io import StringIO
from harness.tools.board_perf import guarded_run
from harness.tools.board_perf import guarded_run, run
from harness import execution
GUARD = Path(__file__).resolve().parents[1] / 'harness/tools/board_guard.py'
@ -24,6 +24,20 @@ class TransportTests(unittest.TestCase):
self.assertEqual(json.loads(guarded_run([sys.executable, '-u', '-c', code], 2)), {'ok': True})
self.assertIn('live-progress', err.getvalue())
def test_pre_auth_ssh_drop_is_replayed_but_real_failures_are_not(self):
with tempfile.TemporaryDirectory() as tmp:
marker = Path(tmp) / 'dropped'
code = (f"import pathlib,sys; m=pathlib.Path({str(marker)!r})\n"
"if not m.exists(): m.touch(); print('kex_exchange_identification: Connection closed by remote host', file=sys.stderr); raise SystemExit(255)\n"
"print('{}')")
with patch('sys.stderr', new_callable=StringIO), patch('time.sleep'):
self.assertEqual(json.loads(guarded_run([sys.executable, '-c', code], 2)), {})
marker.unlink()
self.assertEqual(json.loads(run([sys.executable, '-c', code])), {})
with self.assertRaises(SystemExit) as raised:
run([sys.executable, '-c', "import sys; print('kex_exchange_identification', file=sys.stderr); raise SystemExit(255)"])
self.assertEqual(raised.exception.code, 255) # bounded: gives up after three drops
def test_nonzero_not_hidden_by_json(self):
with self.assertRaises(SystemExit) as raised:
guarded_run([sys.executable, '-c', "print('{}'); raise SystemExit(17)"], 2)