fix: clear a board-side quarantine once its operation has no survivors

board_guard writes .board-inference.quarantine.json before spawning the
inference child. If the guard is killed (ssh drop, SIGKILL) the file stays
and every later 07-board/08-eval on that board exits 78 until a human
deletes it; rdk-sandbox-40009 was blocked this way for 1.5 days.

The guard already holds the board lock every admitted child inherits. It
now also checks /proc for any process carrying the old operation token;
with none left, the record is stale: it is removed and logged as
stale_quarantine_cleared, then the normal occupancy check still applies.
Survivors keep the board quarantined. Unverified-cleanup receipts now
store the token so the same rule covers them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ming01.wang 2026-10-04 08:44:20 +08:00
parent 80a8e246c0
commit f315496d18
2 changed files with 40 additions and 6 deletions

View File

@ -41,6 +41,20 @@ def sync_directory(path):
os.close(descriptor)
def token_holders(token: str, proc: Path = Path('/proc')) -> list[int]:
"""Processes still carrying an admitted operation's token (children inherit it)."""
needle = f'BENCH_BOARD_OPERATION={token}'.encode()
found = []
for directory in proc.iterdir():
if directory.name.isdigit():
try:
if needle in (directory / 'environ').read_bytes().split(b'\0'):
found.append(int(directory.name))
except OSError:
continue
return sorted(found)
def supervise(command: list[str], directory: Path, timeout_s: float,
lease_s: float = 30, grace_s: float = 5, report_s: float = 30, *, probe=None, operation_id=None) -> int:
directory = directory.resolve()
@ -65,9 +79,20 @@ def supervise(command: list[str], directory: Path, timeout_s: float,
event(reason)
return 78
if quarantine.exists():
reason = 'resource_quarantined'
event(reason)
return 78
try:
previous = json.loads(quarantine.read_text())
except ValueError:
previous = {}
# We hold the board lock every admitted child inherits; if no process
# carries the old token either, nothing from that operation survives.
holders = token_holders(previous['token']) if previous.get('token') else None
if holders != []:
reason = 'resource_quarantined'
event(reason, holders=holders)
return 78
quarantine.unlink()
sync_directory(quarantine.parent)
event('stale_quarantine_cleared', previous=previous)
admission = (probe or occupancy)()
event('occupancy', **admission)
if admission['status'] != 'idle':
@ -142,7 +167,7 @@ def supervise(command: list[str], directory: Path, timeout_s: float,
event('reaped', **receipt)
if admitted:
if cleanup['cleanup_state'] != 'VERIFIED':
quarantine.write_text(json.dumps(receipt))
quarantine.write_text(json.dumps({**receipt, 'token': token}))
else:
quarantine.unlink(missing_ok=True)
sync_directory(quarantine.parent)

View File

@ -153,14 +153,23 @@ time.sleep(30)
self.assertTrue(receipt['all_children_reaped'])
self.assertEqual(receipt['cleanup_state'], 'VERIFIED')
def test_sigkill_missing_receipt_blocks_next_board_admission(self):
def test_sigkill_missing_receipt_blocks_only_while_operation_survives(self):
quarantine = self.directory.parent/'.board-inference.quarantine.json'
first = self.start(); pid = self.child()
first.kill(); first.wait(timeout=3); self.assert_dead(pid)
self.assertFalse((self.directory/'guard-result.json').exists())
token = json.loads(quarantine.read_text())['token']
survivor = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'],
env={**os.environ, 'BENCH_BOARD_OPERATION': token})
self.processes.append(survivor)
second = self.start()
second.wait(timeout=3)
self.assertEqual(second.returncode, 78)
self.assertTrue((self.directory.parent/'.board-inference.quarantine.json').exists())
self.assertTrue(quarantine.exists())
survivor.kill(); survivor.wait(timeout=3)
(self.directory/'pid').unlink()
third = self.start(); self.child() # admitted: the stale record no longer blocks the board
self.assertIn(b'stale_quarantine_cleared', third.stderr.read1(4096) + third.stderr.read1(4096))
def test_unmanaged_inference_is_rejected(self):
script = self.root / 'board_bench.py'