fix: clear a board-side quarantine once its operation has no survivors
board_guard writes .board-inference.quarantine.json before spawning the inference child. If the guard is killed (ssh drop, SIGKILL) the file stays and every later 07-board/08-eval on that board exits 78 until a human deletes it; rdk-sandbox-40009 was blocked this way for 1.5 days. The guard already holds the board lock every admitted child inherits. It now also checks /proc for any process carrying the old operation token; with none left, the record is stale: it is removed and logged as stale_quarantine_cleared, then the normal occupancy check still applies. Survivors keep the board quarantined. Unverified-cleanup receipts now store the token so the same rule covers them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
80a8e246c0
commit
f315496d18
@ -41,6 +41,20 @@ def sync_directory(path):
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def token_holders(token: str, proc: Path = Path('/proc')) -> list[int]:
|
||||
"""Processes still carrying an admitted operation's token (children inherit it)."""
|
||||
needle = f'BENCH_BOARD_OPERATION={token}'.encode()
|
||||
found = []
|
||||
for directory in proc.iterdir():
|
||||
if directory.name.isdigit():
|
||||
try:
|
||||
if needle in (directory / 'environ').read_bytes().split(b'\0'):
|
||||
found.append(int(directory.name))
|
||||
except OSError:
|
||||
continue
|
||||
return sorted(found)
|
||||
|
||||
|
||||
def supervise(command: list[str], directory: Path, timeout_s: float,
|
||||
lease_s: float = 30, grace_s: float = 5, report_s: float = 30, *, probe=None, operation_id=None) -> int:
|
||||
directory = directory.resolve()
|
||||
@ -65,9 +79,20 @@ def supervise(command: list[str], directory: Path, timeout_s: float,
|
||||
event(reason)
|
||||
return 78
|
||||
if quarantine.exists():
|
||||
reason = 'resource_quarantined'
|
||||
event(reason)
|
||||
return 78
|
||||
try:
|
||||
previous = json.loads(quarantine.read_text())
|
||||
except ValueError:
|
||||
previous = {}
|
||||
# We hold the board lock every admitted child inherits; if no process
|
||||
# carries the old token either, nothing from that operation survives.
|
||||
holders = token_holders(previous['token']) if previous.get('token') else None
|
||||
if holders != []:
|
||||
reason = 'resource_quarantined'
|
||||
event(reason, holders=holders)
|
||||
return 78
|
||||
quarantine.unlink()
|
||||
sync_directory(quarantine.parent)
|
||||
event('stale_quarantine_cleared', previous=previous)
|
||||
admission = (probe or occupancy)()
|
||||
event('occupancy', **admission)
|
||||
if admission['status'] != 'idle':
|
||||
@ -142,7 +167,7 @@ def supervise(command: list[str], directory: Path, timeout_s: float,
|
||||
event('reaped', **receipt)
|
||||
if admitted:
|
||||
if cleanup['cleanup_state'] != 'VERIFIED':
|
||||
quarantine.write_text(json.dumps(receipt))
|
||||
quarantine.write_text(json.dumps({**receipt, 'token': token}))
|
||||
else:
|
||||
quarantine.unlink(missing_ok=True)
|
||||
sync_directory(quarantine.parent)
|
||||
|
||||
@ -153,14 +153,23 @@ time.sleep(30)
|
||||
self.assertTrue(receipt['all_children_reaped'])
|
||||
self.assertEqual(receipt['cleanup_state'], 'VERIFIED')
|
||||
|
||||
def test_sigkill_missing_receipt_blocks_next_board_admission(self):
|
||||
def test_sigkill_missing_receipt_blocks_only_while_operation_survives(self):
|
||||
quarantine = self.directory.parent/'.board-inference.quarantine.json'
|
||||
first = self.start(); pid = self.child()
|
||||
first.kill(); first.wait(timeout=3); self.assert_dead(pid)
|
||||
self.assertFalse((self.directory/'guard-result.json').exists())
|
||||
token = json.loads(quarantine.read_text())['token']
|
||||
survivor = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'],
|
||||
env={**os.environ, 'BENCH_BOARD_OPERATION': token})
|
||||
self.processes.append(survivor)
|
||||
second = self.start()
|
||||
second.wait(timeout=3)
|
||||
self.assertEqual(second.returncode, 78)
|
||||
self.assertTrue((self.directory.parent/'.board-inference.quarantine.json').exists())
|
||||
self.assertTrue(quarantine.exists())
|
||||
survivor.kill(); survivor.wait(timeout=3)
|
||||
(self.directory/'pid').unlink()
|
||||
third = self.start(); self.child() # admitted: the stale record no longer blocks the board
|
||||
self.assertIn(b'stale_quarantine_cleared', third.stderr.read1(4096) + third.stderr.read1(4096))
|
||||
|
||||
def test_unmanaged_inference_is_rejected(self):
|
||||
script = self.root / 'board_bench.py'
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user