data-gen-bpu/scripts/test_board_cleanup_order.py
bench b4c4b83a88 Process reaper: a process exiting mid-scan is not a cleanup failure
On board kernels without /proc/<pid>/task/*/children, children() scans /proc and
reads each stat; a process that exits between the listing and the read answers
ESRCH (ProcessLookupError), which was not caught. 10-20 staging: the board guard
crashed in its own cleanup after a 20-minute self-test, wrote no result, and the
platform recorded BOARD_CLEANUP_UNCONFIRMED, quarantining a 5-hour rollout.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-20 03:35:59 +08:00

149 lines
8.1 KiB
Python

import fcntl
import json
import os
import shlex
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock, patch
from harness import execution, stageworker
from harness.tools import board_perf
class CleanupOrderTests(unittest.TestCase):
def test_runtime_uses_remaining_phase_and_execution_budget(self):
for phase, total, expected in ((150, 500, 90), (500, 120, 60), (5000, 5000, 3600)):
owner = Mock(); owner.snapshot.return_value = {'deadline_monotonic': total,
'phases': {'op': {'execution_deadline_monotonic': phase}}}
receipt = {}
def invoke(command, timeout):
self.assertEqual(timeout, expected)
args = shlex.split(command[-1]); self.assertEqual(float(args[args.index('--timeout')+1]), expected)
receipt.update(operation_id=args[args.index('--operation-id')+1], guard_version=4,
cleanup_state='VERIFIED', all_children_reaped=True, proof='waitpid:ECHILD')
return '{}'
with patch.object(execution, 'from_environment', return_value=owner), \
patch.object(execution, 'current_operation', return_value='op'), \
patch.object(board_perf.time, 'monotonic', return_value=0), \
patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), \
patch.object(board_perf, 'guarded_run', side_effect=invoke), \
patch.object(board_perf.RemoteSession, 'execute', side_effect=lambda *a, **k: json.dumps(receipt)), \
patch('harness.board_identity.validate'), patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': ''}):
board_perf.RemoteSession('fixture').bench([])
def test_slow_receipt_read_is_retried_not_quarantined(self):
import subprocess
sent = {}
def invoke(command, timeout):
args = shlex.split(command[-1]); sent['id'] = args[args.index('--operation-id')+1]
return '{}'
reads = iter([subprocess.TimeoutExpired('ssh', 60), None])
def read(*a, **k):
failure = next(reads)
if failure: raise failure
return json.dumps(dict(operation_id=sent['id'], guard_version=4, cleanup_state='VERIFIED',
all_children_reaped=True, proof='waitpid:ECHILD'))
with tempfile.TemporaryDirectory() as td, \
patch.object(execution, 'from_environment', return_value=None), \
patch.object(board_perf.time, 'sleep'), \
patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), \
patch.object(board_perf, 'guarded_run', side_effect=invoke), \
patch.object(board_perf.RemoteSession, 'execute', side_effect=read), \
patch('harness.board_identity.validate'), patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td}):
session = board_perf.RemoteSession('fixture')
session.bench([])
self.assertFalse(session.cleanup_unconfirmed)
stored = json.loads(next(Path(td).glob('*.json')).read_text())
self.assertEqual((stored['cleanup_state'], stored.get('receipt_error')), ('VERIFIED', None))
def test_hung_model_load_ends_as_retryable_stall(self):
import sys
child = [sys.executable, '-c', 'import sys; print(\'{"board_progress": "load_start"}\', file=sys.stderr, flush=True); sys.stdin.read()']
with patch.object(board_perf, 'LOAD_STALL_S', .5), patch.object(execution, 'from_environment', return_value=None):
with self.assertRaises(SystemExit) as stop:
board_perf.guarded_run(child, 30)
self.assertEqual(stop.exception.code, 124)
def test_no_remote_launch_without_cleanup_budget(self):
owner = Mock(); owner.snapshot.return_value = {'deadline_monotonic': 59}
with patch.object(execution, 'from_environment', return_value=owner), \
patch.object(board_perf.time, 'monotonic', return_value=0), \
patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), \
patch.object(board_perf, 'guarded_run') as invoke:
with self.assertRaisesRegex(RuntimeError, 'CLEANUP_BUDGET'):
board_perf.RemoteSession('fixture').bench([])
invoke.assert_not_called()
def test_done_waits_for_watchdog_cleanup(self):
with tempfile.TemporaryDirectory() as td:
rd=Path(td); (rd/'handles/h').mkdir(parents=True)
worker=stageworker.Worker(rd)
fd=os.open(rd/'handles/h', os.O_RDONLY|os.O_DIRECTORY)
self.addCleanup(os.close, fd)
queue=rd/'stages/_queue/h'
def run(*a, **kw):
worker._write(queue/'status.json', {'state':'RUNNING'})
kw['stop_check']()
worker._write(queue/'status.json', {'state':'DONE', 'result':'OK'})
kw['stop_check']()
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['state'], 'RUNNING')
return SimpleNamespace(state='OK', cleanup_state='VERIFIED')
with patch.object(worker, '_prepare_request', return_value={}), \
patch.object(stageworker.watchdog, 'run', side_effect=run):
worker.execute('h',fd,{'stage':'07-board'})
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['state'], 'DONE')
with (queue/'operation.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX)
worker.execute('h',fd,{'stage':'07-board'})
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['state'], 'RUNNING')
def test_quarantined_execution_cannot_publish_done(self):
owner=Mock(); owner.stop_reason.return_value='BOARD_CLEANUP_UNCONFIRMED'
worker=stageworker.Worker.__new__(stageworker.Worker);worker.execution=owner
with patch.object(stageworker,'publish') as publish:
worker._publish(123, {'state':'DONE'})
publish.assert_not_called()
class MissingChildrenAbiTests(unittest.TestCase):
def test_missing_kernel_abi_falls_back_to_parent_relationship(self):
from harness.process_reaper import children
with tempfile.TemporaryDirectory() as td:
root=Path(td);(root/'10/task/10').mkdir(parents=True)
for pid,ppid in ((20,10),(21,20),(30,1)):
p=root/str(pid);p.mkdir()
(p/'stat').write_bytes(str(pid).encode()+b' (native \xff (worker)) S '+str(ppid).encode()+b' 0 0')
self.assertEqual(children(10,root),{20})
# A process exiting mid-scan answers ESRCH on read (10-20 staging: crashed the board guard).
original=Path.read_bytes
def read(path):
if path==root/'30/stat':raise ProcessLookupError(3,'No such process')
return original(path)
with patch.object(Path,'read_bytes',read):self.assertEqual(children(10,root),{20})
(root/'10/task/10/children').write_text('20 ')
self.assertEqual(children(10,root),{20})
def test_missing_abi_still_uses_pidfd_and_reaps_owned_process(self):
import subprocess,sys
with tempfile.TemporaryDirectory() as td:
root=Path(td)
code="""import os,sys,subprocess
from pathlib import Path
from unittest.mock import patch
from harness.process_reaper import Reaper
owner=Reaper();child=subprocess.Popen([sys.executable,'-c','import time;time.sleep(30)'])
original=Path.read_text
try:
def read(path,*a,**k):
if path.name=='children':raise FileNotFoundError(str(path))
return original(path,*a,**k)
with patch.object(Path,'read_text',read):result=owner.cleanup(child,.01,2)
assert result['all_children_reaped'] is True,result
assert child.returncode is not None and child.returncode<0,child.returncode
finally:
if child.poll() is None:child.kill();child.wait()
"""
result=subprocess.run([sys.executable,'-c',code],capture_output=True,text=True,timeout=5)
self.assertEqual(result.returncode,0,result.stderr)
if __name__ == '__main__': unittest.main()