On board kernels without /proc/<pid>/task/*/children, children() scans /proc and reads each stat; a process that exits between the listing and the read answers ESRCH (ProcessLookupError), which was not caught. 10-20 staging: the board guard crashed in its own cleanup after a 20-minute self-test, wrote no result, and the platform recorded BOARD_CLEANUP_UNCONFIRMED, quarantining a 5-hour rollout. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
149 lines
8.1 KiB
Python
149 lines
8.1 KiB
Python
import fcntl
|
|
import json
|
|
import os
|
|
import shlex
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import Mock, patch
|
|
from harness import execution, stageworker
|
|
from harness.tools import board_perf
|
|
|
|
class CleanupOrderTests(unittest.TestCase):
|
|
def test_runtime_uses_remaining_phase_and_execution_budget(self):
|
|
for phase, total, expected in ((150, 500, 90), (500, 120, 60), (5000, 5000, 3600)):
|
|
owner = Mock(); owner.snapshot.return_value = {'deadline_monotonic': total,
|
|
'phases': {'op': {'execution_deadline_monotonic': phase}}}
|
|
receipt = {}
|
|
def invoke(command, timeout):
|
|
self.assertEqual(timeout, expected)
|
|
args = shlex.split(command[-1]); self.assertEqual(float(args[args.index('--timeout')+1]), expected)
|
|
receipt.update(operation_id=args[args.index('--operation-id')+1], guard_version=4,
|
|
cleanup_state='VERIFIED', all_children_reaped=True, proof='waitpid:ECHILD')
|
|
return '{}'
|
|
with patch.object(execution, 'from_environment', return_value=owner), \
|
|
patch.object(execution, 'current_operation', return_value='op'), \
|
|
patch.object(board_perf.time, 'monotonic', return_value=0), \
|
|
patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), \
|
|
patch.object(board_perf, 'guarded_run', side_effect=invoke), \
|
|
patch.object(board_perf.RemoteSession, 'execute', side_effect=lambda *a, **k: json.dumps(receipt)), \
|
|
patch('harness.board_identity.validate'), patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': ''}):
|
|
board_perf.RemoteSession('fixture').bench([])
|
|
|
|
def test_slow_receipt_read_is_retried_not_quarantined(self):
|
|
import subprocess
|
|
sent = {}
|
|
def invoke(command, timeout):
|
|
args = shlex.split(command[-1]); sent['id'] = args[args.index('--operation-id')+1]
|
|
return '{}'
|
|
reads = iter([subprocess.TimeoutExpired('ssh', 60), None])
|
|
def read(*a, **k):
|
|
failure = next(reads)
|
|
if failure: raise failure
|
|
return json.dumps(dict(operation_id=sent['id'], guard_version=4, cleanup_state='VERIFIED',
|
|
all_children_reaped=True, proof='waitpid:ECHILD'))
|
|
with tempfile.TemporaryDirectory() as td, \
|
|
patch.object(execution, 'from_environment', return_value=None), \
|
|
patch.object(board_perf.time, 'sleep'), \
|
|
patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), \
|
|
patch.object(board_perf, 'guarded_run', side_effect=invoke), \
|
|
patch.object(board_perf.RemoteSession, 'execute', side_effect=read), \
|
|
patch('harness.board_identity.validate'), patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': td}):
|
|
session = board_perf.RemoteSession('fixture')
|
|
session.bench([])
|
|
self.assertFalse(session.cleanup_unconfirmed)
|
|
stored = json.loads(next(Path(td).glob('*.json')).read_text())
|
|
self.assertEqual((stored['cleanup_state'], stored.get('receipt_error')), ('VERIFIED', None))
|
|
|
|
def test_hung_model_load_ends_as_retryable_stall(self):
|
|
import sys
|
|
child = [sys.executable, '-c', 'import sys; print(\'{"board_progress": "load_start"}\', file=sys.stderr, flush=True); sys.stdin.read()']
|
|
with patch.object(board_perf, 'LOAD_STALL_S', .5), patch.object(execution, 'from_environment', return_value=None):
|
|
with self.assertRaises(SystemExit) as stop:
|
|
board_perf.guarded_run(child, 30)
|
|
self.assertEqual(stop.exception.code, 124)
|
|
|
|
def test_no_remote_launch_without_cleanup_budget(self):
|
|
owner = Mock(); owner.snapshot.return_value = {'deadline_monotonic': 59}
|
|
with patch.object(execution, 'from_environment', return_value=owner), \
|
|
patch.object(board_perf.time, 'monotonic', return_value=0), \
|
|
patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), \
|
|
patch.object(board_perf, 'guarded_run') as invoke:
|
|
with self.assertRaisesRegex(RuntimeError, 'CLEANUP_BUDGET'):
|
|
board_perf.RemoteSession('fixture').bench([])
|
|
invoke.assert_not_called()
|
|
|
|
def test_done_waits_for_watchdog_cleanup(self):
|
|
with tempfile.TemporaryDirectory() as td:
|
|
rd=Path(td); (rd/'handles/h').mkdir(parents=True)
|
|
worker=stageworker.Worker(rd)
|
|
fd=os.open(rd/'handles/h', os.O_RDONLY|os.O_DIRECTORY)
|
|
self.addCleanup(os.close, fd)
|
|
queue=rd/'stages/_queue/h'
|
|
def run(*a, **kw):
|
|
worker._write(queue/'status.json', {'state':'RUNNING'})
|
|
kw['stop_check']()
|
|
worker._write(queue/'status.json', {'state':'DONE', 'result':'OK'})
|
|
kw['stop_check']()
|
|
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['state'], 'RUNNING')
|
|
return SimpleNamespace(state='OK', cleanup_state='VERIFIED')
|
|
with patch.object(worker, '_prepare_request', return_value={}), \
|
|
patch.object(stageworker.watchdog, 'run', side_effect=run):
|
|
worker.execute('h',fd,{'stage':'07-board'})
|
|
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['state'], 'DONE')
|
|
with (queue/'operation.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX)
|
|
worker.execute('h',fd,{'stage':'07-board'})
|
|
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['state'], 'RUNNING')
|
|
|
|
def test_quarantined_execution_cannot_publish_done(self):
|
|
owner=Mock(); owner.stop_reason.return_value='BOARD_CLEANUP_UNCONFIRMED'
|
|
worker=stageworker.Worker.__new__(stageworker.Worker);worker.execution=owner
|
|
with patch.object(stageworker,'publish') as publish:
|
|
worker._publish(123, {'state':'DONE'})
|
|
publish.assert_not_called()
|
|
|
|
class MissingChildrenAbiTests(unittest.TestCase):
|
|
def test_missing_kernel_abi_falls_back_to_parent_relationship(self):
|
|
from harness.process_reaper import children
|
|
with tempfile.TemporaryDirectory() as td:
|
|
root=Path(td);(root/'10/task/10').mkdir(parents=True)
|
|
for pid,ppid in ((20,10),(21,20),(30,1)):
|
|
p=root/str(pid);p.mkdir()
|
|
(p/'stat').write_bytes(str(pid).encode()+b' (native \xff (worker)) S '+str(ppid).encode()+b' 0 0')
|
|
self.assertEqual(children(10,root),{20})
|
|
# A process exiting mid-scan answers ESRCH on read (10-20 staging: crashed the board guard).
|
|
original=Path.read_bytes
|
|
def read(path):
|
|
if path==root/'30/stat':raise ProcessLookupError(3,'No such process')
|
|
return original(path)
|
|
with patch.object(Path,'read_bytes',read):self.assertEqual(children(10,root),{20})
|
|
(root/'10/task/10/children').write_text('20 ')
|
|
self.assertEqual(children(10,root),{20})
|
|
|
|
def test_missing_abi_still_uses_pidfd_and_reaps_owned_process(self):
|
|
import subprocess,sys
|
|
with tempfile.TemporaryDirectory() as td:
|
|
root=Path(td)
|
|
code="""import os,sys,subprocess
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
from harness.process_reaper import Reaper
|
|
owner=Reaper();child=subprocess.Popen([sys.executable,'-c','import time;time.sleep(30)'])
|
|
original=Path.read_text
|
|
try:
|
|
def read(path,*a,**k):
|
|
if path.name=='children':raise FileNotFoundError(str(path))
|
|
return original(path,*a,**k)
|
|
with patch.object(Path,'read_text',read):result=owner.cleanup(child,.01,2)
|
|
assert result['all_children_reaped'] is True,result
|
|
assert child.returncode is not None and child.returncode<0,child.returncode
|
|
finally:
|
|
if child.poll() is None:child.kill();child.wait()
|
|
"""
|
|
result=subprocess.run([sys.executable,'-c',code],capture_output=True,text=True,timeout=5)
|
|
self.assertEqual(result.returncode,0,result.stderr)
|
|
|
|
if __name__ == '__main__': unittest.main()
|