data-gen-bpu/scripts/test_board_cleanup_order.py
ming01.wang 2da29f8050 fix: preserve deployed recovery and board cleanup behavior
Record the deployed continuation, process reaping, recovery, and cleanup fixes with the current board registry. Add focused regression coverage and document the shared production branch and existing startup configuration.
2026-09-29 01:27:19 +08:00

110 lines
5.8 KiB
Python

import fcntl
import json
import os
import shlex
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock, patch
from harness import execution, stageworker
from harness.tools import board_perf
class CleanupOrderTests(unittest.TestCase):
def test_runtime_uses_remaining_phase_and_execution_budget(self):
for phase, total, expected in ((150, 500, 90), (500, 120, 60), (5000, 5000, 3600)):
owner = Mock(); owner.snapshot.return_value = {'deadline_monotonic': total,
'phases': {'op': {'execution_deadline_monotonic': phase}}}
receipt = {}
def invoke(command, timeout):
self.assertEqual(timeout, expected)
args = shlex.split(command[-1]); self.assertEqual(float(args[args.index('--timeout')+1]), expected)
receipt.update(operation_id=args[args.index('--operation-id')+1], guard_version=4,
cleanup_state='VERIFIED', all_children_reaped=True, proof='waitpid:ECHILD')
return '{}'
with patch.object(execution, 'from_environment', return_value=owner), \
patch.object(execution, 'current_operation', return_value='op'), \
patch.object(board_perf.time, 'monotonic', return_value=0), \
patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), \
patch.object(board_perf, 'guarded_run', side_effect=invoke), \
patch.object(board_perf.RemoteSession, 'execute', side_effect=lambda *a, **k: json.dumps(receipt)), \
patch('harness.board_identity.validate'), patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': ''}):
board_perf.RemoteSession('fixture').bench([])
def test_no_remote_launch_without_cleanup_budget(self):
owner = Mock(); owner.snapshot.return_value = {'deadline_monotonic': 59}
with patch.object(execution, 'from_environment', return_value=owner), \
patch.object(board_perf.time, 'monotonic', return_value=0), \
patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), \
patch.object(board_perf, 'guarded_run') as invoke:
with self.assertRaisesRegex(RuntimeError, 'CLEANUP_BUDGET'):
board_perf.RemoteSession('fixture').bench([])
invoke.assert_not_called()
def test_done_waits_for_watchdog_cleanup(self):
with tempfile.TemporaryDirectory() as td:
rd=Path(td); (rd/'handles/h').mkdir(parents=True)
worker=stageworker.Worker(rd)
fd=os.open(rd/'handles/h', os.O_RDONLY|os.O_DIRECTORY)
self.addCleanup(os.close, fd)
queue=rd/'stages/_queue/h'
def run(*a, **kw):
worker._write(queue/'status.json', {'state':'RUNNING'})
kw['stop_check']()
worker._write(queue/'status.json', {'state':'DONE', 'result':'OK'})
kw['stop_check']()
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['state'], 'RUNNING')
return SimpleNamespace(state='OK', cleanup_state='VERIFIED')
with patch.object(worker, '_prepare_request', return_value={}), \
patch.object(stageworker.watchdog, 'run', side_effect=run):
worker.execute('h',fd,{'stage':'07-board'})
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['state'], 'DONE')
with (queue/'operation.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX)
worker.execute('h',fd,{'stage':'07-board'})
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['state'], 'RUNNING')
def test_quarantined_execution_cannot_publish_done(self):
owner=Mock(); owner.stop_reason.return_value='BOARD_CLEANUP_UNCONFIRMED'
worker=stageworker.Worker.__new__(stageworker.Worker);worker.execution=owner
with patch.object(stageworker,'publish') as publish:
worker._publish(123, {'state':'DONE'})
publish.assert_not_called()
class MissingChildrenAbiTests(unittest.TestCase):
def test_missing_kernel_abi_falls_back_to_parent_relationship(self):
from harness.process_reaper import children
with tempfile.TemporaryDirectory() as td:
root=Path(td);(root/'10/task/10').mkdir(parents=True)
for pid,ppid in ((20,10),(21,20),(30,1)):
p=root/str(pid);p.mkdir()
(p/'stat').write_bytes(str(pid).encode()+b' (native \xff (worker)) S '+str(ppid).encode()+b' 0 0')
self.assertEqual(children(10,root),{20})
(root/'10/task/10/children').write_text('20 ')
self.assertEqual(children(10,root),{20})
def test_missing_abi_still_uses_pidfd_and_reaps_owned_process(self):
import subprocess,sys
with tempfile.TemporaryDirectory() as td:
root=Path(td)
code="""import os,sys,subprocess
from pathlib import Path
from unittest.mock import patch
from harness.process_reaper import Reaper
owner=Reaper();child=subprocess.Popen([sys.executable,'-c','import time;time.sleep(30)'])
original=Path.read_text
try:
def read(path,*a,**k):
if path.name=='children':raise FileNotFoundError(str(path))
return original(path,*a,**k)
with patch.object(Path,'read_text',read):result=owner.cleanup(child,.01,2)
assert result['all_children_reaped'] is True,result
assert child.returncode is not None and child.returncode<0,child.returncode
finally:
if child.poll() is None:child.kill();child.wait()
"""
result=subprocess.run([sys.executable,'-c',code],capture_output=True,text=True,timeout=5)
self.assertEqual(result.returncode,0,result.stderr)
if __name__ == '__main__': unittest.main()