Record the deployed continuation, process reaping, recovery, and cleanup fixes with the current board registry. Add focused regression coverage and document the shared production branch and existing startup configuration.
110 lines
5.8 KiB
Python
110 lines
5.8 KiB
Python
import fcntl
|
|
import json
|
|
import os
|
|
import shlex
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from unittest.mock import Mock, patch
|
|
from harness import execution, stageworker
|
|
from harness.tools import board_perf
|
|
|
|
class CleanupOrderTests(unittest.TestCase):
|
|
def test_runtime_uses_remaining_phase_and_execution_budget(self):
|
|
for phase, total, expected in ((150, 500, 90), (500, 120, 60), (5000, 5000, 3600)):
|
|
owner = Mock(); owner.snapshot.return_value = {'deadline_monotonic': total,
|
|
'phases': {'op': {'execution_deadline_monotonic': phase}}}
|
|
receipt = {}
|
|
def invoke(command, timeout):
|
|
self.assertEqual(timeout, expected)
|
|
args = shlex.split(command[-1]); self.assertEqual(float(args[args.index('--timeout')+1]), expected)
|
|
receipt.update(operation_id=args[args.index('--operation-id')+1], guard_version=4,
|
|
cleanup_state='VERIFIED', all_children_reaped=True, proof='waitpid:ECHILD')
|
|
return '{}'
|
|
with patch.object(execution, 'from_environment', return_value=owner), \
|
|
patch.object(execution, 'current_operation', return_value='op'), \
|
|
patch.object(board_perf.time, 'monotonic', return_value=0), \
|
|
patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), \
|
|
patch.object(board_perf, 'guarded_run', side_effect=invoke), \
|
|
patch.object(board_perf.RemoteSession, 'execute', side_effect=lambda *a, **k: json.dumps(receipt)), \
|
|
patch('harness.board_identity.validate'), patch.dict(os.environ, {'BENCH_BOARD_RECEIPT_DIR': ''}):
|
|
board_perf.RemoteSession('fixture').bench([])
|
|
|
|
def test_no_remote_launch_without_cleanup_budget(self):
|
|
owner = Mock(); owner.snapshot.return_value = {'deadline_monotonic': 59}
|
|
with patch.object(execution, 'from_environment', return_value=owner), \
|
|
patch.object(board_perf.time, 'monotonic', return_value=0), \
|
|
patch.object(board_perf, 'ssh_base', return_value=([], 'fixture')), \
|
|
patch.object(board_perf, 'guarded_run') as invoke:
|
|
with self.assertRaisesRegex(RuntimeError, 'CLEANUP_BUDGET'):
|
|
board_perf.RemoteSession('fixture').bench([])
|
|
invoke.assert_not_called()
|
|
|
|
def test_done_waits_for_watchdog_cleanup(self):
|
|
with tempfile.TemporaryDirectory() as td:
|
|
rd=Path(td); (rd/'handles/h').mkdir(parents=True)
|
|
worker=stageworker.Worker(rd)
|
|
fd=os.open(rd/'handles/h', os.O_RDONLY|os.O_DIRECTORY)
|
|
self.addCleanup(os.close, fd)
|
|
queue=rd/'stages/_queue/h'
|
|
def run(*a, **kw):
|
|
worker._write(queue/'status.json', {'state':'RUNNING'})
|
|
kw['stop_check']()
|
|
worker._write(queue/'status.json', {'state':'DONE', 'result':'OK'})
|
|
kw['stop_check']()
|
|
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['state'], 'RUNNING')
|
|
return SimpleNamespace(state='OK', cleanup_state='VERIFIED')
|
|
with patch.object(worker, '_prepare_request', return_value={}), \
|
|
patch.object(stageworker.watchdog, 'run', side_effect=run):
|
|
worker.execute('h',fd,{'stage':'07-board'})
|
|
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['state'], 'DONE')
|
|
with (queue/'operation.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX)
|
|
worker.execute('h',fd,{'stage':'07-board'})
|
|
self.assertEqual(json.loads((rd/'handles/h/status.json').read_text())['state'], 'RUNNING')
|
|
|
|
def test_quarantined_execution_cannot_publish_done(self):
|
|
owner=Mock(); owner.stop_reason.return_value='BOARD_CLEANUP_UNCONFIRMED'
|
|
worker=stageworker.Worker.__new__(stageworker.Worker);worker.execution=owner
|
|
with patch.object(stageworker,'publish') as publish:
|
|
worker._publish(123, {'state':'DONE'})
|
|
publish.assert_not_called()
|
|
|
|
class MissingChildrenAbiTests(unittest.TestCase):
|
|
def test_missing_kernel_abi_falls_back_to_parent_relationship(self):
|
|
from harness.process_reaper import children
|
|
with tempfile.TemporaryDirectory() as td:
|
|
root=Path(td);(root/'10/task/10').mkdir(parents=True)
|
|
for pid,ppid in ((20,10),(21,20),(30,1)):
|
|
p=root/str(pid);p.mkdir()
|
|
(p/'stat').write_bytes(str(pid).encode()+b' (native \xff (worker)) S '+str(ppid).encode()+b' 0 0')
|
|
self.assertEqual(children(10,root),{20})
|
|
(root/'10/task/10/children').write_text('20 ')
|
|
self.assertEqual(children(10,root),{20})
|
|
|
|
def test_missing_abi_still_uses_pidfd_and_reaps_owned_process(self):
|
|
import subprocess,sys
|
|
with tempfile.TemporaryDirectory() as td:
|
|
root=Path(td)
|
|
code="""import os,sys,subprocess
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
from harness.process_reaper import Reaper
|
|
owner=Reaper();child=subprocess.Popen([sys.executable,'-c','import time;time.sleep(30)'])
|
|
original=Path.read_text
|
|
try:
|
|
def read(path,*a,**k):
|
|
if path.name=='children':raise FileNotFoundError(str(path))
|
|
return original(path,*a,**k)
|
|
with patch.object(Path,'read_text',read):result=owner.cleanup(child,.01,2)
|
|
assert result['all_children_reaped'] is True,result
|
|
assert child.returncode is not None and child.returncode<0,child.returncode
|
|
finally:
|
|
if child.poll() is None:child.kill();child.wait()
|
|
"""
|
|
result=subprocess.run([sys.executable,'-c',code],capture_output=True,text=True,timeout=5)
|
|
self.assertEqual(result.returncode,0,result.stderr)
|
|
|
|
if __name__ == '__main__': unittest.main()
|