Three r20260929-06 rollouts sat in PAUSED_INFRA although their stage had finished (status DONE/FAIL, no process left): the completion replay in stageworker._validate_recovered_receipt raised ValueError, so the operation stayed DISPATCHED and the execution was quarantined (one-way). - Nested scopes: a deadline kill of the outer stage leaves the inner reaper with owner_lost=True although both levels reaped via waitpid(ECHILD). owned_processes() already accepted that when the enclosing verified receipt names the exact supervisor; receipt validation demanded VERIFIED. Both now share execution.scope_cleaned(). - Board stages killed by QUEUE_DEADLINE_EXCEEDED never held a board slot (phase_running fires only under the board lock) and so have no board receipt; that missing receipt no longer counts as unconfirmed cleanup. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
122 lines
6.1 KiB
Python
122 lines
6.1 KiB
Python
"""A closed enclosing reaper can prove cleanup after a nested owner is lost."""
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from harness import execution
|
|
|
|
|
|
class ParentReceiptTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temp.cleanup)
|
|
self.root = Path(temp.name)
|
|
|
|
def identity(pid, ppid):
|
|
return dict(host='fixture', boot_id='fixture-boot', pid=pid, ppid=ppid,
|
|
proc_start_ticks=pid * 100)
|
|
|
|
parent = dict(execution_id='execution', epoch=1, proof='waitpid:ECHILD',
|
|
supervisor=identity(100, 1), leader=identity(101, 100))
|
|
child = dict(execution_id='execution', epoch=1, proof='waitpid:ECHILD',
|
|
supervisor=identity(102, 101), leader=identity(103, 102))
|
|
self.state = dict(processes=[], process_sets={}, process_scopes={
|
|
'parent': dict(binding=parent, receipt_path=str(self.root / 'parent.json')),
|
|
'child': dict(binding=child, receipt_path=str(self.root / 'child.json'))})
|
|
self.receipts = {
|
|
'parent': dict(binding=parent, exit_code=-15, cleanup_state='VERIFIED',
|
|
all_children_reaped=True, owner_lost=False,
|
|
members=[copy.deepcopy(child['supervisor'])]),
|
|
'child': dict(binding=child, exit_code=-15, cleanup_state='QUARANTINED',
|
|
all_children_reaped=True, owner_lost=True, supervisor_error=None)}
|
|
|
|
def inspect(self, state=None, receipts=None, status='DEAD'):
|
|
state = self.state if state is None else state
|
|
receipts = self.receipts if receipts is None else receipts
|
|
for name, value in receipts.items():
|
|
(self.root / (name + '.json')).write_text(json.dumps(value))
|
|
original = {p: p.read_bytes() for p in self.root.glob('*.json')}
|
|
with patch.object(execution, 'process_status', return_value=status):
|
|
result = execution.owned_processes(state)
|
|
self.assertEqual(original, {p: p.read_bytes() for p in original}, 'original receipts must not be rewritten')
|
|
return result
|
|
|
|
def test_verified_parent_closes_nested_owner_loss(self):
|
|
self.assertEqual(self.inspect(), [])
|
|
|
|
def test_stage_receipt_validation_uses_the_same_rule(self):
|
|
self.inspect() # write receipts
|
|
scopes = self.state['process_scopes']
|
|
self.assertTrue(execution.scope_cleaned(scopes['child'], scopes))
|
|
self.assertFalse(execution.scope_cleaned(scopes['child'], {'child': scopes['child']}))
|
|
|
|
def test_board_stage_that_never_left_the_queue_touched_no_board(self):
|
|
from types import SimpleNamespace
|
|
from harness.stageworker import Worker
|
|
worker = object.__new__(Worker)
|
|
phases = {'queued': dict(state='QUEUED', execution_started_monotonic=None),
|
|
'started': dict(state='RUNNING', execution_started_monotonic=1.0)}
|
|
worker.execution = SimpleNamespace(snapshot=lambda: {'phases': phases})
|
|
self.assertTrue(worker._board_never_admitted('queued'))
|
|
self.assertFalse(worker._board_never_admitted('started'))
|
|
self.assertFalse(worker._board_never_admitted('unknown'))
|
|
|
|
def test_dead_pids_without_enclosing_receipt_are_not_proof(self):
|
|
self.state['process_scopes'].pop('parent')
|
|
self.assertIsNone(self.inspect())
|
|
|
|
def test_incomplete_or_failed_nested_cleanup_stays_unknown(self):
|
|
for changes in ({'all_children_reaped': False}, {'owner_lost': False},
|
|
{'supervisor_error': 'cleanup deadline exceeded'}, {'exit_code': None}):
|
|
with self.subTest(changes=changes):
|
|
receipts = copy.deepcopy(self.receipts)
|
|
receipts['child'].update(changes)
|
|
self.assertIsNone(self.inspect(receipts=receipts))
|
|
|
|
def test_unverified_enclosing_cleanup_stays_unknown(self):
|
|
for changes in ({'cleanup_state': 'QUARANTINED'}, {'all_children_reaped': False},
|
|
{'owner_lost': True}, {'members': []}, {'exit_code': None}):
|
|
with self.subTest(changes=changes):
|
|
receipts = copy.deepcopy(self.receipts)
|
|
receipts['parent'].update(changes)
|
|
self.assertIsNone(self.inspect(receipts=receipts))
|
|
|
|
def test_reused_pid_or_other_host_is_not_the_enclosed_supervisor(self):
|
|
for field, value in (('pid', 999), ('proc_start_ticks', -1), ('host', 'other-host'),
|
|
('boot_id', 'other-boot')):
|
|
with self.subTest(field=field):
|
|
receipts = copy.deepcopy(self.receipts)
|
|
receipts['parent']['members'][0][field] = value
|
|
self.assertIsNone(self.inspect(receipts=receipts))
|
|
|
|
def test_parent_must_match_execution_epoch_and_immediate_parent(self):
|
|
for field, value in (('execution_id', 'other-execution'), ('epoch', 2), ('leader_pid', 999)):
|
|
with self.subTest(field=field):
|
|
state, receipts = copy.deepcopy((self.state, self.receipts))
|
|
binding = state['process_scopes']['parent']['binding']
|
|
if field == 'leader_pid':
|
|
binding['leader']['pid'] = value
|
|
else:
|
|
binding[field] = value
|
|
# The receipt still binds correctly; its relationship is wrong.
|
|
self.assertIs(receipts['parent']['binding'], binding)
|
|
self.assertIsNone(self.inspect(state=state, receipts=receipts))
|
|
|
|
def test_changed_receipt_binding_stays_unknown(self):
|
|
receipts = copy.deepcopy(self.receipts)
|
|
receipts['parent']['binding']['supervisor']['proc_start_ticks'] += 1
|
|
self.assertIsNone(self.inspect(receipts=receipts))
|
|
|
|
def test_live_recorded_process_is_not_hidden_by_receipts(self):
|
|
self.state['processes'] = [self.state['process_scopes']['child']['binding']['supervisor']]
|
|
members = self.inspect(status='ALIVE')
|
|
self.assertEqual(len(members), 1)
|
|
self.assertEqual(members[0]['pid'], 102)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|