data-gen-bpu/scripts/test_execution_parent_receipt.py
ming01.wang 44f643d290 fix: stop false OPERATION_STATE_UNKNOWN on finished stages
Three r20260929-06 rollouts sat in PAUSED_INFRA although their stage had
finished (status DONE/FAIL, no process left): the completion replay in
stageworker._validate_recovered_receipt raised ValueError, so the operation
stayed DISPATCHED and the execution was quarantined (one-way).

- Nested scopes: a deadline kill of the outer stage leaves the inner reaper
  with owner_lost=True although both levels reaped via waitpid(ECHILD).
  owned_processes() already accepted that when the enclosing verified receipt
  names the exact supervisor; receipt validation demanded VERIFIED. Both now
  share execution.scope_cleaned().
- Board stages killed by QUEUE_DEADLINE_EXCEEDED never held a board slot
  (phase_running fires only under the board lock) and so have no board
  receipt; that missing receipt no longer counts as unconfirmed cleanup.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 09:50:23 +08:00

122 lines
6.1 KiB
Python

"""A closed enclosing reaper can prove cleanup after a nested owner is lost."""
import copy
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from harness import execution
class ParentReceiptTests(unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory()
self.addCleanup(temp.cleanup)
self.root = Path(temp.name)
def identity(pid, ppid):
return dict(host='fixture', boot_id='fixture-boot', pid=pid, ppid=ppid,
proc_start_ticks=pid * 100)
parent = dict(execution_id='execution', epoch=1, proof='waitpid:ECHILD',
supervisor=identity(100, 1), leader=identity(101, 100))
child = dict(execution_id='execution', epoch=1, proof='waitpid:ECHILD',
supervisor=identity(102, 101), leader=identity(103, 102))
self.state = dict(processes=[], process_sets={}, process_scopes={
'parent': dict(binding=parent, receipt_path=str(self.root / 'parent.json')),
'child': dict(binding=child, receipt_path=str(self.root / 'child.json'))})
self.receipts = {
'parent': dict(binding=parent, exit_code=-15, cleanup_state='VERIFIED',
all_children_reaped=True, owner_lost=False,
members=[copy.deepcopy(child['supervisor'])]),
'child': dict(binding=child, exit_code=-15, cleanup_state='QUARANTINED',
all_children_reaped=True, owner_lost=True, supervisor_error=None)}
def inspect(self, state=None, receipts=None, status='DEAD'):
state = self.state if state is None else state
receipts = self.receipts if receipts is None else receipts
for name, value in receipts.items():
(self.root / (name + '.json')).write_text(json.dumps(value))
original = {p: p.read_bytes() for p in self.root.glob('*.json')}
with patch.object(execution, 'process_status', return_value=status):
result = execution.owned_processes(state)
self.assertEqual(original, {p: p.read_bytes() for p in original}, 'original receipts must not be rewritten')
return result
def test_verified_parent_closes_nested_owner_loss(self):
self.assertEqual(self.inspect(), [])
def test_stage_receipt_validation_uses_the_same_rule(self):
self.inspect() # write receipts
scopes = self.state['process_scopes']
self.assertTrue(execution.scope_cleaned(scopes['child'], scopes))
self.assertFalse(execution.scope_cleaned(scopes['child'], {'child': scopes['child']}))
def test_board_stage_that_never_left_the_queue_touched_no_board(self):
from types import SimpleNamespace
from harness.stageworker import Worker
worker = object.__new__(Worker)
phases = {'queued': dict(state='QUEUED', execution_started_monotonic=None),
'started': dict(state='RUNNING', execution_started_monotonic=1.0)}
worker.execution = SimpleNamespace(snapshot=lambda: {'phases': phases})
self.assertTrue(worker._board_never_admitted('queued'))
self.assertFalse(worker._board_never_admitted('started'))
self.assertFalse(worker._board_never_admitted('unknown'))
def test_dead_pids_without_enclosing_receipt_are_not_proof(self):
self.state['process_scopes'].pop('parent')
self.assertIsNone(self.inspect())
def test_incomplete_or_failed_nested_cleanup_stays_unknown(self):
for changes in ({'all_children_reaped': False}, {'owner_lost': False},
{'supervisor_error': 'cleanup deadline exceeded'}, {'exit_code': None}):
with self.subTest(changes=changes):
receipts = copy.deepcopy(self.receipts)
receipts['child'].update(changes)
self.assertIsNone(self.inspect(receipts=receipts))
def test_unverified_enclosing_cleanup_stays_unknown(self):
for changes in ({'cleanup_state': 'QUARANTINED'}, {'all_children_reaped': False},
{'owner_lost': True}, {'members': []}, {'exit_code': None}):
with self.subTest(changes=changes):
receipts = copy.deepcopy(self.receipts)
receipts['parent'].update(changes)
self.assertIsNone(self.inspect(receipts=receipts))
def test_reused_pid_or_other_host_is_not_the_enclosed_supervisor(self):
for field, value in (('pid', 999), ('proc_start_ticks', -1), ('host', 'other-host'),
('boot_id', 'other-boot')):
with self.subTest(field=field):
receipts = copy.deepcopy(self.receipts)
receipts['parent']['members'][0][field] = value
self.assertIsNone(self.inspect(receipts=receipts))
def test_parent_must_match_execution_epoch_and_immediate_parent(self):
for field, value in (('execution_id', 'other-execution'), ('epoch', 2), ('leader_pid', 999)):
with self.subTest(field=field):
state, receipts = copy.deepcopy((self.state, self.receipts))
binding = state['process_scopes']['parent']['binding']
if field == 'leader_pid':
binding['leader']['pid'] = value
else:
binding[field] = value
# The receipt still binds correctly; its relationship is wrong.
self.assertIs(receipts['parent']['binding'], binding)
self.assertIsNone(self.inspect(state=state, receipts=receipts))
def test_changed_receipt_binding_stays_unknown(self):
receipts = copy.deepcopy(self.receipts)
receipts['parent']['binding']['supervisor']['proc_start_ticks'] += 1
self.assertIsNone(self.inspect(receipts=receipts))
def test_live_recorded_process_is_not_hidden_by_receipts(self):
self.state['processes'] = [self.state['process_scopes']['child']['binding']['supervisor']]
members = self.inspect(status='ALIVE')
self.assertEqual(len(members), 1)
self.assertEqual(members[0]['pid'], 102)
if __name__ == '__main__':
unittest.main()