fix: stop false OPERATION_STATE_UNKNOWN on finished stages

Three r20260929-06 rollouts sat in PAUSED_INFRA although their stage had
finished (status DONE/FAIL, no process left): the completion replay in
stageworker._validate_recovered_receipt raised ValueError, so the operation
stayed DISPATCHED and the execution was quarantined (one-way).

- Nested scopes: a deadline kill of the outer stage leaves the inner reaper
  with owner_lost=True although both levels reaped via waitpid(ECHILD).
  owned_processes() already accepted that when the enclosing verified receipt
  names the exact supervisor; receipt validation demanded VERIFIED. Both now
  share execution.scope_cleaned().
- Board stages killed by QUEUE_DEADLINE_EXCEEDED never held a board slot
  (phase_running fires only under the board lock) and so have no board
  receipt; that missing receipt no longer counts as unconfirmed cleanup.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ming01.wang 2026-10-04 09:50:23 +08:00
parent f61d2efebb
commit 44f643d290
3 changed files with 54 additions and 28 deletions

View File

@ -144,35 +144,37 @@ def owned_processes(state):
if token not in scopes:
return None # Legacy observations cannot prove a complete owned set.
for scope in scopes.values():
receipt = scope_receipt(scope)
if receipt and receipt['cleanup_state'] == 'VERIFIED':
continue
supervisor = scope['binding']['supervisor']
# A nested reaper may lose its parent during cancellation even though
# both levels finish waitpid(ECHILD). The enclosing verified receipt
# must name this exact supervisor, including boot/start identity.
if (receipt and receipt.get('cleanup_state') == 'QUARANTINED'
and receipt.get('all_children_reaped') is True
and receipt.get('owner_lost') is True
and receipt.get('supervisor_error') is None):
enclosed = False
for parent in scopes.values():
proof = scope_receipt(parent)
binding = parent['binding']
if (proof and proof['cleanup_state'] == 'VERIFIED'
and binding.get('execution_id') == scope['binding'].get('execution_id')
and binding.get('epoch') == scope['binding'].get('epoch')
and supervisor.get('ppid') == binding['leader'].get('pid')
and process_key(supervisor) in {process_key(p) for p in proof.get('members', [])}):
enclosed = True
break
if enclosed:
continue
if process_status(supervisor) != 'ALIVE':
if not scope_cleaned(scope, scopes) and process_status(scope['binding']['supervisor']) != 'ALIVE':
return None
return list(members.values())
def scope_cleaned(scope, scopes) -> bool:
"""One rule for every caller that asks whether a process scope left nothing behind."""
receipt = scope_receipt(scope)
if receipt and receipt['cleanup_state'] == 'VERIFIED':
return True
# A nested reaper may lose its parent during cancellation even though
# both levels finish waitpid(ECHILD). The enclosing verified receipt
# must name this exact supervisor, including boot/start identity.
if not (receipt and receipt.get('cleanup_state') == 'QUARANTINED'
and receipt.get('all_children_reaped') is True
and receipt.get('owner_lost') is True
and receipt.get('supervisor_error') is None):
return False
supervisor = scope['binding']['supervisor']
for parent in scopes.values():
proof = scope_receipt(parent)
binding = parent['binding']
if (proof and proof['cleanup_state'] == 'VERIFIED'
and binding.get('execution_id') == scope['binding'].get('execution_id')
and binding.get('epoch') == scope['binding'].get('epoch')
and supervisor.get('ppid') == binding['leader'].get('pid')
and process_key(supervisor) in {process_key(p) for p in proof.get('members', [])}):
return True
return False
def scope_receipt(scope):
try:
receipt = json.loads(Path(scope['receipt_path']).read_text())

View File

@ -182,6 +182,12 @@ class Worker:
value = dict(value, **self._receipt_identity(operation))
lifecycle.atomic_json(path, value)
def _board_never_admitted(self, hid):
"""phase_running fires only while the board lock is held, so a phase
that never left QUEUED (e.g. QUEUE_DEADLINE_EXCEEDED) touched no board."""
phase = self.execution.snapshot().get('phases', {}).get(hid) if self.execution else None
return bool(phase) and phase.get('execution_started_monotonic') is None
@staticmethod
def _receipt_identity(operation):
return {key: operation.get(key) for key in
@ -318,7 +324,8 @@ class Worker:
self._write(status_file, result)
if self.execution and (name in ('07-board', '08-eval') or (name == 'terminal' and trusted.get('target') == 'board')) and result.get('result') != 'SKIPPED':
receipts = list((queue / 'board-receipts').glob('*.json'))
if not receipts or any(json.loads(p.read_text()).get('cleanup_state') != 'VERIFIED' for p in receipts):
if ((not receipts and not self._board_never_admitted(hid))
or any(json.loads(p.read_text()).get('cleanup_state') != 'VERIFIED' for p in receipts)):
self.execution.quarantine('BOARD_CLEANUP_UNCONFIRMED')
if self.execution and self.execution.snapshot().get('cancel_requested') and 'meta_sha256' not in result:
# A completed protected receipt records computation, while the
@ -514,7 +521,7 @@ class Worker:
binding = scope['binding']
receipt = lifecycle.scope_receipt(scope)
if (binding.get('execution_id') != identity['execution_id'] or binding.get('epoch') != identity['epoch']
or not receipt or receipt.get('cleanup_state') != 'VERIFIED'):
or not receipt or not lifecycle.scope_cleaned(scope, state.get('process_scopes', {}))):
raise ValueError('scope cleanup unconfirmed')
if lifecycle.process_key(binding['leader']) == lifecycle.process_key(executor):
if result.get('result') in ('OK', 'SKIPPED') and ((receipt['exit_code'] != 0 and not completed_prerequisite) or
@ -548,7 +555,7 @@ class Worker:
raise ValueError('completed artifact changed')
if (name in ('07-board', '08-eval') or (name == 'terminal' and json.loads((self.rd / 'stages' / '_queue' / hid / 'request.json').read_text()).get('target') == 'board')) and result.get('result') != 'SKIPPED':
receipts = list((self.rd / 'stages' / '_queue' / hid / 'board-receipts').glob('*.json'))
if not receipts:
if not receipts and not self._board_never_admitted(hid):
raise ValueError('board cleanup receipt missing')
for path in receipts:
receipt = json.loads(path.read_text())

View File

@ -47,6 +47,23 @@ class ParentReceiptTests(unittest.TestCase):
def test_verified_parent_closes_nested_owner_loss(self):
self.assertEqual(self.inspect(), [])
def test_stage_receipt_validation_uses_the_same_rule(self):
self.inspect() # write receipts
scopes = self.state['process_scopes']
self.assertTrue(execution.scope_cleaned(scopes['child'], scopes))
self.assertFalse(execution.scope_cleaned(scopes['child'], {'child': scopes['child']}))
def test_board_stage_that_never_left_the_queue_touched_no_board(self):
from types import SimpleNamespace
from harness.stageworker import Worker
worker = object.__new__(Worker)
phases = {'queued': dict(state='QUEUED', execution_started_monotonic=None),
'started': dict(state='RUNNING', execution_started_monotonic=1.0)}
worker.execution = SimpleNamespace(snapshot=lambda: {'phases': phases})
self.assertTrue(worker._board_never_admitted('queued'))
self.assertFalse(worker._board_never_admitted('started'))
self.assertFalse(worker._board_never_admitted('unknown'))
def test_dead_pids_without_enclosing_receipt_are_not_proof(self):
self.state['process_scopes'].pop('parent')
self.assertIsNone(self.inspect())