fix: stop false OPERATION_STATE_UNKNOWN on finished stages
Three r20260929-06 rollouts sat in PAUSED_INFRA although their stage had finished (status DONE/FAIL, no process left): the completion replay in stageworker._validate_recovered_receipt raised ValueError, so the operation stayed DISPATCHED and the execution was quarantined (one-way). - Nested scopes: a deadline kill of the outer stage leaves the inner reaper with owner_lost=True although both levels reaped via waitpid(ECHILD). owned_processes() already accepted that when the enclosing verified receipt names the exact supervisor; receipt validation demanded VERIFIED. Both now share execution.scope_cleaned(). - Board stages killed by QUEUE_DEADLINE_EXCEEDED never held a board slot (phase_running fires only under the board lock) and so have no board receipt; that missing receipt no longer counts as unconfirmed cleanup. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f61d2efebb
commit
44f643d290
@ -144,35 +144,37 @@ def owned_processes(state):
|
||||
if token not in scopes:
|
||||
return None # Legacy observations cannot prove a complete owned set.
|
||||
for scope in scopes.values():
|
||||
receipt = scope_receipt(scope)
|
||||
if receipt and receipt['cleanup_state'] == 'VERIFIED':
|
||||
continue
|
||||
supervisor = scope['binding']['supervisor']
|
||||
# A nested reaper may lose its parent during cancellation even though
|
||||
# both levels finish waitpid(ECHILD). The enclosing verified receipt
|
||||
# must name this exact supervisor, including boot/start identity.
|
||||
if (receipt and receipt.get('cleanup_state') == 'QUARANTINED'
|
||||
and receipt.get('all_children_reaped') is True
|
||||
and receipt.get('owner_lost') is True
|
||||
and receipt.get('supervisor_error') is None):
|
||||
enclosed = False
|
||||
for parent in scopes.values():
|
||||
proof = scope_receipt(parent)
|
||||
binding = parent['binding']
|
||||
if (proof and proof['cleanup_state'] == 'VERIFIED'
|
||||
and binding.get('execution_id') == scope['binding'].get('execution_id')
|
||||
and binding.get('epoch') == scope['binding'].get('epoch')
|
||||
and supervisor.get('ppid') == binding['leader'].get('pid')
|
||||
and process_key(supervisor) in {process_key(p) for p in proof.get('members', [])}):
|
||||
enclosed = True
|
||||
break
|
||||
if enclosed:
|
||||
continue
|
||||
if process_status(supervisor) != 'ALIVE':
|
||||
if not scope_cleaned(scope, scopes) and process_status(scope['binding']['supervisor']) != 'ALIVE':
|
||||
return None
|
||||
return list(members.values())
|
||||
|
||||
|
||||
def scope_cleaned(scope, scopes) -> bool:
|
||||
"""One rule for every caller that asks whether a process scope left nothing behind."""
|
||||
receipt = scope_receipt(scope)
|
||||
if receipt and receipt['cleanup_state'] == 'VERIFIED':
|
||||
return True
|
||||
# A nested reaper may lose its parent during cancellation even though
|
||||
# both levels finish waitpid(ECHILD). The enclosing verified receipt
|
||||
# must name this exact supervisor, including boot/start identity.
|
||||
if not (receipt and receipt.get('cleanup_state') == 'QUARANTINED'
|
||||
and receipt.get('all_children_reaped') is True
|
||||
and receipt.get('owner_lost') is True
|
||||
and receipt.get('supervisor_error') is None):
|
||||
return False
|
||||
supervisor = scope['binding']['supervisor']
|
||||
for parent in scopes.values():
|
||||
proof = scope_receipt(parent)
|
||||
binding = parent['binding']
|
||||
if (proof and proof['cleanup_state'] == 'VERIFIED'
|
||||
and binding.get('execution_id') == scope['binding'].get('execution_id')
|
||||
and binding.get('epoch') == scope['binding'].get('epoch')
|
||||
and supervisor.get('ppid') == binding['leader'].get('pid')
|
||||
and process_key(supervisor) in {process_key(p) for p in proof.get('members', [])}):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def scope_receipt(scope):
|
||||
try:
|
||||
receipt = json.loads(Path(scope['receipt_path']).read_text())
|
||||
|
||||
@ -182,6 +182,12 @@ class Worker:
|
||||
value = dict(value, **self._receipt_identity(operation))
|
||||
lifecycle.atomic_json(path, value)
|
||||
|
||||
def _board_never_admitted(self, hid):
|
||||
"""phase_running fires only while the board lock is held, so a phase
|
||||
that never left QUEUED (e.g. QUEUE_DEADLINE_EXCEEDED) touched no board."""
|
||||
phase = self.execution.snapshot().get('phases', {}).get(hid) if self.execution else None
|
||||
return bool(phase) and phase.get('execution_started_monotonic') is None
|
||||
|
||||
@staticmethod
|
||||
def _receipt_identity(operation):
|
||||
return {key: operation.get(key) for key in
|
||||
@ -318,7 +324,8 @@ class Worker:
|
||||
self._write(status_file, result)
|
||||
if self.execution and (name in ('07-board', '08-eval') or (name == 'terminal' and trusted.get('target') == 'board')) and result.get('result') != 'SKIPPED':
|
||||
receipts = list((queue / 'board-receipts').glob('*.json'))
|
||||
if not receipts or any(json.loads(p.read_text()).get('cleanup_state') != 'VERIFIED' for p in receipts):
|
||||
if ((not receipts and not self._board_never_admitted(hid))
|
||||
or any(json.loads(p.read_text()).get('cleanup_state') != 'VERIFIED' for p in receipts)):
|
||||
self.execution.quarantine('BOARD_CLEANUP_UNCONFIRMED')
|
||||
if self.execution and self.execution.snapshot().get('cancel_requested') and 'meta_sha256' not in result:
|
||||
# A completed protected receipt records computation, while the
|
||||
@ -514,7 +521,7 @@ class Worker:
|
||||
binding = scope['binding']
|
||||
receipt = lifecycle.scope_receipt(scope)
|
||||
if (binding.get('execution_id') != identity['execution_id'] or binding.get('epoch') != identity['epoch']
|
||||
or not receipt or receipt.get('cleanup_state') != 'VERIFIED'):
|
||||
or not receipt or not lifecycle.scope_cleaned(scope, state.get('process_scopes', {}))):
|
||||
raise ValueError('scope cleanup unconfirmed')
|
||||
if lifecycle.process_key(binding['leader']) == lifecycle.process_key(executor):
|
||||
if result.get('result') in ('OK', 'SKIPPED') and ((receipt['exit_code'] != 0 and not completed_prerequisite) or
|
||||
@ -548,7 +555,7 @@ class Worker:
|
||||
raise ValueError('completed artifact changed')
|
||||
if (name in ('07-board', '08-eval') or (name == 'terminal' and json.loads((self.rd / 'stages' / '_queue' / hid / 'request.json').read_text()).get('target') == 'board')) and result.get('result') != 'SKIPPED':
|
||||
receipts = list((self.rd / 'stages' / '_queue' / hid / 'board-receipts').glob('*.json'))
|
||||
if not receipts:
|
||||
if not receipts and not self._board_never_admitted(hid):
|
||||
raise ValueError('board cleanup receipt missing')
|
||||
for path in receipts:
|
||||
receipt = json.loads(path.read_text())
|
||||
|
||||
@ -47,6 +47,23 @@ class ParentReceiptTests(unittest.TestCase):
|
||||
def test_verified_parent_closes_nested_owner_loss(self):
|
||||
self.assertEqual(self.inspect(), [])
|
||||
|
||||
def test_stage_receipt_validation_uses_the_same_rule(self):
|
||||
self.inspect() # write receipts
|
||||
scopes = self.state['process_scopes']
|
||||
self.assertTrue(execution.scope_cleaned(scopes['child'], scopes))
|
||||
self.assertFalse(execution.scope_cleaned(scopes['child'], {'child': scopes['child']}))
|
||||
|
||||
def test_board_stage_that_never_left_the_queue_touched_no_board(self):
|
||||
from types import SimpleNamespace
|
||||
from harness.stageworker import Worker
|
||||
worker = object.__new__(Worker)
|
||||
phases = {'queued': dict(state='QUEUED', execution_started_monotonic=None),
|
||||
'started': dict(state='RUNNING', execution_started_monotonic=1.0)}
|
||||
worker.execution = SimpleNamespace(snapshot=lambda: {'phases': phases})
|
||||
self.assertTrue(worker._board_never_admitted('queued'))
|
||||
self.assertFalse(worker._board_never_admitted('started'))
|
||||
self.assertFalse(worker._board_never_admitted('unknown'))
|
||||
|
||||
def test_dead_pids_without_enclosing_receipt_are_not_proof(self):
|
||||
self.state['process_scopes'].pop('parent')
|
||||
self.assertIsNone(self.inspect())
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user