diff --git a/harness/execution.py b/harness/execution.py index 998da65..3ada9c1 100644 --- a/harness/execution.py +++ b/harness/execution.py @@ -144,35 +144,37 @@ def owned_processes(state): if token not in scopes: return None # Legacy observations cannot prove a complete owned set. for scope in scopes.values(): - receipt = scope_receipt(scope) - if receipt and receipt['cleanup_state'] == 'VERIFIED': - continue - supervisor = scope['binding']['supervisor'] - # A nested reaper may lose its parent during cancellation even though - # both levels finish waitpid(ECHILD). The enclosing verified receipt - # must name this exact supervisor, including boot/start identity. - if (receipt and receipt.get('cleanup_state') == 'QUARANTINED' - and receipt.get('all_children_reaped') is True - and receipt.get('owner_lost') is True - and receipt.get('supervisor_error') is None): - enclosed = False - for parent in scopes.values(): - proof = scope_receipt(parent) - binding = parent['binding'] - if (proof and proof['cleanup_state'] == 'VERIFIED' - and binding.get('execution_id') == scope['binding'].get('execution_id') - and binding.get('epoch') == scope['binding'].get('epoch') - and supervisor.get('ppid') == binding['leader'].get('pid') - and process_key(supervisor) in {process_key(p) for p in proof.get('members', [])}): - enclosed = True - break - if enclosed: - continue - if process_status(supervisor) != 'ALIVE': + if not scope_cleaned(scope, scopes) and process_status(scope['binding']['supervisor']) != 'ALIVE': return None return list(members.values()) +def scope_cleaned(scope, scopes) -> bool: + """One rule for every caller that asks whether a process scope left nothing behind.""" + receipt = scope_receipt(scope) + if receipt and receipt['cleanup_state'] == 'VERIFIED': + return True + # A nested reaper may lose its parent during cancellation even though + # both levels finish waitpid(ECHILD). The enclosing verified receipt + # must name this exact supervisor, including boot/start identity. + if not (receipt and receipt.get('cleanup_state') == 'QUARANTINED' + and receipt.get('all_children_reaped') is True + and receipt.get('owner_lost') is True + and receipt.get('supervisor_error') is None): + return False + supervisor = scope['binding']['supervisor'] + for parent in scopes.values(): + proof = scope_receipt(parent) + binding = parent['binding'] + if (proof and proof['cleanup_state'] == 'VERIFIED' + and binding.get('execution_id') == scope['binding'].get('execution_id') + and binding.get('epoch') == scope['binding'].get('epoch') + and supervisor.get('ppid') == binding['leader'].get('pid') + and process_key(supervisor) in {process_key(p) for p in proof.get('members', [])}): + return True + return False + + def scope_receipt(scope): try: receipt = json.loads(Path(scope['receipt_path']).read_text()) diff --git a/harness/stageworker.py b/harness/stageworker.py index 4529831..678e514 100644 --- a/harness/stageworker.py +++ b/harness/stageworker.py @@ -182,6 +182,12 @@ class Worker: value = dict(value, **self._receipt_identity(operation)) lifecycle.atomic_json(path, value) + def _board_never_admitted(self, hid): + """phase_running fires only while the board lock is held, so a phase + that never left QUEUED (e.g. QUEUE_DEADLINE_EXCEEDED) touched no board.""" + phase = self.execution.snapshot().get('phases', {}).get(hid) if self.execution else None + return bool(phase) and phase.get('execution_started_monotonic') is None + @staticmethod def _receipt_identity(operation): return {key: operation.get(key) for key in @@ -318,7 +324,8 @@ class Worker: self._write(status_file, result) if self.execution and (name in ('07-board', '08-eval') or (name == 'terminal' and trusted.get('target') == 'board')) and result.get('result') != 'SKIPPED': receipts = list((queue / 'board-receipts').glob('*.json')) - if not receipts or any(json.loads(p.read_text()).get('cleanup_state') != 'VERIFIED' for p in receipts): + if ((not receipts and not self._board_never_admitted(hid)) + or any(json.loads(p.read_text()).get('cleanup_state') != 'VERIFIED' for p in receipts)): self.execution.quarantine('BOARD_CLEANUP_UNCONFIRMED') if self.execution and self.execution.snapshot().get('cancel_requested') and 'meta_sha256' not in result: # A completed protected receipt records computation, while the @@ -514,7 +521,7 @@ class Worker: binding = scope['binding'] receipt = lifecycle.scope_receipt(scope) if (binding.get('execution_id') != identity['execution_id'] or binding.get('epoch') != identity['epoch'] - or not receipt or receipt.get('cleanup_state') != 'VERIFIED'): + or not receipt or not lifecycle.scope_cleaned(scope, state.get('process_scopes', {}))): raise ValueError('scope cleanup unconfirmed') if lifecycle.process_key(binding['leader']) == lifecycle.process_key(executor): if result.get('result') in ('OK', 'SKIPPED') and ((receipt['exit_code'] != 0 and not completed_prerequisite) or @@ -548,7 +555,7 @@ class Worker: raise ValueError('completed artifact changed') if (name in ('07-board', '08-eval') or (name == 'terminal' and json.loads((self.rd / 'stages' / '_queue' / hid / 'request.json').read_text()).get('target') == 'board')) and result.get('result') != 'SKIPPED': receipts = list((self.rd / 'stages' / '_queue' / hid / 'board-receipts').glob('*.json')) - if not receipts: + if not receipts and not self._board_never_admitted(hid): raise ValueError('board cleanup receipt missing') for path in receipts: receipt = json.loads(path.read_text()) diff --git a/scripts/test_execution_parent_receipt.py b/scripts/test_execution_parent_receipt.py index 03c2985..e7d144e 100644 --- a/scripts/test_execution_parent_receipt.py +++ b/scripts/test_execution_parent_receipt.py @@ -47,6 +47,23 @@ class ParentReceiptTests(unittest.TestCase): def test_verified_parent_closes_nested_owner_loss(self): self.assertEqual(self.inspect(), []) + def test_stage_receipt_validation_uses_the_same_rule(self): + self.inspect() # write receipts + scopes = self.state['process_scopes'] + self.assertTrue(execution.scope_cleaned(scopes['child'], scopes)) + self.assertFalse(execution.scope_cleaned(scopes['child'], {'child': scopes['child']})) + + def test_board_stage_that_never_left_the_queue_touched_no_board(self): + from types import SimpleNamespace + from harness.stageworker import Worker + worker = object.__new__(Worker) + phases = {'queued': dict(state='QUEUED', execution_started_monotonic=None), + 'started': dict(state='RUNNING', execution_started_monotonic=1.0)} + worker.execution = SimpleNamespace(snapshot=lambda: {'phases': phases}) + self.assertTrue(worker._board_never_admitted('queued')) + self.assertFalse(worker._board_never_admitted('started')) + self.assertFalse(worker._board_never_admitted('unknown')) + def test_dead_pids_without_enclosing_receipt_are_not_proof(self): self.state['process_scopes'].pop('parent') self.assertIsNone(self.inspect())