diff --git a/harness/execution.py b/harness/execution.py index 16012b5..f079b6f 100644 --- a/harness/execution.py +++ b/harness/execution.py @@ -722,12 +722,13 @@ class Execution: finished_at=time.time()) return self._save(state, 'EXECUTION_FINISHED', outcome=outcome, reason=reason) - def reconcile(self, crash_checkpoint=None): + def reconcile(self, crash_checkpoint=None, owner_lost_at=None): """Fail closed; never replay an unknown side effect or take a live owner. crash_checkpoint: the agent's last durable session boundary, frozen by the caller after its owner (the collector) died with every process proven gone and every operation settled. The rollout then pauses - (CONTROLLER_RESTARTED) and resumes from there instead of being lost.""" + (CONTROLLER_RESTARTED) and resumes from there instead of being lost. + owner_lost_at: when the dead owner last wrote this state, read before the caller's own saves.""" with self.locked(): state = self.snapshot() if self.epoch is not None: @@ -757,9 +758,14 @@ class Execution: call['state'] = 'INTERRUPTED' # it died with its owner; the resumed agent is told so for scope in state.get('process_scopes', {}).values(): scope.setdefault('owner', state['owner']) # scopes registered before owners were recorded + # No collector ran, so none of the downtime was the agent's: hand all of it back now, uncapped (the + # paused-credit cap is for pauses that may never resume). 11-06..08: a 33 h planned stop, then + # updated_at refreshed by the restart's own settling, timed out 10 rollouts the moment they resumed. + downtime = max(0.0, time.time() - (owner_lost_at or state.get('updated_at') or time.time())) + state['deadline_monotonic'] += downtime + state['downtime_s'] = state.get('downtime_s', 0.0) + downtime state.update(execution_state='PAUSED_INFRA', stop_reason='CONTROLLER_RESTARTED', checkpoint=checkpoint, - paused_at=state.get('updated_at') or time.time(), activity='WAITING_QUOTA', - activity_details={}) + paused_at=time.time(), activity='WAITING_QUOTA', activity_details={}) return self._save(state, 'EXECUTION_RECONCILED', owner_status=owner, members=members, pending_operations=0, reconciliation='CONTROLLER_RESTARTED') if owner == 'DEAD' and members == [] and not operations: diff --git a/harness/recovery.py b/harness/recovery.py index f879b97..299df0d 100644 --- a/harness/recovery.py +++ b/harness/recovery.py @@ -210,7 +210,8 @@ def reconcile_run(run_id: str, *, attempts=None) -> dict: if record['execution_state'] == 'ACTIVE' and execution.process_status(record.get('owner', {})) == 'DEAD': from .collection_runtime import crash_checkpoint crash = crash_checkpoint(rd, owner) - record = owner.reconcile(crash_checkpoint=crash) + # record is the snapshot read before Worker settled anything: its updated_at is the dead owner's last write. + record = owner.reconcile(crash_checkpoint=crash, owner_lost_at=record.get('updated_at')) if record.get('stop_reason') == 'CONTROLLER_RESTARTED' and record['execution_state'] == 'PAUSED_INFRA': try: with db.connect(timeout_s=0) as con: diff --git a/scripts/test_execution.py b/scripts/test_execution.py index cf797c4..d4d0f9f 100644 --- a/scripts/test_execution.py +++ b/scripts/test_execution.py @@ -353,6 +353,25 @@ class ExecutionTests(unittest.TestCase): state = self.ex.resume(counted=False) self.assertEqual(state['paused_s'], 10) # at most one execution budget (policy: 10 s) + def test_collector_downtime_is_handed_back_in_full_on_restart(self): + # 11-06..08: a 33 h planned stop. The restart settled interrupted operations first, which rewrote updated_at, + # so the pause "began" at the restart, nothing was handed back and 10 rollouts were judged 24 h timeouts. + before = self.ex.snapshot() + lost_at, restart = before['updated_at'], before['updated_at'] + 1000 # 100 budgets later (policy: 10 s) + stale = json.loads(self.ex.path.read_text()); stale['updated_at'] = restart + self.ex.path.write_text(json.dumps(stale)) # what the restart's settle did + with patch.object(execution, 'process_status', return_value='DEAD'), \ + patch.object(execution, 'owned_processes', return_value=[]), \ + patch.object(execution, 'marker_processes', return_value=[]), \ + patch.object(execution.time, 'time', return_value=restart), \ + patch.object(execution.time, 'monotonic', return_value=before['deadline_monotonic'] + 990): + restarted = execution.Execution(self.rd) + state = restarted.reconcile(crash_checkpoint=self.checkpoint(), owner_lost_at=lost_at) + self.assertEqual((state['execution_state'], state['stop_reason']), ('PAUSED_INFRA', 'CONTROLLER_RESTARTED')) + self.assertEqual(state['deadline_monotonic'], before['deadline_monotonic'] + 1000) # uncapped + self.assertNotEqual(restarted.stop_reason(), 'EXECUTION_DEADLINE_EXCEEDED') + self.assertEqual(restarted.resume(automatic=False)['execution_state'], 'ACTIVE') + def test_paused_past_deadline_after_reboot_is_deadline_not_resumable(self): self.ex.pause('CONTROLLER_RESTARTED', self.checkpoint()) state = self.ex.snapshot()