recovery: hand a stopped collector's downtime back to its rollouts
A rollout whose collector died (handover, crash) is paused on the next start (CONTROLLER_RESTARTED) with paused_at taken from execution.json's updated_at. But reconcile_run first settles interrupted operations through the stage Worker, which rewrites execution.json, so updated_at was already "now": no downtime was handed back and any stop longer than the remaining budget timed the rollout out the moment it resumed. Those timeouts look like real 24 h runs to the sibling rule, so their requeued attempts were never claimed either (11-06..08: a 33 h stop, 10 of 12 Qwen rollouts lost). reconcile() now takes owner_lost_at (the dead owner's last write, read before the caller's own saves) and adds the whole downtime to the deadline, uncapped: the paused-credit cap is for pauses that may never resume, while a restarted collector resumes at once. downtime_s records it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
487df8d7ac
commit
dcf976dc95
@ -722,12 +722,13 @@ class Execution:
|
||||
finished_at=time.time())
|
||||
return self._save(state, 'EXECUTION_FINISHED', outcome=outcome, reason=reason)
|
||||
|
||||
def reconcile(self, crash_checkpoint=None):
|
||||
def reconcile(self, crash_checkpoint=None, owner_lost_at=None):
|
||||
"""Fail closed; never replay an unknown side effect or take a live owner.
|
||||
|
||||
crash_checkpoint: the agent's last durable session boundary, frozen by the caller after its owner (the
|
||||
collector) died with every process proven gone and every operation settled. The rollout then pauses
|
||||
(CONTROLLER_RESTARTED) and resumes from there instead of being lost."""
|
||||
(CONTROLLER_RESTARTED) and resumes from there instead of being lost.
|
||||
owner_lost_at: when the dead owner last wrote this state, read before the caller's own saves."""
|
||||
with self.locked():
|
||||
state = self.snapshot()
|
||||
if self.epoch is not None:
|
||||
@ -757,9 +758,14 @@ class Execution:
|
||||
call['state'] = 'INTERRUPTED' # it died with its owner; the resumed agent is told so
|
||||
for scope in state.get('process_scopes', {}).values():
|
||||
scope.setdefault('owner', state['owner']) # scopes registered before owners were recorded
|
||||
# No collector ran, so none of the downtime was the agent's: hand all of it back now, uncapped (the
|
||||
# paused-credit cap is for pauses that may never resume). 11-06..08: a 33 h planned stop, then
|
||||
# updated_at refreshed by the restart's own settling, timed out 10 rollouts the moment they resumed.
|
||||
downtime = max(0.0, time.time() - (owner_lost_at or state.get('updated_at') or time.time()))
|
||||
state['deadline_monotonic'] += downtime
|
||||
state['downtime_s'] = state.get('downtime_s', 0.0) + downtime
|
||||
state.update(execution_state='PAUSED_INFRA', stop_reason='CONTROLLER_RESTARTED', checkpoint=checkpoint,
|
||||
paused_at=state.get('updated_at') or time.time(), activity='WAITING_QUOTA',
|
||||
activity_details={})
|
||||
paused_at=time.time(), activity='WAITING_QUOTA', activity_details={})
|
||||
return self._save(state, 'EXECUTION_RECONCILED', owner_status=owner, members=members,
|
||||
pending_operations=0, reconciliation='CONTROLLER_RESTARTED')
|
||||
if owner == 'DEAD' and members == [] and not operations:
|
||||
|
||||
@ -210,7 +210,8 @@ def reconcile_run(run_id: str, *, attempts=None) -> dict:
|
||||
if record['execution_state'] == 'ACTIVE' and execution.process_status(record.get('owner', {})) == 'DEAD':
|
||||
from .collection_runtime import crash_checkpoint
|
||||
crash = crash_checkpoint(rd, owner)
|
||||
record = owner.reconcile(crash_checkpoint=crash)
|
||||
# record is the snapshot read before Worker settled anything: its updated_at is the dead owner's last write.
|
||||
record = owner.reconcile(crash_checkpoint=crash, owner_lost_at=record.get('updated_at'))
|
||||
if record.get('stop_reason') == 'CONTROLLER_RESTARTED' and record['execution_state'] == 'PAUSED_INFRA':
|
||||
try:
|
||||
with db.connect(timeout_s=0) as con:
|
||||
|
||||
@ -353,6 +353,25 @@ class ExecutionTests(unittest.TestCase):
|
||||
state = self.ex.resume(counted=False)
|
||||
self.assertEqual(state['paused_s'], 10) # at most one execution budget (policy: 10 s)
|
||||
|
||||
def test_collector_downtime_is_handed_back_in_full_on_restart(self):
|
||||
# 11-06..08: a 33 h planned stop. The restart settled interrupted operations first, which rewrote updated_at,
|
||||
# so the pause "began" at the restart, nothing was handed back and 10 rollouts were judged 24 h timeouts.
|
||||
before = self.ex.snapshot()
|
||||
lost_at, restart = before['updated_at'], before['updated_at'] + 1000 # 100 budgets later (policy: 10 s)
|
||||
stale = json.loads(self.ex.path.read_text()); stale['updated_at'] = restart
|
||||
self.ex.path.write_text(json.dumps(stale)) # what the restart's settle did
|
||||
with patch.object(execution, 'process_status', return_value='DEAD'), \
|
||||
patch.object(execution, 'owned_processes', return_value=[]), \
|
||||
patch.object(execution, 'marker_processes', return_value=[]), \
|
||||
patch.object(execution.time, 'time', return_value=restart), \
|
||||
patch.object(execution.time, 'monotonic', return_value=before['deadline_monotonic'] + 990):
|
||||
restarted = execution.Execution(self.rd)
|
||||
state = restarted.reconcile(crash_checkpoint=self.checkpoint(), owner_lost_at=lost_at)
|
||||
self.assertEqual((state['execution_state'], state['stop_reason']), ('PAUSED_INFRA', 'CONTROLLER_RESTARTED'))
|
||||
self.assertEqual(state['deadline_monotonic'], before['deadline_monotonic'] + 1000) # uncapped
|
||||
self.assertNotEqual(restarted.stop_reason(), 'EXECUTION_DEADLINE_EXCEEDED')
|
||||
self.assertEqual(restarted.resume(automatic=False)['execution_state'], 'ACTIVE')
|
||||
|
||||
def test_paused_past_deadline_after_reboot_is_deadline_not_resumable(self):
|
||||
self.ex.pause('CONTROLLER_RESTARTED', self.checkpoint())
|
||||
state = self.ex.snapshot()
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user