"""Small durable lifecycle ledger. One lock serializes cancel, spawn and finish. Only Linux /proc identities authorize signals. Unknown ownership is quarantined, never inferred from a PID or from an old RUNNING label. """ from __future__ import annotations import contextlib import fcntl import hashlib import json import math import os import socket import signal import tempfile import threading import time import uuid from pathlib import Path PHASES = ('01-fetch', '02-baseline', '03-export', '04-calibrate', '05-compile', '06-hbm', '07-board', '08-eval') POLICY_FIELDS = ('queue_deadline_s', 'execution_deadline_s', 'suspect_after_s', 'cancel_grace_s', 'cleanup_deadline_s') _local = threading.local() class ExecutionStopped(RuntimeError): pass def atomic_json(path: Path, value): path.parent.mkdir(parents=True, exist_ok=True) fd, name = tempfile.mkstemp(prefix='.' + path.name + '-', dir=path.parent) try: with os.fdopen(fd, 'w') as out: json.dump(value, out, ensure_ascii=False, sort_keys=True) out.flush(); os.fsync(out.fileno()) os.replace(name, path) directory = os.open(path.parent, os.O_RDONLY) try: os.fsync(directory) finally: os.close(directory) finally: if os.path.exists(name): os.unlink(name) def boot_id(): try: return Path('/proc/sys/kernel/random/boot_id').read_text().strip() except OSError: return None def process_identity(pid: int) -> dict: result = dict(host=socket.gethostname(), boot_id=boot_id(), pid=pid) try: raw = Path(f'/proc/{pid}/stat').read_bytes() row = raw.rsplit(b')', 1)[1].split() result.update(proc_start_ticks=int(row[19]), ppid=int(row[1]), pgrp=int(row[2]), session=int(row[3]), state=row[0].decode('ascii'), cpu_ticks=int(row[11]) + int(row[12]), status='KNOWN') except (OSError, ValueError, IndexError): result['status'] = 'UNKNOWN' return result def earlier_boot(identity: dict) -> bool: """This host rebooted since the identity was taken: nothing of that boot still runs.""" return bool(identity.get('host') == socket.gethostname() and identity.get('boot_id') and boot_id() and identity['boot_id'] != boot_id()) def process_status(identity: dict) -> str: if earlier_boot(identity): return 'DEAD' if (identity.get('host') != socket.gethostname() or not identity.get('boot_id') or identity.get('boot_id') != boot_id() or identity.get('proc_start_ticks') is None): return 'UNKNOWN' current = process_identity(identity['pid']) if current['status'] == 'UNKNOWN': return 'DEAD' if not Path(f"/proc/{identity['pid']}").exists() else 'UNKNOWN' if current['proc_start_ticks'] != identity['proc_start_ticks'] or current['state'] == 'Z': return 'DEAD' return 'ALIVE' def signal_process(identity, sig): """A pidfd closes the identity-check/kill PID-reuse race on supported Linux.""" if process_status(identity) != 'ALIVE': return process_status(identity) == 'DEAD' fd = None try: if hasattr(os, 'pidfd_open') and hasattr(signal, 'pidfd_send_signal'): fd = os.pidfd_open(identity['pid']) if process_status(identity) != 'ALIVE': return True signal.pidfd_send_signal(fd, sig) else: # Refuse rather than authorize a signal from an unverifiable identifier. return False return True except ProcessLookupError: return True except (PermissionError, OSError): return False finally: if fd is not None: os.close(fd) def process_key(identity): return tuple(identity.get(k) for k in ('host', 'boot_id', 'pid', 'proc_start_ticks')) def marker_processes(key, value): """Enumerate /proc only; an unguessable inherited marker joins detached children.""" proc = Path('/proc') if not proc.is_dir(): return None needle = f'{key}={value}'.encode() result = [] for entry in proc.iterdir(): if not entry.name.isdigit(): continue try: fields = (entry / 'environ').read_bytes().split(b'\0') matches = needle in fields if key == 'BENCH_PROCESS_ANCESTRY': prefix = (key + '=').encode() matches = any(value.encode() in field[len(prefix):].split(b',') for field in fields if field.startswith(prefix)) if matches: identity = process_identity(int(entry.name)) if identity['status'] == 'KNOWN' and identity['state'] != 'Z': result.append(identity) except (OSError, ValueError): continue return result def owned_processes(state): """Incomplete containment receipts remain unknown even with no visible PID.""" members = {} for identity in state.get('processes', []): if process_status(identity) != 'DEAD': members[process_key(identity)] = dict(identity, ownership_verified=True) scopes = state.get('process_scopes', {}) for token in state.get('process_sets', {}): if token not in scopes: return None # Legacy observations cannot prove a complete owned set. for scope in scopes.values(): if not scope_cleaned(scope, scopes, state.get('owner')) and process_status(scope['binding']['supervisor']) != 'ALIVE': return None return list(members.values()) def scope_cleaned(scope, scopes, owner=None) -> bool: """One rule for every caller that asks whether a process scope left nothing behind.""" receipt = scope_receipt(scope) if receipt and receipt['cleanup_state'] == 'VERIFIED': return True supervisor = scope['binding']['supervisor'] if earlier_boot(supervisor): return True # The execution's owner (the collector) itself died: crash, kill or planned handover. Its own top-level reaper # killed the command and still proved that subtree empty with waitpid(ECHILD). A nested reaper whose parent is # another scope's leader keeps needing that parent's receipt below. The owner is the one that started the scope # (recorded with it): a resumed execution has a new owner, its earlier scopes keep theirs. owner = scope.get('owner') or owner if (owner and receipt and receipt.get('all_children_reaped') is True and receipt.get('owner_lost') is True and receipt.get('supervisor_error') is None and (supervisor.get('ppid'), supervisor.get('host'), supervisor.get('boot_id')) == (owner.get('pid'), owner.get('host'), owner.get('boot_id'))): return True # A nested reaper may lose its parent during cancellation even though # both levels finish waitpid(ECHILD). The enclosing verified receipt # must name this exact supervisor, including boot/start identity. if not (receipt and receipt.get('cleanup_state') == 'QUARANTINED' and receipt.get('all_children_reaped') is True and receipt.get('owner_lost') is True and receipt.get('supervisor_error') is None): return False supervisor = scope['binding']['supervisor'] for parent in scopes.values(): proof = scope_receipt(parent) binding = parent['binding'] if (proof and proof['cleanup_state'] == 'VERIFIED' and binding.get('execution_id') == scope['binding'].get('execution_id') and binding.get('epoch') == scope['binding'].get('epoch') and supervisor.get('ppid') == binding['leader'].get('pid') and process_key(supervisor) in {process_key(p) for p in proof.get('members', [])}): return True return False def scope_receipt(scope): try: receipt = json.loads(Path(scope['receipt_path']).read_text()) if (receipt.get('binding') != scope['binding'] or scope['binding'].get('proof') != 'waitpid:ECHILD' or not isinstance(receipt.get('exit_code'), int)): return None if receipt.get('cleanup_state') == 'VERIFIED' and ( receipt.get('all_children_reaped') is not True or receipt.get('owner_lost') is not False): return None return receipt except (OSError, ValueError, KeyError, TypeError): return None def validate_policy(config): if not isinstance(config, dict) or not config.get('version'): raise ValueError('LIFECYCLE_CONFIG_INCOMPLETE: version') config = json.loads(json.dumps(config)) config.setdefault('phases', {}).setdefault('terminal', {field: config.get(field) for field in POLICY_FIELDS}) for name, row in [('terminal', config['phases']['terminal']), ('execution', config), *[(s, config.get('phases', {}).get(s)) for s in PHASES]]: if not isinstance(row, dict): raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}') for field in POLICY_FIELDS: value = row.get(field) if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0: raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}.{field}') if row['cleanup_deadline_s'] < row['cancel_grace_s']: raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}.cleanup_deadline_s') config = json.loads(json.dumps(config)) cap = config.setdefault('ordinary_tool_deadline_s', 300) if isinstance(cap, bool) or not isinstance(cap, (int, float)) or not math.isfinite(cap) or cap <= 0: raise ValueError('LIFECYCLE_CONFIG_INCOMPLETE: ordinary_tool_deadline_s') return config def resume_policy(state): """Reject an implicit policy upgrade before any epoch, quota or DB mutation.""" cap = state.get('config', {}).get('ordinary_tool_deadline_s') if isinstance(cap, bool) or not isinstance(cap, (int, float)) or not math.isfinite(cap) or cap <= 0: raise ExecutionStopped('RESUME_TOOL_POLICY_MISSING') def paused_credit(state): """Time paused for infrastructure (quota, network) is not the agent's: it is handed back on resume, at most one execution budget in total so a paused rollout that can never resume still ends. 10-14: three rollouts paused 7–16 h by GLM outages resumed with < 1 h left and were cut off mid-compile or mid-eval.""" paused = state.get('paused_at') if state['execution_state'] == 'PAUSED_INFRA' else None if not paused: return 0.0 return max(0.0, min(time.time() - paused, state['config']['execution_deadline_s'] - state.get('paused_s', 0.0))) def current(): return getattr(_local, 'execution', None) def current_phase(): return getattr(_local, 'phase', None) def current_operation(): return getattr(_local, 'operation', None) or os.environ.get('BENCH_OPERATION_ID') class Execution: def __init__(self, rd): self.rd = Path(rd).resolve() self.path = self.rd / 'execution.json' self.epoch = None @contextlib.contextmanager def locked(self): self.rd.mkdir(parents=True, exist_ok=True) with (self.rd / 'execution.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX) try: yield finally: fcntl.flock(lock, fcntl.LOCK_UN) def snapshot(self): return json.loads(self.path.read_text()) def assert_owner(self, state): if self.epoch is None or self.epoch != state['epoch']: raise ExecutionStopped('OWNER_FENCED') def _save(self, state, event_type, **fields): self.assert_owner(state) state['sequence'] = state.get('sequence', 0) + 1 state['updated_at'] = time.time() event = dict(event_id=f"{state['execution_id']}:{state['sequence']}", execution_id=state['execution_id'], sequence=state['sequence'], time=state['updated_at'], type=event_type, epoch=state['epoch'], **fields) state['last_event'] = event atomic_json(self.path, state) with (self.rd / 'execution.events.jsonl').open('a') as out: out.write(json.dumps(event, sort_keys=True) + '\n'); out.flush(); os.fsync(out.fileno()) return state def start(self, config, activity='WAITING_LLM'): config = validate_policy(config) with self.locked(): if self.path.exists(): raise ExecutionStopped('EXECUTION_EXISTS: reconcile; never reset an existing deadline') now = time.monotonic() state = dict(execution_id=uuid.uuid4().hex, epoch=1, owner=process_identity(os.getpid()), boot_id=boot_id(), started_at=time.time(), started_monotonic=now, deadline_monotonic=now + config['execution_deadline_s'], config=config, config_version=config['version'], config_sha256=hashlib.sha256( json.dumps(config, sort_keys=True).encode()).hexdigest(), activity=activity, execution_state='ACTIVE', execution_outcome=None, cleanup_state='NOT_NEEDED', cancel_requested=None, stop_reason=None, processes=[], token=uuid.uuid4().hex, operations={}, phases={}, stage_sequence=0, progress={}, health='UNKNOWN', auto_resume_count=0) self.epoch = state['epoch'] return self._save(state, 'EXECUTION_STARTED') def attach(self): self.epoch = self.snapshot()['epoch'] return self def _reason(self, state, *, admission=False): if self.epoch is not None and self.epoch != state['epoch']: return 'OWNER_FENCED' if state['cancel_requested']: return 'CANCEL_REQUESTED' if state['cleanup_state'] == 'QUARANTINED': return state.get('stop_reason') or 'RESOURCE_QUARANTINED' if state['execution_state'] != 'ACTIVE' and (admission or state['execution_state'] != 'PAUSED_INFRA'): return state.get('stop_reason') or state['execution_state'] if state.get('boot_id') != boot_id(): return 'BOOT_CHANGED_RECONCILE_REQUIRED' if time.monotonic() >= state['deadline_monotonic'] + paused_credit(state): return 'EXECUTION_DEADLINE_EXCEEDED' owner_status = process_status(state['owner']) if owner_status == 'DEAD': return 'OWNER_LOST' if admission and owner_status != 'ALIVE': return 'OWNER_STATUS_UNKNOWN' return None def stop_reason(self, operation_id=None): state = self.snapshot() return self._reason(state) or self._phase_reason(state, operation_id or current_operation()) def _phase_reason(self, state, operation_id): phase = state.get('phases', {}).get(operation_id) if not phase or phase['state'] == 'DONE': return None if phase['boot_id'] != boot_id(): return 'PHASE_CLOCK_UNKNOWN' key = 'queue_deadline_monotonic' if phase['state'] == 'QUEUED' else 'execution_deadline_monotonic' if time.monotonic() >= phase[key]: return 'QUEUE_DEADLINE_EXCEEDED' if phase['state'] == 'QUEUED' else 'PHASE_EXECUTION_DEADLINE_EXCEEDED' return None def begin_phase(self, operation_id, phase, *, queued_monotonic=None): with self.admission() as state: if operation_id in state.get('phases', {}): return state['phases'][operation_id] # Never reset a persisted phase clock. policy = state['config']['phases'][phase] started = time.monotonic() if queued_monotonic is None else queued_monotonic row = dict(phase=phase, state='QUEUED', boot_id=boot_id(), epoch=state['epoch'], queued_monotonic=started, queue_deadline_monotonic=started + policy['queue_deadline_s'], execution_started_monotonic=None, execution_deadline_monotonic=None) state.setdefault('phases', {})[operation_id] = row self._save(state, 'PHASE_QUEUED', operation_id=operation_id, phase=phase) return row def phase_running(self, operation_id): with self.admission() as state: phase = state.get('phases', {}).get(operation_id) if not phase or phase['state'] != 'QUEUED': return reason = self._phase_reason(state, operation_id) if reason: raise ExecutionStopped(reason) now = time.monotonic() phase.update(state='RUNNING', execution_started_monotonic=now, execution_deadline_monotonic=now + state['config']['phases'][phase['phase']]['execution_deadline_s']) state['stage_sequence'] = state.get('stage_sequence', 0) + 1 self._save(state, 'PHASE_STARTED', operation_id=operation_id, phase=phase['phase']) def phase_done_locked(self, state, operation_id): self.assert_owner(state) if operation_id in state.get('phases', {}): state['phases'][operation_id]['state'] = 'DONE' state['stage_sequence'] = state.get('stage_sequence', 0) + 1 @contextlib.contextmanager def admission(self): """Caller must spawn AND register within this interval; cancel uses this lock.""" with self.locked(): state = self.snapshot() reason = self._reason(state, admission=True) or self._phase_reason(state, current_operation()) if reason: raise ExecutionStopped(reason) yield state @contextlib.contextmanager def mutation(self): with self.locked(): state = self.snapshot() self.assert_owner(state) yield state @contextlib.contextmanager def scope(self, phase=None, operation_id=None): old = (current(), current_phase(), getattr(_local, 'operation', None)) _local.execution, _local.phase, _local.operation = self, phase, operation_id try: yield self finally: _local.execution, _local.phase, _local.operation = old def policy(self, phase=None): config = self.snapshot()['config'] return config['phases'][phase] if phase in config.get('phases', {}) else config def environment(self, env=None): state = self.snapshot() return dict(os.environ if env is None else env, BENCH_EXECUTION_DIR=str(self.rd), BENCH_EXECUTION_EPOCH=str(state['epoch']), BENCH_EXECUTION_TOKEN=state['token']) def register_locked(self, state, proc, role, scope_token): identity = getattr(proc, 'leader', None) or process_identity(proc.pid) state['processes'].append(dict(identity, role=role, scope_token=scope_token, epoch=state['epoch'])) self._save(state, 'PROCESS_STARTED', process=identity, role=role) def register_scope_locked(self, state, scope, role): state.setdefault('process_scopes', {})[scope.token] = { 'binding': scope.binding, 'receipt_path': str(scope.path), 'owner': state['owner']} self.register_locked(state, scope.supervisor, 'supervisor', scope.token) self.register_locked(state, scope, role, scope.token) def record_process_set(self, token, leader, identities, unknown): with self.mutation() as state: recorded = {process_key(p) for p in state['processes']} added = [dict(p, role='descendant', scope_token=token, epoch=state['epoch']) for p in identities if process_key(p) not in recorded] scope = {'leader': leader, 'unknown': unknown} previous = state.setdefault('process_sets', {}).get(token) if added or previous != scope: state['processes'].extend(added) state['process_sets'][token] = scope self._save(state, 'PROCESS_SET_OBSERVED', scope_token=token, added=len(added), unknown=len(unknown)) def activity(self, name, **details): with self.locked(): state = self.snapshot() if self.epoch != state['epoch']: raise ExecutionStopped('OWNER_FENCED') if state['execution_state'] != 'TERMINAL': state['activity'] = name state['activity_details'] = details return self._save(state, 'ACTIVITY_CHANGED', activity=name, details=details) def observe_tools(self, events): """Persist typed SDK tool boundaries; a timeout fences all new admission.""" with self.locked(): state = self.snapshot() self.assert_owner(state) if state['execution_state'] != 'ACTIVE': return state.get('stop_reason') calls = state.setdefault('tool_calls', {}) changed = False for event in events: if event.get('execution_id') != state['execution_id'] or event.get('epoch') != state['epoch']: raise ExecutionStopped('TOOL_EVENT_IDENTITY_MISMATCH') key = event['tool_call_id'] row = calls.get(key) if row is not None and any(event.get(field) != row[field] for field in ('started_monotonic', 'tool_name', 'cap_s')): # Provider call IDs are not occurrence identities. Keep the # first occurrence immutable and reject ambiguous reuse, # including a late end that belongs to another occurrence. reason = 'INFRA_TOOL_IDENTITY_CONFLICT' state.update(execution_state='STOP_REQUESTED', cleanup_state='PENDING', stop_reason=reason, tool_conflict=dict(event), activity='RUNNING_TOOL', activity_details=dict(event)) self._save(state, reason) return reason if event['state'] == 'RUNNING' and row is None: started, cap = event['started_monotonic'], event['cap_s'] if (not isinstance(key, str) or not key or not math.isfinite(started) or not math.isfinite(cap) or not 0 < cap <= (state['config'].get('managed_tool_deadline_s', state['config']['execution_deadline_s']) if event.get('tool_name') == 'terminal_exec' else state['config'].get('ordinary_tool_deadline_s', 300)) or not state['started_monotonic'] <= started <= time.monotonic() + 1): raise ExecutionStopped('TOOL_EVENT_INVALID') row = dict(tool_call_id=key, tool_name=event['tool_name'], state='RUNNING', started_monotonic=started, deadline_monotonic=started + cap, cap_s=cap) calls[key] = row changed = True elif event['state'] in ('DONE', 'TIMED_OUT') and row is not None: # A late result cannot move the absolute deadline or erase its expiry. row['state'] = ('TIMED_OUT' if row['state'] == 'TIMED_OUT' or event['state'] == 'TIMED_OUT' or event['observed_monotonic'] >= row['deadline_monotonic'] else 'DONE') changed = True overdue = next((v for v in calls.values() if v['state'] == 'TIMED_OUT' or (v['state'] == 'RUNNING' and time.monotonic() >= v['deadline_monotonic'])), None) active = overdue or next((v for v in calls.values() if v['state'] == 'RUNNING'), None) if active: state.update(activity='RUNNING_TOOL', activity_details=dict(active)) elif changed and state['activity'] == 'RUNNING_TOOL': state.update(activity='WAITING_LLM', activity_details={}) if overdue: overdue['state'] = 'TIMED_OUT' state.update(execution_state='STOP_REQUESTED', cleanup_state='PENDING', stop_reason='TOOL_DEADLINE_EXCEEDED', activity_details=dict(overdue)) if changed or overdue: self._save(state, 'TOOL_DEADLINE_EXCEEDED' if overdue else 'TOOL_OBSERVED') return state.get('stop_reason') def record_progress(self, source, signals, *, phase=None, operation_id=None, role='tool'): """One typed sample drives health and the read-only monitor; no log-text heuristics.""" with self.locked(): state = self.snapshot() self.assert_owner(state) now, wall = time.monotonic(), time.time() progress = state.setdefault('progress', {}) previous = progress.setdefault('sources', {}).get(source, {}) old = previous.get('signals', {}) changes = {key: (None if value is None or old.get(key) is None else value != old[key]) for key, value in signals.items()} unknown = any(value is None for value in signals.values()) changed = any(value is True for value in changes.values()) idle_since = None if unknown or changed else previous.get('idle_since_monotonic') or now threshold = state['config']['phases'][phase]['suspect_after_s'] if phase in PHASES else state['config']['suspect_after_s'] phase_row = state.get('phases', {}).get(operation_id, {}) waiting = phase_row.get('state') == 'QUEUED' or (not operation_id and state['activity'] in ('QUEUED', 'WAITING_QUOTA', 'WAITING_RESOURCE')) health = ('WAITING' if waiting else 'UNKNOWN' if unknown else 'SUSPECT' if idle_since is not None and now - idle_since >= threshold else 'OBSERVED') sample = dict(sampled_at=wall, sampled_monotonic=now, signals=signals, changed=changes, idle_since_monotonic=idle_since, suspect_after_s=threshold, health=health, role=role, operation_id=operation_id, phase=phase, reason='SIGNAL_UNKNOWN' if unknown else 'ALL_SIGNALS_IDLE' if health == 'SUSPECT' else 'PROGRESS_OBSERVED') progress['sources'][source] = sample active = {key for key, value in state.get('phases', {}).items() if value['state'] in ('QUEUED', 'RUNNING')} # Agent silence while a separately owned stage is active is expected. if role != 'agent' or not active: progress.update(sampled_at=wall, current_source=source, **{k: sample[k] for k in ('signals', 'changed', 'idle_since_monotonic', 'suspect_after_s', 'reason')}) state['health'] = health return self._save(state, 'PROGRESS_SAMPLED', source=source, health=health) def cancel(self, reason): if not isinstance(reason, str) or not reason.strip(): raise ValueError('cancel requires a reason') with self.locked(): state = self.snapshot() if self.epoch is None: # Explicit control-plane cancellation binds under the same lock. self.epoch = state['epoch'] self.assert_owner(state) if state['execution_state'] in ('TERMINAL', 'STOP_REQUESTED'): return dict(state, reason='ALREADY_TERMINAL') if not state['cancel_requested']: state['cancel_requested'] = dict(reason=reason.strip(), time=time.time(), epoch=state['epoch']) state.update(execution_state='CANCEL_REQUESTED', cleanup_state='PENDING', stop_reason='CANCEL_REQUESTED') self._save(state, 'CANCEL_REQUESTED', reason=reason.strip()) return state def quarantine(self, reason): with self.locked(): state = self.snapshot() self.assert_owner(state) state.update(cleanup_state='QUARANTINED', stop_reason=reason) if state['execution_state'] == 'ACTIVE': state['execution_state'] = 'PAUSED_INFRA' return self._save(state, 'RESOURCE_QUARANTINED', reason=reason) def _checkpoint(self, value): if not isinstance(value, dict) or not (value.get('resumable') is True and value.get('phase') == 'PAUSED_INFRA' or value.get('complete') is True and 'resumable' not in value): raise ExecutionStopped('CHECKPOINT_INCOMPLETE') supplied = Path(value.get('path', '')) path = supplied.resolve() if supplied.is_symlink() or not path.is_relative_to(self.rd) or not path.is_file(): raise ExecutionStopped('CHECKPOINT_PATH_INVALID') digest = hashlib.sha256(path.read_bytes()).hexdigest() if digest != value.get('sha256'): raise ExecutionStopped('CHECKPOINT_HASH_MISMATCH') return dict(value, path=str(path)) def pause(self, reason, checkpoint): if not isinstance(reason, str) or not reason.strip(): raise ValueError('pause requires a reason') with self.admission() as state: checkpoint = self._checkpoint(checkpoint) state.update(execution_state='PAUSED_INFRA', stop_reason=reason, checkpoint=checkpoint, paused_at=time.time()) return self._save(state, 'EXECUTION_PAUSED', reason=reason) def resume(self, *, automatic=True, counted=True): with self.locked(): state = self.snapshot() if self.epoch is not None and self.epoch != state['epoch']: raise ExecutionStopped('OWNER_FENCED') resume_policy(state) if state['execution_state'] != 'PAUSED_INFRA' or state.get('cancel_requested'): raise ExecutionStopped('EXECUTION_NOT_RESUMABLE') if state['cleanup_state'] == 'QUARANTINED': raise ExecutionStopped('SIDE_EFFECT_UNKNOWN') if state.get('boot_id') != boot_id(): if not earlier_boot(state['owner']): raise ExecutionStopped('BOOT_CHANGED_RECONCILE_REQUIRED') self._rebase_clock(state) credit = paused_credit(state) state['deadline_monotonic'] += credit state['paused_s'] = state.get('paused_s', 0.0) + credit state.pop('paused_at', None) if time.monotonic() >= state['deadline_monotonic']: raise ExecutionStopped('EXECUTION_DEADLINE_EXCEEDED') self._checkpoint(state.get('checkpoint')) owner_status = process_status(state['owner']) same_owner = process_key(state['owner']) == process_key(process_identity(os.getpid())) if owner_status != 'DEAD' and not same_owner: raise ExecutionStopped('OWNER_NOT_RELEASABLE') members = owned_processes(state) pending = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE'] if members != [] or pending: raise ExecutionStopped('SIDE_EFFECT_UNKNOWN' if members is None else 'WORK_STILL_ACTIVE') count = state.get('auto_resume_count', 0) if automatic and counted and count >= state['config'].get('max_auto_resumes', 3): raise ExecutionStopped('AUTO_RESUME_BUDGET_EXHAUSTED') state['epoch'] += 1 self.epoch = state['epoch'] state.update(owner=process_identity(os.getpid()), execution_state='ACTIVE', stop_reason=None, auto_resume_count=count + int(automatic and counted), resumed_at=time.time(), activity='WAITING_QUOTA') return self._save(state, 'EXECUTION_RESUMED', automatic=automatic) @staticmethod def _rebase_clock(state): """Monotonic clocks restart with the host. Carry every deadline over by wall time; the downtime itself is a pause (paused_at), credited back like any other infrastructure pause.""" offset = (time.monotonic() - time.time()) - (state['started_monotonic'] - state['started_at']) state['started_monotonic'] += offset state['deadline_monotonic'] += offset for row in [*state.get('phases', {}).values(), *state.get('tool_calls', {}).values()]: for key, value in row.items(): if key.endswith('_monotonic') and isinstance(value, (int, float)): row[key] = value + offset if 'boot_id' in row: row['boot_id'] = boot_id() state['boot_id'] = boot_id() def cleanup(self): """Bounded cancellation cleanup; unknown remote receipts keep leases isolated.""" with self.locked(): state = self.snapshot() self.assert_owner(state) if not state.get('cancel_requested'): raise ExecutionStopped('CLEANUP_REQUIRES_CANCEL') if state['execution_state'] == 'TERMINAL': return state state['execution_state'] = 'CLEANING' self._save(state, 'CLEANUP_STARTED') policy = state['config'] start = time.monotonic() while True: members = owned_processes(state) latest = self.snapshot() self.assert_owner(latest) pending = [v for v in latest.get('operations', {}).values() if v.get('state') != 'DONE'] if members == [] and not pending: break elapsed = time.monotonic() - start if members is None or elapsed >= policy['cleanup_deadline_s']: return self.quarantine('CLEANUP_UNCONFIRMED') # Each supervisor performs its own bounded TERM/KILL/reap sequence. # Killing the reaper itself would destroy the complete-set proof. for scope in latest.get('process_scopes', {}).values(): signal_process(scope['binding']['supervisor'], signal.SIGTERM) time.sleep(.1) with self.locked(): latest = self.snapshot() self.assert_owner(latest) if latest['cleanup_state'] != 'QUARANTINED': latest['cleanup_state'] = 'VERIFIED' return self._save(latest, 'CLEANUP_FINISHED', cleanup_state=latest['cleanup_state']) def finish(self, exit_code, *, protocol_complete, stop_reason=None, cleanup_state='VERIFIED'): with self.locked(): state = self.snapshot() if self.epoch != state['epoch']: raise ExecutionStopped('OWNER_FENCED') if state['execution_state'] == 'TERMINAL': return state reason = stop_reason or state.get('stop_reason') members = owned_processes(state) if state['processes'] else [] pending = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE'] known = [process_status(p) for p in state['processes']] if (members not in ([], None) or pending or 'ALIVE' in known or (Path('/proc').is_dir() and ('UNKNOWN' in known or members is None))): cleanup_state = 'QUARANTINED' reason = reason or 'CLEANUP_UNCONFIRMED' if state['cleanup_state'] == 'QUARANTINED' or cleanup_state not in ('VERIFIED', 'NOT_NEEDED'): cleanup_state = 'QUARANTINED' if state['cancel_requested']: outcome = 'CANCELLED' elif reason == 'OWNER_LOST': outcome = 'LOST' elif reason and ('DEADLINE' in reason or 'hard_ceiling' in reason): outcome = 'DEADLINE_EXCEEDED' elif reason and 'BUDGET' in reason: outcome = 'BUDGET_EXHAUSTED' elif reason and any(word in reason for word in ('INFRA', 'UNKNOWN', 'monitor failed')): outcome = 'INFRA_ERROR' elif exit_code == 0 and protocol_complete and cleanup_state == 'VERIFIED': outcome = 'COMPLETED' else: outcome = 'AGENT_ERROR' state.update(execution_state='TERMINAL', execution_outcome=outcome, cleanup_state=cleanup_state, stop_reason=reason, exit_code=exit_code, protocol_complete=bool(protocol_complete), signal=(-exit_code if isinstance(exit_code, int) and exit_code < 0 else exit_code - 128 if isinstance(exit_code, int) and exit_code > 128 else None), finished_at=time.time()) return self._save(state, 'EXECUTION_FINISHED', outcome=outcome, reason=reason) def reconcile(self, crash_checkpoint=None): """Fail closed; never replay an unknown side effect or take a live owner. crash_checkpoint: the agent's last durable session boundary, frozen by the caller after its owner (the collector) died with every process proven gone and every operation settled. The rollout then pauses (CONTROLLER_RESTARTED) and resumes from there instead of being lost.""" with self.locked(): state = self.snapshot() if self.epoch is not None: self.assert_owner(state) if state['execution_state'] == 'TERMINAL': return state owner = process_status(state['owner']) members = owned_processes(state) operations = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE'] if owner == 'ALIVE': return dict(state, reconciliation='OWNER_ALIVE') if state['execution_state'] == 'PAUSED_INFRA' and owner == 'DEAD' and members == [] and not operations: try: self._checkpoint(state.get('checkpoint')) except ExecutionStopped: pass else: return dict(state, reconciliation='CHECKPOINT_READY_FOR_RESUME') state['epoch'] += 1 # Fence every old dispatch path before changing ownership. self.epoch = state['epoch'] restartable = (owner == 'DEAD' and members == [] and not operations and crash_checkpoint is not None and state['execution_state'] == 'ACTIVE' and not state.get('cancel_requested')) if restartable: checkpoint = self._checkpoint(crash_checkpoint) for call in state.get('tool_calls', {}).values(): if call['state'] == 'RUNNING': call['state'] = 'INTERRUPTED' # it died with its owner; the resumed agent is told so for scope in state.get('process_scopes', {}).values(): scope.setdefault('owner', state['owner']) # scopes registered before owners were recorded state.update(execution_state='PAUSED_INFRA', stop_reason='CONTROLLER_RESTARTED', checkpoint=checkpoint, paused_at=state.get('updated_at') or time.time(), activity='WAITING_QUOTA', activity_details={}) return self._save(state, 'EXECUTION_RECONCILED', owner_status=owner, members=members, pending_operations=0, reconciliation='CONTROLLER_RESTARTED') if owner == 'DEAD' and members == [] and not operations: state.update(execution_state='TERMINAL', execution_outcome='LOST', cleanup_state='VERIFIED', stop_reason='OWNER_LOST', finished_at=time.time()) else: state.update(execution_state='PAUSED_INFRA', cleanup_state='QUARANTINED', stop_reason='SIDE_EFFECT_UNKNOWN' if operations else 'OWNERSHIP_UNKNOWN') return self._save(state, 'EXECUTION_RECONCILED', owner_status=owner, members=members, pending_operations=len(operations)) def from_environment(): value = os.environ.get('BENCH_EXECUTION_DIR') if not value: return None result = Execution(value) result.epoch = int(os.environ['BENCH_EXECUTION_EPOCH']) return result