"""Small durable lifecycle ledger. One lock serializes cancel, spawn and finish. Only Linux /proc identities authorize signals. Unknown ownership is quarantined, never inferred from a PID or from an old RUNNING label. """ from __future__ import annotations import contextlib import fcntl import hashlib import json import math import os import socket import signal import tempfile import threading import time import uuid from pathlib import Path PHASES = ('01-fetch', '02-baseline', '03-export', '04-calibrate', '05-compile', '06-hbm', '07-board', '08-eval') POLICY_FIELDS = ('queue_deadline_s', 'execution_deadline_s', 'suspect_after_s', 'cancel_grace_s', 'cleanup_deadline_s') _local = threading.local() class ExecutionStopped(RuntimeError): pass def atomic_json(path: Path, value): path.parent.mkdir(parents=True, exist_ok=True) fd, name = tempfile.mkstemp(prefix='.' + path.name + '-', dir=path.parent) try: with os.fdopen(fd, 'w') as out: json.dump(value, out, ensure_ascii=False, sort_keys=True) out.flush(); os.fsync(out.fileno()) os.replace(name, path) directory = os.open(path.parent, os.O_RDONLY) try: os.fsync(directory) finally: os.close(directory) finally: if os.path.exists(name): os.unlink(name) def boot_id(): try: return Path('/proc/sys/kernel/random/boot_id').read_text().strip() except OSError: return None def process_identity(pid: int) -> dict: result = dict(host=socket.gethostname(), boot_id=boot_id(), pid=pid) try: raw = Path(f'/proc/{pid}/stat').read_bytes() row = raw.rsplit(b')', 1)[1].split() result.update(proc_start_ticks=int(row[19]), ppid=int(row[1]), pgrp=int(row[2]), session=int(row[3]), state=row[0].decode('ascii'), cpu_ticks=int(row[11]) + int(row[12]), status='KNOWN') except (OSError, ValueError, IndexError): result['status'] = 'UNKNOWN' return result def process_status(identity: dict) -> str: if (identity.get('host') != socket.gethostname() or not identity.get('boot_id') or identity.get('boot_id') != boot_id() or identity.get('proc_start_ticks') is None): return 'UNKNOWN' current = process_identity(identity['pid']) if current['status'] == 'UNKNOWN': return 'DEAD' if not Path(f"/proc/{identity['pid']}").exists() else 'UNKNOWN' if current['proc_start_ticks'] != identity['proc_start_ticks'] or current['state'] == 'Z': return 'DEAD' return 'ALIVE' def signal_process(identity, sig): """A pidfd closes the identity-check/kill PID-reuse race on supported Linux.""" if process_status(identity) != 'ALIVE': return process_status(identity) == 'DEAD' fd = None try: if hasattr(os, 'pidfd_open') and hasattr(signal, 'pidfd_send_signal'): fd = os.pidfd_open(identity['pid']) if process_status(identity) != 'ALIVE': return True signal.pidfd_send_signal(fd, sig) else: # Refuse rather than authorize a signal from an unverifiable identifier. return False return True except ProcessLookupError: return True except (PermissionError, OSError): return False finally: if fd is not None: os.close(fd) def process_key(identity): return tuple(identity.get(k) for k in ('host', 'boot_id', 'pid', 'proc_start_ticks')) def marker_processes(key, value): """Enumerate /proc only; an unguessable inherited marker joins detached children.""" proc = Path('/proc') if not proc.is_dir(): return None needle = f'{key}={value}'.encode() result = [] for entry in proc.iterdir(): if not entry.name.isdigit(): continue try: fields = (entry / 'environ').read_bytes().split(b'\0') matches = needle in fields if key == 'BENCH_PROCESS_ANCESTRY': prefix = (key + '=').encode() matches = any(value.encode() in field[len(prefix):].split(b',') for field in fields if field.startswith(prefix)) if matches: identity = process_identity(int(entry.name)) if identity['status'] == 'KNOWN' and identity['state'] != 'Z': result.append(identity) except (OSError, ValueError): continue return result def owned_processes(state): """Incomplete containment receipts remain unknown even with no visible PID.""" members = {} for identity in state.get('processes', []): if process_status(identity) != 'DEAD': members[process_key(identity)] = dict(identity, ownership_verified=True) scopes = state.get('process_scopes', {}) for token in state.get('process_sets', {}): if token not in scopes: return None # Legacy observations cannot prove a complete owned set. for scope in scopes.values(): if not scope_cleaned(scope, scopes) and process_status(scope['binding']['supervisor']) != 'ALIVE': return None return list(members.values()) def scope_cleaned(scope, scopes) -> bool: """One rule for every caller that asks whether a process scope left nothing behind.""" receipt = scope_receipt(scope) if receipt and receipt['cleanup_state'] == 'VERIFIED': return True # A nested reaper may lose its parent during cancellation even though # both levels finish waitpid(ECHILD). The enclosing verified receipt # must name this exact supervisor, including boot/start identity. if not (receipt and receipt.get('cleanup_state') == 'QUARANTINED' and receipt.get('all_children_reaped') is True and receipt.get('owner_lost') is True and receipt.get('supervisor_error') is None): return False supervisor = scope['binding']['supervisor'] for parent in scopes.values(): proof = scope_receipt(parent) binding = parent['binding'] if (proof and proof['cleanup_state'] == 'VERIFIED' and binding.get('execution_id') == scope['binding'].get('execution_id') and binding.get('epoch') == scope['binding'].get('epoch') and supervisor.get('ppid') == binding['leader'].get('pid') and process_key(supervisor) in {process_key(p) for p in proof.get('members', [])}): return True return False def scope_receipt(scope): try: receipt = json.loads(Path(scope['receipt_path']).read_text()) if (receipt.get('binding') != scope['binding'] or scope['binding'].get('proof') != 'waitpid:ECHILD' or not isinstance(receipt.get('exit_code'), int)): return None if receipt.get('cleanup_state') == 'VERIFIED' and ( receipt.get('all_children_reaped') is not True or receipt.get('owner_lost') is not False): return None return receipt except (OSError, ValueError, KeyError, TypeError): return None def validate_policy(config): if not isinstance(config, dict) or not config.get('version'): raise ValueError('LIFECYCLE_CONFIG_INCOMPLETE: version') config = json.loads(json.dumps(config)) config.setdefault('phases', {}).setdefault('terminal', {field: config.get(field) for field in POLICY_FIELDS}) for name, row in [('terminal', config['phases']['terminal']), ('execution', config), *[(s, config.get('phases', {}).get(s)) for s in PHASES]]: if not isinstance(row, dict): raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}') for field in POLICY_FIELDS: value = row.get(field) if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0: raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}.{field}') if row['cleanup_deadline_s'] < row['cancel_grace_s']: raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}.cleanup_deadline_s') config = json.loads(json.dumps(config)) cap = config.setdefault('ordinary_tool_deadline_s', 300) if isinstance(cap, bool) or not isinstance(cap, (int, float)) or not math.isfinite(cap) or cap <= 0: raise ValueError('LIFECYCLE_CONFIG_INCOMPLETE: ordinary_tool_deadline_s') return config def resume_policy(state): """Reject an implicit policy upgrade before any epoch, quota or DB mutation.""" cap = state.get('config', {}).get('ordinary_tool_deadline_s') if isinstance(cap, bool) or not isinstance(cap, (int, float)) or not math.isfinite(cap) or cap <= 0: raise ExecutionStopped('RESUME_TOOL_POLICY_MISSING') def current(): return getattr(_local, 'execution', None) def current_phase(): return getattr(_local, 'phase', None) def current_operation(): return getattr(_local, 'operation', None) or os.environ.get('BENCH_OPERATION_ID') class Execution: def __init__(self, rd): self.rd = Path(rd).resolve() self.path = self.rd / 'execution.json' self.epoch = None @contextlib.contextmanager def locked(self): self.rd.mkdir(parents=True, exist_ok=True) with (self.rd / 'execution.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX) try: yield finally: fcntl.flock(lock, fcntl.LOCK_UN) def snapshot(self): return json.loads(self.path.read_text()) def assert_owner(self, state): if self.epoch is None or self.epoch != state['epoch']: raise ExecutionStopped('OWNER_FENCED') def _save(self, state, event_type, **fields): self.assert_owner(state) state['sequence'] = state.get('sequence', 0) + 1 state['updated_at'] = time.time() event = dict(event_id=f"{state['execution_id']}:{state['sequence']}", execution_id=state['execution_id'], sequence=state['sequence'], time=state['updated_at'], type=event_type, epoch=state['epoch'], **fields) state['last_event'] = event atomic_json(self.path, state) with (self.rd / 'execution.events.jsonl').open('a') as out: out.write(json.dumps(event, sort_keys=True) + '\n'); out.flush(); os.fsync(out.fileno()) return state def start(self, config, activity='WAITING_LLM'): config = validate_policy(config) with self.locked(): if self.path.exists(): raise ExecutionStopped('EXECUTION_EXISTS: reconcile; never reset an existing deadline') now = time.monotonic() state = dict(execution_id=uuid.uuid4().hex, epoch=1, owner=process_identity(os.getpid()), boot_id=boot_id(), started_at=time.time(), started_monotonic=now, deadline_monotonic=now + config['execution_deadline_s'], config=config, config_version=config['version'], config_sha256=hashlib.sha256( json.dumps(config, sort_keys=True).encode()).hexdigest(), activity=activity, execution_state='ACTIVE', execution_outcome=None, cleanup_state='NOT_NEEDED', cancel_requested=None, stop_reason=None, processes=[], token=uuid.uuid4().hex, operations={}, phases={}, stage_sequence=0, progress={}, health='UNKNOWN', auto_resume_count=0) self.epoch = state['epoch'] return self._save(state, 'EXECUTION_STARTED') def attach(self): self.epoch = self.snapshot()['epoch'] return self def _reason(self, state, *, admission=False): if self.epoch is not None and self.epoch != state['epoch']: return 'OWNER_FENCED' if state['cancel_requested']: return 'CANCEL_REQUESTED' if state['cleanup_state'] == 'QUARANTINED': return state.get('stop_reason') or 'RESOURCE_QUARANTINED' if state['execution_state'] != 'ACTIVE' and (admission or state['execution_state'] != 'PAUSED_INFRA'): return state.get('stop_reason') or state['execution_state'] if state.get('boot_id') != boot_id(): return 'BOOT_CHANGED_RECONCILE_REQUIRED' if time.monotonic() >= state['deadline_monotonic']: return 'EXECUTION_DEADLINE_EXCEEDED' owner_status = process_status(state['owner']) if owner_status == 'DEAD': return 'OWNER_LOST' if admission and owner_status != 'ALIVE': return 'OWNER_STATUS_UNKNOWN' return None def stop_reason(self, operation_id=None): state = self.snapshot() return self._reason(state) or self._phase_reason(state, operation_id or current_operation()) def _phase_reason(self, state, operation_id): phase = state.get('phases', {}).get(operation_id) if not phase or phase['state'] == 'DONE': return None if phase['boot_id'] != boot_id(): return 'PHASE_CLOCK_UNKNOWN' key = 'queue_deadline_monotonic' if phase['state'] == 'QUEUED' else 'execution_deadline_monotonic' if time.monotonic() >= phase[key]: return 'QUEUE_DEADLINE_EXCEEDED' if phase['state'] == 'QUEUED' else 'PHASE_EXECUTION_DEADLINE_EXCEEDED' return None def begin_phase(self, operation_id, phase, *, queued_monotonic=None): with self.admission() as state: if operation_id in state.get('phases', {}): return state['phases'][operation_id] # Never reset a persisted phase clock. policy = state['config']['phases'][phase] started = time.monotonic() if queued_monotonic is None else queued_monotonic row = dict(phase=phase, state='QUEUED', boot_id=boot_id(), epoch=state['epoch'], queued_monotonic=started, queue_deadline_monotonic=started + policy['queue_deadline_s'], execution_started_monotonic=None, execution_deadline_monotonic=None) state.setdefault('phases', {})[operation_id] = row self._save(state, 'PHASE_QUEUED', operation_id=operation_id, phase=phase) return row def phase_running(self, operation_id): with self.admission() as state: phase = state.get('phases', {}).get(operation_id) if not phase or phase['state'] != 'QUEUED': return reason = self._phase_reason(state, operation_id) if reason: raise ExecutionStopped(reason) now = time.monotonic() phase.update(state='RUNNING', execution_started_monotonic=now, execution_deadline_monotonic=now + state['config']['phases'][phase['phase']]['execution_deadline_s']) state['stage_sequence'] = state.get('stage_sequence', 0) + 1 self._save(state, 'PHASE_STARTED', operation_id=operation_id, phase=phase['phase']) def phase_done_locked(self, state, operation_id): self.assert_owner(state) if operation_id in state.get('phases', {}): state['phases'][operation_id]['state'] = 'DONE' state['stage_sequence'] = state.get('stage_sequence', 0) + 1 @contextlib.contextmanager def admission(self): """Caller must spawn AND register within this interval; cancel uses this lock.""" with self.locked(): state = self.snapshot() reason = self._reason(state, admission=True) or self._phase_reason(state, current_operation()) if reason: raise ExecutionStopped(reason) yield state @contextlib.contextmanager def mutation(self): with self.locked(): state = self.snapshot() self.assert_owner(state) yield state @contextlib.contextmanager def scope(self, phase=None, operation_id=None): old = (current(), current_phase(), getattr(_local, 'operation', None)) _local.execution, _local.phase, _local.operation = self, phase, operation_id try: yield self finally: _local.execution, _local.phase, _local.operation = old def policy(self, phase=None): config = self.snapshot()['config'] return config['phases'][phase] if phase in config.get('phases', {}) else config def environment(self, env=None): state = self.snapshot() return dict(os.environ if env is None else env, BENCH_EXECUTION_DIR=str(self.rd), BENCH_EXECUTION_EPOCH=str(state['epoch']), BENCH_EXECUTION_TOKEN=state['token']) def register_locked(self, state, proc, role, scope_token): identity = getattr(proc, 'leader', None) or process_identity(proc.pid) state['processes'].append(dict(identity, role=role, scope_token=scope_token, epoch=state['epoch'])) self._save(state, 'PROCESS_STARTED', process=identity, role=role) def register_scope_locked(self, state, scope, role): state.setdefault('process_scopes', {})[scope.token] = { 'binding': scope.binding, 'receipt_path': str(scope.path)} self.register_locked(state, scope.supervisor, 'supervisor', scope.token) self.register_locked(state, scope, role, scope.token) def record_process_set(self, token, leader, identities, unknown): with self.mutation() as state: recorded = {process_key(p) for p in state['processes']} added = [dict(p, role='descendant', scope_token=token, epoch=state['epoch']) for p in identities if process_key(p) not in recorded] scope = {'leader': leader, 'unknown': unknown} previous = state.setdefault('process_sets', {}).get(token) if added or previous != scope: state['processes'].extend(added) state['process_sets'][token] = scope self._save(state, 'PROCESS_SET_OBSERVED', scope_token=token, added=len(added), unknown=len(unknown)) def activity(self, name, **details): with self.locked(): state = self.snapshot() if self.epoch != state['epoch']: raise ExecutionStopped('OWNER_FENCED') if state['execution_state'] != 'TERMINAL': state['activity'] = name state['activity_details'] = details return self._save(state, 'ACTIVITY_CHANGED', activity=name, details=details) def observe_tools(self, events): """Persist typed SDK tool boundaries; a timeout fences all new admission.""" with self.locked(): state = self.snapshot() self.assert_owner(state) if state['execution_state'] != 'ACTIVE': return state.get('stop_reason') calls = state.setdefault('tool_calls', {}) changed = False for event in events: if event.get('execution_id') != state['execution_id'] or event.get('epoch') != state['epoch']: raise ExecutionStopped('TOOL_EVENT_IDENTITY_MISMATCH') key = event['tool_call_id'] row = calls.get(key) if row is not None and any(event.get(field) != row[field] for field in ('started_monotonic', 'tool_name', 'cap_s')): # Provider call IDs are not occurrence identities. Keep the # first occurrence immutable and reject ambiguous reuse, # including a late end that belongs to another occurrence. reason = 'INFRA_TOOL_IDENTITY_CONFLICT' state.update(execution_state='STOP_REQUESTED', cleanup_state='PENDING', stop_reason=reason, tool_conflict=dict(event), activity='RUNNING_TOOL', activity_details=dict(event)) self._save(state, reason) return reason if event['state'] == 'RUNNING' and row is None: started, cap = event['started_monotonic'], event['cap_s'] if (not isinstance(key, str) or not key or not math.isfinite(started) or not math.isfinite(cap) or not 0 < cap <= (state['config'].get('managed_tool_deadline_s', state['config']['execution_deadline_s']) if event.get('tool_name') == 'terminal_exec' else state['config'].get('ordinary_tool_deadline_s', 300)) or not state['started_monotonic'] <= started <= time.monotonic() + 1): raise ExecutionStopped('TOOL_EVENT_INVALID') row = dict(tool_call_id=key, tool_name=event['tool_name'], state='RUNNING', started_monotonic=started, deadline_monotonic=started + cap, cap_s=cap) calls[key] = row changed = True elif event['state'] in ('DONE', 'TIMED_OUT') and row is not None: # A late result cannot move the absolute deadline or erase its expiry. row['state'] = ('TIMED_OUT' if row['state'] == 'TIMED_OUT' or event['state'] == 'TIMED_OUT' or event['observed_monotonic'] >= row['deadline_monotonic'] else 'DONE') changed = True overdue = next((v for v in calls.values() if v['state'] == 'TIMED_OUT' or (v['state'] == 'RUNNING' and time.monotonic() >= v['deadline_monotonic'])), None) active = overdue or next((v for v in calls.values() if v['state'] == 'RUNNING'), None) if active: state.update(activity='RUNNING_TOOL', activity_details=dict(active)) elif changed and state['activity'] == 'RUNNING_TOOL': state.update(activity='WAITING_LLM', activity_details={}) if overdue: overdue['state'] = 'TIMED_OUT' state.update(execution_state='STOP_REQUESTED', cleanup_state='PENDING', stop_reason='TOOL_DEADLINE_EXCEEDED', activity_details=dict(overdue)) if changed or overdue: self._save(state, 'TOOL_DEADLINE_EXCEEDED' if overdue else 'TOOL_OBSERVED') return state.get('stop_reason') def record_progress(self, source, signals, *, phase=None, operation_id=None, role='tool'): """One typed sample drives health and the read-only monitor; no log-text heuristics.""" with self.locked(): state = self.snapshot() self.assert_owner(state) now, wall = time.monotonic(), time.time() progress = state.setdefault('progress', {}) previous = progress.setdefault('sources', {}).get(source, {}) old = previous.get('signals', {}) changes = {key: (None if value is None or old.get(key) is None else value != old[key]) for key, value in signals.items()} unknown = any(value is None for value in signals.values()) changed = any(value is True for value in changes.values()) idle_since = None if unknown or changed else previous.get('idle_since_monotonic') or now threshold = state['config']['phases'][phase]['suspect_after_s'] if phase in PHASES else state['config']['suspect_after_s'] phase_row = state.get('phases', {}).get(operation_id, {}) waiting = phase_row.get('state') == 'QUEUED' or (not operation_id and state['activity'] in ('QUEUED', 'WAITING_QUOTA', 'WAITING_RESOURCE')) health = ('WAITING' if waiting else 'UNKNOWN' if unknown else 'SUSPECT' if idle_since is not None and now - idle_since >= threshold else 'OBSERVED') sample = dict(sampled_at=wall, sampled_monotonic=now, signals=signals, changed=changes, idle_since_monotonic=idle_since, suspect_after_s=threshold, health=health, role=role, operation_id=operation_id, phase=phase, reason='SIGNAL_UNKNOWN' if unknown else 'ALL_SIGNALS_IDLE' if health == 'SUSPECT' else 'PROGRESS_OBSERVED') progress['sources'][source] = sample active = {key for key, value in state.get('phases', {}).items() if value['state'] in ('QUEUED', 'RUNNING')} # Agent silence while a separately owned stage is active is expected. if role != 'agent' or not active: progress.update(sampled_at=wall, current_source=source, **{k: sample[k] for k in ('signals', 'changed', 'idle_since_monotonic', 'suspect_after_s', 'reason')}) state['health'] = health return self._save(state, 'PROGRESS_SAMPLED', source=source, health=health) def cancel(self, reason): if not isinstance(reason, str) or not reason.strip(): raise ValueError('cancel requires a reason') with self.locked(): state = self.snapshot() if self.epoch is None: # Explicit control-plane cancellation binds under the same lock. self.epoch = state['epoch'] self.assert_owner(state) if state['execution_state'] in ('TERMINAL', 'STOP_REQUESTED'): return dict(state, reason='ALREADY_TERMINAL') if not state['cancel_requested']: state['cancel_requested'] = dict(reason=reason.strip(), time=time.time(), epoch=state['epoch']) state.update(execution_state='CANCEL_REQUESTED', cleanup_state='PENDING', stop_reason='CANCEL_REQUESTED') self._save(state, 'CANCEL_REQUESTED', reason=reason.strip()) return state def quarantine(self, reason): with self.locked(): state = self.snapshot() self.assert_owner(state) state.update(cleanup_state='QUARANTINED', stop_reason=reason) if state['execution_state'] == 'ACTIVE': state['execution_state'] = 'PAUSED_INFRA' return self._save(state, 'RESOURCE_QUARANTINED', reason=reason) def _checkpoint(self, value): if not isinstance(value, dict) or not (value.get('resumable') is True and value.get('phase') == 'PAUSED_INFRA' or value.get('complete') is True and 'resumable' not in value): raise ExecutionStopped('CHECKPOINT_INCOMPLETE') supplied = Path(value.get('path', '')) path = supplied.resolve() if supplied.is_symlink() or not path.is_relative_to(self.rd) or not path.is_file(): raise ExecutionStopped('CHECKPOINT_PATH_INVALID') digest = hashlib.sha256(path.read_bytes()).hexdigest() if digest != value.get('sha256'): raise ExecutionStopped('CHECKPOINT_HASH_MISMATCH') return dict(value, path=str(path)) def pause(self, reason, checkpoint): if not isinstance(reason, str) or not reason.strip(): raise ValueError('pause requires a reason') with self.admission() as state: checkpoint = self._checkpoint(checkpoint) state.update(execution_state='PAUSED_INFRA', stop_reason=reason, checkpoint=checkpoint, paused_at=time.time()) return self._save(state, 'EXECUTION_PAUSED', reason=reason) def resume(self, *, automatic=True): with self.locked(): state = self.snapshot() if self.epoch is not None and self.epoch != state['epoch']: raise ExecutionStopped('OWNER_FENCED') resume_policy(state) if state['execution_state'] != 'PAUSED_INFRA' or state.get('cancel_requested'): raise ExecutionStopped('EXECUTION_NOT_RESUMABLE') if state['cleanup_state'] == 'QUARANTINED': raise ExecutionStopped('SIDE_EFFECT_UNKNOWN') if state.get('boot_id') != boot_id(): raise ExecutionStopped('BOOT_CHANGED_RECONCILE_REQUIRED') if time.monotonic() >= state['deadline_monotonic']: raise ExecutionStopped('EXECUTION_DEADLINE_EXCEEDED') self._checkpoint(state.get('checkpoint')) owner_status = process_status(state['owner']) same_owner = process_key(state['owner']) == process_key(process_identity(os.getpid())) if owner_status != 'DEAD' and not same_owner: raise ExecutionStopped('OWNER_NOT_RELEASABLE') members = owned_processes(state) pending = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE'] if members != [] or pending: raise ExecutionStopped('SIDE_EFFECT_UNKNOWN' if members is None else 'WORK_STILL_ACTIVE') count = state.get('auto_resume_count', 0) if automatic and count >= state['config'].get('max_auto_resumes', 3): raise ExecutionStopped('AUTO_RESUME_BUDGET_EXHAUSTED') state['epoch'] += 1 self.epoch = state['epoch'] state.update(owner=process_identity(os.getpid()), execution_state='ACTIVE', stop_reason=None, auto_resume_count=count + int(automatic), resumed_at=time.time(), activity='WAITING_QUOTA') return self._save(state, 'EXECUTION_RESUMED', automatic=automatic) def cleanup(self): """Bounded cancellation cleanup; unknown remote receipts keep leases isolated.""" with self.locked(): state = self.snapshot() self.assert_owner(state) if not state.get('cancel_requested'): raise ExecutionStopped('CLEANUP_REQUIRES_CANCEL') if state['execution_state'] == 'TERMINAL': return state state['execution_state'] = 'CLEANING' self._save(state, 'CLEANUP_STARTED') policy = state['config'] start = time.monotonic() while True: members = owned_processes(state) latest = self.snapshot() self.assert_owner(latest) pending = [v for v in latest.get('operations', {}).values() if v.get('state') != 'DONE'] if members == [] and not pending: break elapsed = time.monotonic() - start if members is None or elapsed >= policy['cleanup_deadline_s']: return self.quarantine('CLEANUP_UNCONFIRMED') # Each supervisor performs its own bounded TERM/KILL/reap sequence. # Killing the reaper itself would destroy the complete-set proof. for scope in latest.get('process_scopes', {}).values(): signal_process(scope['binding']['supervisor'], signal.SIGTERM) time.sleep(.1) with self.locked(): latest = self.snapshot() self.assert_owner(latest) if latest['cleanup_state'] != 'QUARANTINED': latest['cleanup_state'] = 'VERIFIED' return self._save(latest, 'CLEANUP_FINISHED', cleanup_state=latest['cleanup_state']) def finish(self, exit_code, *, protocol_complete, stop_reason=None, cleanup_state='VERIFIED'): with self.locked(): state = self.snapshot() if self.epoch != state['epoch']: raise ExecutionStopped('OWNER_FENCED') if state['execution_state'] == 'TERMINAL': return state reason = stop_reason or state.get('stop_reason') members = owned_processes(state) if state['processes'] else [] pending = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE'] known = [process_status(p) for p in state['processes']] if (members not in ([], None) or pending or 'ALIVE' in known or (Path('/proc').is_dir() and ('UNKNOWN' in known or members is None))): cleanup_state = 'QUARANTINED' reason = reason or 'CLEANUP_UNCONFIRMED' if state['cleanup_state'] == 'QUARANTINED' or cleanup_state not in ('VERIFIED', 'NOT_NEEDED'): cleanup_state = 'QUARANTINED' if state['cancel_requested']: outcome = 'CANCELLED' elif reason == 'OWNER_LOST': outcome = 'LOST' elif reason and ('DEADLINE' in reason or 'hard_ceiling' in reason): outcome = 'DEADLINE_EXCEEDED' elif reason and 'BUDGET' in reason: outcome = 'BUDGET_EXHAUSTED' elif reason and any(word in reason for word in ('INFRA', 'UNKNOWN', 'monitor failed')): outcome = 'INFRA_ERROR' elif exit_code == 0 and protocol_complete and cleanup_state == 'VERIFIED': outcome = 'COMPLETED' else: outcome = 'AGENT_ERROR' state.update(execution_state='TERMINAL', execution_outcome=outcome, cleanup_state=cleanup_state, stop_reason=reason, exit_code=exit_code, protocol_complete=bool(protocol_complete), signal=(-exit_code if isinstance(exit_code, int) and exit_code < 0 else exit_code - 128 if isinstance(exit_code, int) and exit_code > 128 else None), finished_at=time.time()) return self._save(state, 'EXECUTION_FINISHED', outcome=outcome, reason=reason) def reconcile(self): """Fail closed; never replay an unknown side effect or take a live owner.""" with self.locked(): state = self.snapshot() if self.epoch is not None: self.assert_owner(state) if state['execution_state'] == 'TERMINAL': return state owner = process_status(state['owner']) members = owned_processes(state) operations = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE'] if owner == 'ALIVE': return dict(state, reconciliation='OWNER_ALIVE') if state['execution_state'] == 'PAUSED_INFRA' and owner == 'DEAD' and members == [] and not operations: try: self._checkpoint(state.get('checkpoint')) except ExecutionStopped: pass else: return dict(state, reconciliation='CHECKPOINT_READY_FOR_RESUME') state['epoch'] += 1 # Fence every old dispatch path before changing ownership. self.epoch = state['epoch'] if owner == 'DEAD' and members == [] and not operations: state.update(execution_state='TERMINAL', execution_outcome='LOST', cleanup_state='VERIFIED', stop_reason='OWNER_LOST', finished_at=time.time()) else: state.update(execution_state='PAUSED_INFRA', cleanup_state='QUARANTINED', stop_reason='SIDE_EFFECT_UNKNOWN' if operations else 'OWNERSHIP_UNKNOWN') return self._save(state, 'EXECUTION_RECONCILED', owner_status=owner, members=members, pending_operations=len(operations)) def from_environment(): value = os.environ.get('BENCH_EXECUTION_DIR') if not value: return None result = Execution(value) result.epoch = int(os.environ['BENCH_EXECUTION_EPOCH']) return result