data-gen-bpu/harness/execution.py
ming01.wang 44f643d290 fix: stop false OPERATION_STATE_UNKNOWN on finished stages
Three r20260929-06 rollouts sat in PAUSED_INFRA although their stage had
finished (status DONE/FAIL, no process left): the completion replay in
stageworker._validate_recovered_receipt raised ValueError, so the operation
stayed DISPATCHED and the execution was quarantined (one-way).

- Nested scopes: a deadline kill of the outer stage leaves the inner reaper
  with owner_lost=True although both levels reaped via waitpid(ECHILD).
  owned_processes() already accepted that when the enclosing verified receipt
  names the exact supervisor; receipt validation demanded VERIFIED. Both now
  share execution.scope_cleaned().
- Board stages killed by QUEUE_DEADLINE_EXCEEDED never held a board slot
  (phase_running fires only under the board lock) and so have no board
  receipt; that missing receipt no longer counts as unconfirmed cleanup.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 09:50:23 +08:00

705 lines
35 KiB
Python

"""Small durable lifecycle ledger. One lock serializes cancel, spawn and finish.
Only Linux /proc identities authorize signals. Unknown ownership is quarantined,
never inferred from a PID or from an old RUNNING label.
"""
from __future__ import annotations
import contextlib
import fcntl
import hashlib
import json
import math
import os
import socket
import signal
import tempfile
import threading
import time
import uuid
from pathlib import Path
PHASES = ('01-fetch', '02-baseline', '03-export', '04-calibrate', '05-compile',
'06-hbm', '07-board', '08-eval')
POLICY_FIELDS = ('queue_deadline_s', 'execution_deadline_s', 'suspect_after_s',
'cancel_grace_s', 'cleanup_deadline_s')
_local = threading.local()
class ExecutionStopped(RuntimeError):
pass
def atomic_json(path: Path, value):
path.parent.mkdir(parents=True, exist_ok=True)
fd, name = tempfile.mkstemp(prefix='.' + path.name + '-', dir=path.parent)
try:
with os.fdopen(fd, 'w') as out:
json.dump(value, out, ensure_ascii=False, sort_keys=True)
out.flush(); os.fsync(out.fileno())
os.replace(name, path)
directory = os.open(path.parent, os.O_RDONLY)
try:
os.fsync(directory)
finally:
os.close(directory)
finally:
if os.path.exists(name):
os.unlink(name)
def boot_id():
try:
return Path('/proc/sys/kernel/random/boot_id').read_text().strip()
except OSError:
return None
def process_identity(pid: int) -> dict:
result = dict(host=socket.gethostname(), boot_id=boot_id(), pid=pid)
try:
raw = Path(f'/proc/{pid}/stat').read_bytes()
row = raw.rsplit(b')', 1)[1].split()
result.update(proc_start_ticks=int(row[19]), ppid=int(row[1]), pgrp=int(row[2]),
session=int(row[3]), state=row[0].decode('ascii'), cpu_ticks=int(row[11]) + int(row[12]), status='KNOWN')
except (OSError, ValueError, IndexError):
result['status'] = 'UNKNOWN'
return result
def process_status(identity: dict) -> str:
if (identity.get('host') != socket.gethostname() or not identity.get('boot_id')
or identity.get('boot_id') != boot_id() or identity.get('proc_start_ticks') is None):
return 'UNKNOWN'
current = process_identity(identity['pid'])
if current['status'] == 'UNKNOWN':
return 'DEAD' if not Path(f"/proc/{identity['pid']}").exists() else 'UNKNOWN'
if current['proc_start_ticks'] != identity['proc_start_ticks'] or current['state'] == 'Z':
return 'DEAD'
return 'ALIVE'
def signal_process(identity, sig):
"""A pidfd closes the identity-check/kill PID-reuse race on supported Linux."""
if process_status(identity) != 'ALIVE':
return process_status(identity) == 'DEAD'
fd = None
try:
if hasattr(os, 'pidfd_open') and hasattr(signal, 'pidfd_send_signal'):
fd = os.pidfd_open(identity['pid'])
if process_status(identity) != 'ALIVE':
return True
signal.pidfd_send_signal(fd, sig)
else:
# Refuse rather than authorize a signal from an unverifiable identifier.
return False
return True
except ProcessLookupError:
return True
except (PermissionError, OSError):
return False
finally:
if fd is not None:
os.close(fd)
def process_key(identity):
return tuple(identity.get(k) for k in ('host', 'boot_id', 'pid', 'proc_start_ticks'))
def marker_processes(key, value):
"""Enumerate /proc only; an unguessable inherited marker joins detached children."""
proc = Path('/proc')
if not proc.is_dir():
return None
needle = f'{key}={value}'.encode()
result = []
for entry in proc.iterdir():
if not entry.name.isdigit():
continue
try:
fields = (entry / 'environ').read_bytes().split(b'\0')
matches = needle in fields
if key == 'BENCH_PROCESS_ANCESTRY':
prefix = (key + '=').encode()
matches = any(value.encode() in field[len(prefix):].split(b',')
for field in fields if field.startswith(prefix))
if matches:
identity = process_identity(int(entry.name))
if identity['status'] == 'KNOWN' and identity['state'] != 'Z':
result.append(identity)
except (OSError, ValueError):
continue
return result
def owned_processes(state):
"""Incomplete containment receipts remain unknown even with no visible PID."""
members = {}
for identity in state.get('processes', []):
if process_status(identity) != 'DEAD':
members[process_key(identity)] = dict(identity, ownership_verified=True)
scopes = state.get('process_scopes', {})
for token in state.get('process_sets', {}):
if token not in scopes:
return None # Legacy observations cannot prove a complete owned set.
for scope in scopes.values():
if not scope_cleaned(scope, scopes) and process_status(scope['binding']['supervisor']) != 'ALIVE':
return None
return list(members.values())
def scope_cleaned(scope, scopes) -> bool:
"""One rule for every caller that asks whether a process scope left nothing behind."""
receipt = scope_receipt(scope)
if receipt and receipt['cleanup_state'] == 'VERIFIED':
return True
# A nested reaper may lose its parent during cancellation even though
# both levels finish waitpid(ECHILD). The enclosing verified receipt
# must name this exact supervisor, including boot/start identity.
if not (receipt and receipt.get('cleanup_state') == 'QUARANTINED'
and receipt.get('all_children_reaped') is True
and receipt.get('owner_lost') is True
and receipt.get('supervisor_error') is None):
return False
supervisor = scope['binding']['supervisor']
for parent in scopes.values():
proof = scope_receipt(parent)
binding = parent['binding']
if (proof and proof['cleanup_state'] == 'VERIFIED'
and binding.get('execution_id') == scope['binding'].get('execution_id')
and binding.get('epoch') == scope['binding'].get('epoch')
and supervisor.get('ppid') == binding['leader'].get('pid')
and process_key(supervisor) in {process_key(p) for p in proof.get('members', [])}):
return True
return False
def scope_receipt(scope):
try:
receipt = json.loads(Path(scope['receipt_path']).read_text())
if (receipt.get('binding') != scope['binding'] or
scope['binding'].get('proof') != 'waitpid:ECHILD' or
not isinstance(receipt.get('exit_code'), int)):
return None
if receipt.get('cleanup_state') == 'VERIFIED' and (
receipt.get('all_children_reaped') is not True or receipt.get('owner_lost') is not False):
return None
return receipt
except (OSError, ValueError, KeyError, TypeError):
return None
def validate_policy(config):
if not isinstance(config, dict) or not config.get('version'):
raise ValueError('LIFECYCLE_CONFIG_INCOMPLETE: version')
config = json.loads(json.dumps(config))
config.setdefault('phases', {}).setdefault('terminal', {field: config.get(field) for field in POLICY_FIELDS})
for name, row in [('terminal', config['phases']['terminal']), ('execution', config), *[(s, config.get('phases', {}).get(s)) for s in PHASES]]:
if not isinstance(row, dict):
raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}')
for field in POLICY_FIELDS:
value = row.get(field)
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0:
raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}.{field}')
if row['cleanup_deadline_s'] < row['cancel_grace_s']:
raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}.cleanup_deadline_s')
config = json.loads(json.dumps(config))
cap = config.setdefault('ordinary_tool_deadline_s', 300)
if isinstance(cap, bool) or not isinstance(cap, (int, float)) or not math.isfinite(cap) or cap <= 0:
raise ValueError('LIFECYCLE_CONFIG_INCOMPLETE: ordinary_tool_deadline_s')
return config
def resume_policy(state):
"""Reject an implicit policy upgrade before any epoch, quota or DB mutation."""
cap = state.get('config', {}).get('ordinary_tool_deadline_s')
if isinstance(cap, bool) or not isinstance(cap, (int, float)) or not math.isfinite(cap) or cap <= 0:
raise ExecutionStopped('RESUME_TOOL_POLICY_MISSING')
def current():
return getattr(_local, 'execution', None)
def current_phase():
return getattr(_local, 'phase', None)
def current_operation():
return getattr(_local, 'operation', None) or os.environ.get('BENCH_OPERATION_ID')
class Execution:
def __init__(self, rd):
self.rd = Path(rd).resolve()
self.path = self.rd / 'execution.json'
self.epoch = None
@contextlib.contextmanager
def locked(self):
self.rd.mkdir(parents=True, exist_ok=True)
with (self.rd / 'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(lock, fcntl.LOCK_UN)
def snapshot(self):
return json.loads(self.path.read_text())
def assert_owner(self, state):
if self.epoch is None or self.epoch != state['epoch']:
raise ExecutionStopped('OWNER_FENCED')
def _save(self, state, event_type, **fields):
self.assert_owner(state)
state['sequence'] = state.get('sequence', 0) + 1
state['updated_at'] = time.time()
event = dict(event_id=f"{state['execution_id']}:{state['sequence']}",
execution_id=state['execution_id'], sequence=state['sequence'],
time=state['updated_at'], type=event_type, epoch=state['epoch'], **fields)
state['last_event'] = event
atomic_json(self.path, state)
with (self.rd / 'execution.events.jsonl').open('a') as out:
out.write(json.dumps(event, sort_keys=True) + '\n'); out.flush(); os.fsync(out.fileno())
return state
def start(self, config, activity='WAITING_LLM'):
config = validate_policy(config)
with self.locked():
if self.path.exists():
raise ExecutionStopped('EXECUTION_EXISTS: reconcile; never reset an existing deadline')
now = time.monotonic()
state = dict(execution_id=uuid.uuid4().hex, epoch=1, owner=process_identity(os.getpid()),
boot_id=boot_id(), started_at=time.time(), started_monotonic=now,
deadline_monotonic=now + config['execution_deadline_s'], config=config,
config_version=config['version'], config_sha256=hashlib.sha256(
json.dumps(config, sort_keys=True).encode()).hexdigest(),
activity=activity, execution_state='ACTIVE', execution_outcome=None,
cleanup_state='NOT_NEEDED', cancel_requested=None, stop_reason=None,
processes=[], token=uuid.uuid4().hex, operations={}, phases={}, stage_sequence=0,
progress={}, health='UNKNOWN', auto_resume_count=0)
self.epoch = state['epoch']
return self._save(state, 'EXECUTION_STARTED')
def attach(self):
self.epoch = self.snapshot()['epoch']
return self
def _reason(self, state, *, admission=False):
if self.epoch is not None and self.epoch != state['epoch']:
return 'OWNER_FENCED'
if state['cancel_requested']:
return 'CANCEL_REQUESTED'
if state['cleanup_state'] == 'QUARANTINED':
return state.get('stop_reason') or 'RESOURCE_QUARANTINED'
if state['execution_state'] != 'ACTIVE' and (admission or state['execution_state'] != 'PAUSED_INFRA'):
return state.get('stop_reason') or state['execution_state']
if state.get('boot_id') != boot_id():
return 'BOOT_CHANGED_RECONCILE_REQUIRED'
if time.monotonic() >= state['deadline_monotonic']:
return 'EXECUTION_DEADLINE_EXCEEDED'
owner_status = process_status(state['owner'])
if owner_status == 'DEAD':
return 'OWNER_LOST'
if admission and owner_status != 'ALIVE':
return 'OWNER_STATUS_UNKNOWN'
return None
def stop_reason(self, operation_id=None):
state = self.snapshot()
return self._reason(state) or self._phase_reason(state, operation_id or current_operation())
def _phase_reason(self, state, operation_id):
phase = state.get('phases', {}).get(operation_id)
if not phase or phase['state'] == 'DONE':
return None
if phase['boot_id'] != boot_id():
return 'PHASE_CLOCK_UNKNOWN'
key = 'queue_deadline_monotonic' if phase['state'] == 'QUEUED' else 'execution_deadline_monotonic'
if time.monotonic() >= phase[key]:
return 'QUEUE_DEADLINE_EXCEEDED' if phase['state'] == 'QUEUED' else 'PHASE_EXECUTION_DEADLINE_EXCEEDED'
return None
def begin_phase(self, operation_id, phase, *, queued_monotonic=None):
with self.admission() as state:
if operation_id in state.get('phases', {}):
return state['phases'][operation_id] # Never reset a persisted phase clock.
policy = state['config']['phases'][phase]
started = time.monotonic() if queued_monotonic is None else queued_monotonic
row = dict(phase=phase, state='QUEUED', boot_id=boot_id(), epoch=state['epoch'],
queued_monotonic=started, queue_deadline_monotonic=started + policy['queue_deadline_s'],
execution_started_monotonic=None, execution_deadline_monotonic=None)
state.setdefault('phases', {})[operation_id] = row
self._save(state, 'PHASE_QUEUED', operation_id=operation_id, phase=phase)
return row
def phase_running(self, operation_id):
with self.admission() as state:
phase = state.get('phases', {}).get(operation_id)
if not phase or phase['state'] != 'QUEUED':
return
reason = self._phase_reason(state, operation_id)
if reason:
raise ExecutionStopped(reason)
now = time.monotonic()
phase.update(state='RUNNING', execution_started_monotonic=now,
execution_deadline_monotonic=now + state['config']['phases'][phase['phase']]['execution_deadline_s'])
state['stage_sequence'] = state.get('stage_sequence', 0) + 1
self._save(state, 'PHASE_STARTED', operation_id=operation_id, phase=phase['phase'])
def phase_done_locked(self, state, operation_id):
self.assert_owner(state)
if operation_id in state.get('phases', {}):
state['phases'][operation_id]['state'] = 'DONE'
state['stage_sequence'] = state.get('stage_sequence', 0) + 1
@contextlib.contextmanager
def admission(self):
"""Caller must spawn AND register within this interval; cancel uses this lock."""
with self.locked():
state = self.snapshot()
reason = self._reason(state, admission=True) or self._phase_reason(state, current_operation())
if reason:
raise ExecutionStopped(reason)
yield state
@contextlib.contextmanager
def mutation(self):
with self.locked():
state = self.snapshot()
self.assert_owner(state)
yield state
@contextlib.contextmanager
def scope(self, phase=None, operation_id=None):
old = (current(), current_phase(), getattr(_local, 'operation', None))
_local.execution, _local.phase, _local.operation = self, phase, operation_id
try:
yield self
finally:
_local.execution, _local.phase, _local.operation = old
def policy(self, phase=None):
config = self.snapshot()['config']
return config['phases'][phase] if phase in config.get('phases', {}) else config
def environment(self, env=None):
state = self.snapshot()
return dict(os.environ if env is None else env, BENCH_EXECUTION_DIR=str(self.rd),
BENCH_EXECUTION_EPOCH=str(state['epoch']), BENCH_EXECUTION_TOKEN=state['token'])
def register_locked(self, state, proc, role, scope_token):
identity = getattr(proc, 'leader', None) or process_identity(proc.pid)
state['processes'].append(dict(identity, role=role, scope_token=scope_token, epoch=state['epoch']))
self._save(state, 'PROCESS_STARTED', process=identity, role=role)
def register_scope_locked(self, state, scope, role):
state.setdefault('process_scopes', {})[scope.token] = {
'binding': scope.binding, 'receipt_path': str(scope.path)}
self.register_locked(state, scope.supervisor, 'supervisor', scope.token)
self.register_locked(state, scope, role, scope.token)
def record_process_set(self, token, leader, identities, unknown):
with self.mutation() as state:
recorded = {process_key(p) for p in state['processes']}
added = [dict(p, role='descendant', scope_token=token, epoch=state['epoch'])
for p in identities if process_key(p) not in recorded]
scope = {'leader': leader, 'unknown': unknown}
previous = state.setdefault('process_sets', {}).get(token)
if added or previous != scope:
state['processes'].extend(added)
state['process_sets'][token] = scope
self._save(state, 'PROCESS_SET_OBSERVED', scope_token=token, added=len(added), unknown=len(unknown))
def activity(self, name, **details):
with self.locked():
state = self.snapshot()
if self.epoch != state['epoch']:
raise ExecutionStopped('OWNER_FENCED')
if state['execution_state'] != 'TERMINAL':
state['activity'] = name
state['activity_details'] = details
return self._save(state, 'ACTIVITY_CHANGED', activity=name, details=details)
def observe_tools(self, events):
"""Persist typed SDK tool boundaries; a timeout fences all new admission."""
with self.locked():
state = self.snapshot()
self.assert_owner(state)
if state['execution_state'] != 'ACTIVE':
return state.get('stop_reason')
calls = state.setdefault('tool_calls', {})
changed = False
for event in events:
if event.get('execution_id') != state['execution_id'] or event.get('epoch') != state['epoch']:
raise ExecutionStopped('TOOL_EVENT_IDENTITY_MISMATCH')
key = event['tool_call_id']
row = calls.get(key)
if row is not None and any(event.get(field) != row[field] for field in
('started_monotonic', 'tool_name', 'cap_s')):
# Provider call IDs are not occurrence identities. Keep the
# first occurrence immutable and reject ambiguous reuse,
# including a late end that belongs to another occurrence.
reason = 'INFRA_TOOL_IDENTITY_CONFLICT'
state.update(execution_state='STOP_REQUESTED', cleanup_state='PENDING',
stop_reason=reason, tool_conflict=dict(event),
activity='RUNNING_TOOL', activity_details=dict(event))
self._save(state, reason)
return reason
if event['state'] == 'RUNNING' and row is None:
started, cap = event['started_monotonic'], event['cap_s']
if (not isinstance(key, str) or not key or not math.isfinite(started) or
not math.isfinite(cap) or not 0 < cap <= (state['config'].get('managed_tool_deadline_s', state['config']['execution_deadline_s']) if event.get('tool_name') == 'terminal_exec' else state['config'].get('ordinary_tool_deadline_s', 300)) or
not state['started_monotonic'] <= started <= time.monotonic() + 1):
raise ExecutionStopped('TOOL_EVENT_INVALID')
row = dict(tool_call_id=key, tool_name=event['tool_name'], state='RUNNING',
started_monotonic=started, deadline_monotonic=started + cap, cap_s=cap)
calls[key] = row
changed = True
elif event['state'] in ('DONE', 'TIMED_OUT') and row is not None:
# A late result cannot move the absolute deadline or erase its expiry.
row['state'] = ('TIMED_OUT' if row['state'] == 'TIMED_OUT' or event['state'] == 'TIMED_OUT' or
event['observed_monotonic'] >= row['deadline_monotonic'] else 'DONE')
changed = True
overdue = next((v for v in calls.values() if v['state'] == 'TIMED_OUT' or
(v['state'] == 'RUNNING' and time.monotonic() >= v['deadline_monotonic'])), None)
active = overdue or next((v for v in calls.values() if v['state'] == 'RUNNING'), None)
if active:
state.update(activity='RUNNING_TOOL', activity_details=dict(active))
elif changed and state['activity'] == 'RUNNING_TOOL':
state.update(activity='WAITING_LLM', activity_details={})
if overdue:
overdue['state'] = 'TIMED_OUT'
state.update(execution_state='STOP_REQUESTED', cleanup_state='PENDING',
stop_reason='TOOL_DEADLINE_EXCEEDED', activity_details=dict(overdue))
if changed or overdue:
self._save(state, 'TOOL_DEADLINE_EXCEEDED' if overdue else 'TOOL_OBSERVED')
return state.get('stop_reason')
def record_progress(self, source, signals, *, phase=None, operation_id=None, role='tool'):
"""One typed sample drives health and the read-only monitor; no log-text heuristics."""
with self.locked():
state = self.snapshot()
self.assert_owner(state)
now, wall = time.monotonic(), time.time()
progress = state.setdefault('progress', {})
previous = progress.setdefault('sources', {}).get(source, {})
old = previous.get('signals', {})
changes = {key: (None if value is None or old.get(key) is None else value != old[key])
for key, value in signals.items()}
unknown = any(value is None for value in signals.values())
changed = any(value is True for value in changes.values())
idle_since = None if unknown or changed else previous.get('idle_since_monotonic') or now
threshold = state['config']['phases'][phase]['suspect_after_s'] if phase in PHASES else state['config']['suspect_after_s']
phase_row = state.get('phases', {}).get(operation_id, {})
waiting = phase_row.get('state') == 'QUEUED' or (not operation_id and state['activity'] in
('QUEUED', 'WAITING_QUOTA', 'WAITING_RESOURCE'))
health = ('WAITING' if waiting else 'UNKNOWN' if unknown else
'SUSPECT' if idle_since is not None and now - idle_since >= threshold else 'OBSERVED')
sample = dict(sampled_at=wall, sampled_monotonic=now, signals=signals, changed=changes,
idle_since_monotonic=idle_since, suspect_after_s=threshold, health=health,
role=role, operation_id=operation_id, phase=phase,
reason='SIGNAL_UNKNOWN' if unknown else 'ALL_SIGNALS_IDLE' if health == 'SUSPECT' else 'PROGRESS_OBSERVED')
progress['sources'][source] = sample
active = {key for key, value in state.get('phases', {}).items() if value['state'] in ('QUEUED', 'RUNNING')}
# Agent silence while a separately owned stage is active is expected.
if role != 'agent' or not active:
progress.update(sampled_at=wall, current_source=source, **{k: sample[k] for k in
('signals', 'changed', 'idle_since_monotonic', 'suspect_after_s', 'reason')})
state['health'] = health
return self._save(state, 'PROGRESS_SAMPLED', source=source, health=health)
def cancel(self, reason):
if not isinstance(reason, str) or not reason.strip():
raise ValueError('cancel requires a reason')
with self.locked():
state = self.snapshot()
if self.epoch is None: # Explicit control-plane cancellation binds under the same lock.
self.epoch = state['epoch']
self.assert_owner(state)
if state['execution_state'] in ('TERMINAL', 'STOP_REQUESTED'):
return dict(state, reason='ALREADY_TERMINAL')
if not state['cancel_requested']:
state['cancel_requested'] = dict(reason=reason.strip(), time=time.time(), epoch=state['epoch'])
state.update(execution_state='CANCEL_REQUESTED', cleanup_state='PENDING', stop_reason='CANCEL_REQUESTED')
self._save(state, 'CANCEL_REQUESTED', reason=reason.strip())
return state
def quarantine(self, reason):
with self.locked():
state = self.snapshot()
self.assert_owner(state)
state.update(cleanup_state='QUARANTINED', stop_reason=reason)
if state['execution_state'] == 'ACTIVE':
state['execution_state'] = 'PAUSED_INFRA'
return self._save(state, 'RESOURCE_QUARANTINED', reason=reason)
def _checkpoint(self, value):
if not isinstance(value, dict) or not (value.get('resumable') is True and value.get('phase') == 'PAUSED_INFRA' or value.get('complete') is True and 'resumable' not in value):
raise ExecutionStopped('CHECKPOINT_INCOMPLETE')
supplied = Path(value.get('path', ''))
path = supplied.resolve()
if supplied.is_symlink() or not path.is_relative_to(self.rd) or not path.is_file():
raise ExecutionStopped('CHECKPOINT_PATH_INVALID')
digest = hashlib.sha256(path.read_bytes()).hexdigest()
if digest != value.get('sha256'):
raise ExecutionStopped('CHECKPOINT_HASH_MISMATCH')
return dict(value, path=str(path))
def pause(self, reason, checkpoint):
if not isinstance(reason, str) or not reason.strip():
raise ValueError('pause requires a reason')
with self.admission() as state:
checkpoint = self._checkpoint(checkpoint)
state.update(execution_state='PAUSED_INFRA', stop_reason=reason,
checkpoint=checkpoint, paused_at=time.time())
return self._save(state, 'EXECUTION_PAUSED', reason=reason)
def resume(self, *, automatic=True):
with self.locked():
state = self.snapshot()
if self.epoch is not None and self.epoch != state['epoch']:
raise ExecutionStopped('OWNER_FENCED')
resume_policy(state)
if state['execution_state'] != 'PAUSED_INFRA' or state.get('cancel_requested'):
raise ExecutionStopped('EXECUTION_NOT_RESUMABLE')
if state['cleanup_state'] == 'QUARANTINED':
raise ExecutionStopped('SIDE_EFFECT_UNKNOWN')
if state.get('boot_id') != boot_id():
raise ExecutionStopped('BOOT_CHANGED_RECONCILE_REQUIRED')
if time.monotonic() >= state['deadline_monotonic']:
raise ExecutionStopped('EXECUTION_DEADLINE_EXCEEDED')
self._checkpoint(state.get('checkpoint'))
owner_status = process_status(state['owner'])
same_owner = process_key(state['owner']) == process_key(process_identity(os.getpid()))
if owner_status != 'DEAD' and not same_owner:
raise ExecutionStopped('OWNER_NOT_RELEASABLE')
members = owned_processes(state)
pending = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE']
if members != [] or pending:
raise ExecutionStopped('SIDE_EFFECT_UNKNOWN' if members is None else 'WORK_STILL_ACTIVE')
count = state.get('auto_resume_count', 0)
if automatic and count >= state['config'].get('max_auto_resumes', 3):
raise ExecutionStopped('AUTO_RESUME_BUDGET_EXHAUSTED')
state['epoch'] += 1
self.epoch = state['epoch']
state.update(owner=process_identity(os.getpid()), execution_state='ACTIVE', stop_reason=None,
auto_resume_count=count + int(automatic), resumed_at=time.time(), activity='WAITING_QUOTA')
return self._save(state, 'EXECUTION_RESUMED', automatic=automatic)
def cleanup(self):
"""Bounded cancellation cleanup; unknown remote receipts keep leases isolated."""
with self.locked():
state = self.snapshot()
self.assert_owner(state)
if not state.get('cancel_requested'):
raise ExecutionStopped('CLEANUP_REQUIRES_CANCEL')
if state['execution_state'] == 'TERMINAL':
return state
state['execution_state'] = 'CLEANING'
self._save(state, 'CLEANUP_STARTED')
policy = state['config']
start = time.monotonic()
while True:
members = owned_processes(state)
latest = self.snapshot()
self.assert_owner(latest)
pending = [v for v in latest.get('operations', {}).values() if v.get('state') != 'DONE']
if members == [] and not pending:
break
elapsed = time.monotonic() - start
if members is None or elapsed >= policy['cleanup_deadline_s']:
return self.quarantine('CLEANUP_UNCONFIRMED')
# Each supervisor performs its own bounded TERM/KILL/reap sequence.
# Killing the reaper itself would destroy the complete-set proof.
for scope in latest.get('process_scopes', {}).values():
signal_process(scope['binding']['supervisor'], signal.SIGTERM)
time.sleep(.1)
with self.locked():
latest = self.snapshot()
self.assert_owner(latest)
if latest['cleanup_state'] != 'QUARANTINED':
latest['cleanup_state'] = 'VERIFIED'
return self._save(latest, 'CLEANUP_FINISHED', cleanup_state=latest['cleanup_state'])
def finish(self, exit_code, *, protocol_complete, stop_reason=None, cleanup_state='VERIFIED'):
with self.locked():
state = self.snapshot()
if self.epoch != state['epoch']:
raise ExecutionStopped('OWNER_FENCED')
if state['execution_state'] == 'TERMINAL':
return state
reason = stop_reason or state.get('stop_reason')
members = owned_processes(state) if state['processes'] else []
pending = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE']
known = [process_status(p) for p in state['processes']]
if (members not in ([], None) or pending or 'ALIVE' in known or
(Path('/proc').is_dir() and ('UNKNOWN' in known or members is None))):
cleanup_state = 'QUARANTINED'
reason = reason or 'CLEANUP_UNCONFIRMED'
if state['cleanup_state'] == 'QUARANTINED' or cleanup_state not in ('VERIFIED', 'NOT_NEEDED'):
cleanup_state = 'QUARANTINED'
if state['cancel_requested']:
outcome = 'CANCELLED'
elif reason == 'OWNER_LOST':
outcome = 'LOST'
elif reason and ('DEADLINE' in reason or 'hard_ceiling' in reason):
outcome = 'DEADLINE_EXCEEDED'
elif reason and 'BUDGET' in reason:
outcome = 'BUDGET_EXHAUSTED'
elif reason and any(word in reason for word in ('INFRA', 'UNKNOWN', 'monitor failed')):
outcome = 'INFRA_ERROR'
elif exit_code == 0 and protocol_complete and cleanup_state == 'VERIFIED':
outcome = 'COMPLETED'
else:
outcome = 'AGENT_ERROR'
state.update(execution_state='TERMINAL', execution_outcome=outcome, cleanup_state=cleanup_state,
stop_reason=reason, exit_code=exit_code, protocol_complete=bool(protocol_complete),
signal=(-exit_code if isinstance(exit_code, int) and exit_code < 0 else
exit_code - 128 if isinstance(exit_code, int) and exit_code > 128 else None),
finished_at=time.time())
return self._save(state, 'EXECUTION_FINISHED', outcome=outcome, reason=reason)
def reconcile(self):
"""Fail closed; never replay an unknown side effect or take a live owner."""
with self.locked():
state = self.snapshot()
if self.epoch is not None:
self.assert_owner(state)
if state['execution_state'] == 'TERMINAL':
return state
owner = process_status(state['owner'])
members = owned_processes(state)
operations = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE']
if owner == 'ALIVE':
return dict(state, reconciliation='OWNER_ALIVE')
if state['execution_state'] == 'PAUSED_INFRA' and owner == 'DEAD' and members == [] and not operations:
try:
self._checkpoint(state.get('checkpoint'))
except ExecutionStopped:
pass
else:
return dict(state, reconciliation='CHECKPOINT_READY_FOR_RESUME')
state['epoch'] += 1 # Fence every old dispatch path before changing ownership.
self.epoch = state['epoch']
if owner == 'DEAD' and members == [] and not operations:
state.update(execution_state='TERMINAL', execution_outcome='LOST', cleanup_state='VERIFIED',
stop_reason='OWNER_LOST', finished_at=time.time())
else:
state.update(execution_state='PAUSED_INFRA', cleanup_state='QUARANTINED',
stop_reason='SIDE_EFFECT_UNKNOWN' if operations else 'OWNERSHIP_UNKNOWN')
return self._save(state, 'EXECUTION_RECONCILED', owner_status=owner,
members=members, pending_operations=len(operations))
def from_environment():
value = os.environ.get('BENCH_EXECUTION_DIR')
if not value:
return None
result = Execution(value)
result.epoch = int(os.environ['BENCH_EXECUTION_EPOCH'])
return result