data-gen-bpu/harness/execution.py
bench dcf976dc95 recovery: hand a stopped collector's downtime back to its rollouts
A rollout whose collector died (handover, crash) is paused on the next start
(CONTROLLER_RESTARTED) with paused_at taken from execution.json's updated_at.
But reconcile_run first settles interrupted operations through the stage
Worker, which rewrites execution.json, so updated_at was already "now": no
downtime was handed back and any stop longer than the remaining budget timed
the rollout out the moment it resumed. Those timeouts look like real 24 h runs
to the sibling rule, so their requeued attempts were never claimed either
(11-06..08: a 33 h stop, 10 of 12 Qwen rollouts lost).

reconcile() now takes owner_lost_at (the dead owner's last write, read before
the caller's own saves) and adds the whole downtime to the deadline, uncapped:
the paused-credit cap is for pauses that may never resume, while a restarted
collector resumes at once. downtime_s records it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-08 08:33:08 +08:00

788 lines
41 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Small durable lifecycle ledger. One lock serializes cancel, spawn and finish.
Only Linux /proc identities authorize signals. Unknown ownership is quarantined,
never inferred from a PID or from an old RUNNING label.
"""
from __future__ import annotations
import contextlib
import fcntl
import hashlib
import json
import math
import os
import socket
import signal
import tempfile
import threading
import time
import uuid
from pathlib import Path
PHASES = ('01-fetch', '02-baseline', '03-export', '04-calibrate', '05-compile',
'06-hbm', '07-board', '08-eval')
POLICY_FIELDS = ('queue_deadline_s', 'execution_deadline_s', 'suspect_after_s',
'cancel_grace_s', 'cleanup_deadline_s')
_local = threading.local()
class ExecutionStopped(RuntimeError):
pass
def atomic_json(path: Path, value):
path.parent.mkdir(parents=True, exist_ok=True)
fd, name = tempfile.mkstemp(prefix='.' + path.name + '-', dir=path.parent)
try:
with os.fdopen(fd, 'w') as out:
json.dump(value, out, ensure_ascii=False, sort_keys=True)
out.flush(); os.fsync(out.fileno())
os.replace(name, path)
directory = os.open(path.parent, os.O_RDONLY)
try:
os.fsync(directory)
finally:
os.close(directory)
finally:
if os.path.exists(name):
os.unlink(name)
def boot_id():
try:
return Path('/proc/sys/kernel/random/boot_id').read_text().strip()
except OSError:
return None
def process_identity(pid: int) -> dict:
result = dict(host=socket.gethostname(), boot_id=boot_id(), pid=pid)
try:
raw = Path(f'/proc/{pid}/stat').read_bytes()
row = raw.rsplit(b')', 1)[1].split()
result.update(proc_start_ticks=int(row[19]), ppid=int(row[1]), pgrp=int(row[2]),
session=int(row[3]), state=row[0].decode('ascii'), cpu_ticks=int(row[11]) + int(row[12]), status='KNOWN')
except (OSError, ValueError, IndexError):
result['status'] = 'UNKNOWN'
return result
def earlier_boot(identity: dict) -> bool:
"""This host rebooted since the identity was taken: nothing of that boot still runs."""
return bool(identity.get('host') == socket.gethostname() and identity.get('boot_id') and boot_id()
and identity['boot_id'] != boot_id())
def process_status(identity: dict) -> str:
if earlier_boot(identity):
return 'DEAD'
if (identity.get('host') != socket.gethostname() or not identity.get('boot_id')
or identity.get('boot_id') != boot_id() or identity.get('proc_start_ticks') is None):
return 'UNKNOWN'
current = process_identity(identity['pid'])
if current['status'] == 'UNKNOWN':
return 'DEAD' if not Path(f"/proc/{identity['pid']}").exists() else 'UNKNOWN'
if current['proc_start_ticks'] != identity['proc_start_ticks'] or current['state'] == 'Z':
return 'DEAD'
return 'ALIVE'
def signal_process(identity, sig):
"""A pidfd closes the identity-check/kill PID-reuse race on supported Linux."""
if process_status(identity) != 'ALIVE':
return process_status(identity) == 'DEAD'
fd = None
try:
if hasattr(os, 'pidfd_open') and hasattr(signal, 'pidfd_send_signal'):
fd = os.pidfd_open(identity['pid'])
if process_status(identity) != 'ALIVE':
return True
signal.pidfd_send_signal(fd, sig)
else:
# Refuse rather than authorize a signal from an unverifiable identifier.
return False
return True
except ProcessLookupError:
return True
except (PermissionError, OSError):
return False
finally:
if fd is not None:
os.close(fd)
def process_key(identity):
return tuple(identity.get(k) for k in ('host', 'boot_id', 'pid', 'proc_start_ticks'))
def marker_processes(key, value):
"""Enumerate /proc only; an unguessable inherited marker joins detached children."""
proc = Path('/proc')
if not proc.is_dir():
return None
needle = f'{key}={value}'.encode()
result = []
for entry in proc.iterdir():
if not entry.name.isdigit():
continue
try:
fields = (entry / 'environ').read_bytes().split(b'\0')
matches = needle in fields
if key == 'BENCH_PROCESS_ANCESTRY':
prefix = (key + '=').encode()
matches = any(value.encode() in field[len(prefix):].split(b',')
for field in fields if field.startswith(prefix))
if matches:
identity = process_identity(int(entry.name))
if identity['status'] == 'KNOWN' and identity['state'] != 'Z':
result.append(identity)
except (OSError, ValueError):
continue
return result
def owned_processes(state):
"""Incomplete containment receipts remain unknown even with no visible PID."""
members = {}
for identity in state.get('processes', []):
if process_status(identity) != 'DEAD':
members[process_key(identity)] = dict(identity, ownership_verified=True)
scopes = state.get('process_scopes', {})
for token in state.get('process_sets', {}):
if token not in scopes:
return None # Legacy observations cannot prove a complete owned set.
for scope in scopes.values():
if not scope_cleaned(scope, scopes, state.get('owner')) and process_status(scope['binding']['supervisor']) != 'ALIVE':
return None
return list(members.values())
def scope_cleaned(scope, scopes, owner=None) -> bool:
"""One rule for every caller that asks whether a process scope left nothing behind."""
receipt = scope_receipt(scope)
if receipt and receipt['cleanup_state'] == 'VERIFIED':
return True
supervisor = scope['binding']['supervisor']
if earlier_boot(supervisor):
return True
# The execution's owner (the collector) itself died: crash, kill or planned handover. Its own top-level reaper
# killed the command and still proved that subtree empty with waitpid(ECHILD). A nested reaper whose parent is
# another scope's leader keeps needing that parent's receipt below. The owner is the one that started the scope
# (recorded with it): a resumed execution has a new owner, its earlier scopes keep theirs.
owner = scope.get('owner') or owner
if (owner and receipt and receipt.get('all_children_reaped') is True and receipt.get('owner_lost') is True
and receipt.get('supervisor_error') is None
and (supervisor.get('ppid'), supervisor.get('host'), supervisor.get('boot_id'))
== (owner.get('pid'), owner.get('host'), owner.get('boot_id'))):
return True
# A nested reaper may lose its parent during cancellation even though
# both levels finish waitpid(ECHILD). The enclosing verified receipt
# must name this exact supervisor, including boot/start identity.
if not (receipt and receipt.get('cleanup_state') == 'QUARANTINED'
and receipt.get('all_children_reaped') is True
and receipt.get('owner_lost') is True
and receipt.get('supervisor_error') is None):
return False
supervisor = scope['binding']['supervisor']
for parent in scopes.values():
proof = scope_receipt(parent)
binding = parent['binding']
if (proof and proof['cleanup_state'] == 'VERIFIED'
and binding.get('execution_id') == scope['binding'].get('execution_id')
and binding.get('epoch') == scope['binding'].get('epoch')
and supervisor.get('ppid') == binding['leader'].get('pid')
and process_key(supervisor) in {process_key(p) for p in proof.get('members', [])}):
return True
return False
def scope_receipt(scope):
try:
receipt = json.loads(Path(scope['receipt_path']).read_text())
if (receipt.get('binding') != scope['binding'] or
scope['binding'].get('proof') != 'waitpid:ECHILD' or
not isinstance(receipt.get('exit_code'), int)):
return None
if receipt.get('cleanup_state') == 'VERIFIED' and (
receipt.get('all_children_reaped') is not True or receipt.get('owner_lost') is not False):
return None
return receipt
except (OSError, ValueError, KeyError, TypeError):
return None
def validate_policy(config):
if not isinstance(config, dict) or not config.get('version'):
raise ValueError('LIFECYCLE_CONFIG_INCOMPLETE: version')
config = json.loads(json.dumps(config))
config.setdefault('phases', {}).setdefault('terminal', {field: config.get(field) for field in POLICY_FIELDS})
for name, row in [('terminal', config['phases']['terminal']), ('execution', config), *[(s, config.get('phases', {}).get(s)) for s in PHASES]]:
if not isinstance(row, dict):
raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}')
for field in POLICY_FIELDS:
value = row.get(field)
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0:
raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}.{field}')
if row['cleanup_deadline_s'] < row['cancel_grace_s']:
raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}.cleanup_deadline_s')
config = json.loads(json.dumps(config))
cap = config.setdefault('ordinary_tool_deadline_s', 300)
if isinstance(cap, bool) or not isinstance(cap, (int, float)) or not math.isfinite(cap) or cap <= 0:
raise ValueError('LIFECYCLE_CONFIG_INCOMPLETE: ordinary_tool_deadline_s')
return config
def resume_policy(state):
"""Reject an implicit policy upgrade before any epoch, quota or DB mutation."""
cap = state.get('config', {}).get('ordinary_tool_deadline_s')
if isinstance(cap, bool) or not isinstance(cap, (int, float)) or not math.isfinite(cap) or cap <= 0:
raise ExecutionStopped('RESUME_TOOL_POLICY_MISSING')
def paused_credit(state):
"""Time paused for infrastructure (quota, network) is not the agent's: it is handed back on resume, at most one
execution budget in total so a paused rollout that can never resume still ends. 10-14: three rollouts paused
7–16 h by GLM outages resumed with < 1 h left and were cut off mid-compile or mid-eval."""
paused = state.get('paused_at') if state['execution_state'] == 'PAUSED_INFRA' else None
if not paused:
return 0.0
return max(0.0, min(time.time() - paused, state['config']['execution_deadline_s'] - state.get('paused_s', 0.0)))
def current():
return getattr(_local, 'execution', None)
def current_phase():
return getattr(_local, 'phase', None)
def current_operation():
return getattr(_local, 'operation', None) or os.environ.get('BENCH_OPERATION_ID')
class Execution:
def __init__(self, rd):
self.rd = Path(rd).resolve()
self.path = self.rd / 'execution.json'
self.epoch = None
@contextlib.contextmanager
def locked(self):
self.rd.mkdir(parents=True, exist_ok=True)
with (self.rd / 'execution.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(lock, fcntl.LOCK_UN)
def snapshot(self):
return json.loads(self.path.read_text())
def assert_owner(self, state):
if self.epoch is None or self.epoch != state['epoch']:
raise ExecutionStopped('OWNER_FENCED')
def _save(self, state, event_type, **fields):
self.assert_owner(state)
state['sequence'] = state.get('sequence', 0) + 1
state['updated_at'] = time.time()
event = dict(event_id=f"{state['execution_id']}:{state['sequence']}",
execution_id=state['execution_id'], sequence=state['sequence'],
time=state['updated_at'], type=event_type, epoch=state['epoch'], **fields)
state['last_event'] = event
atomic_json(self.path, state)
with (self.rd / 'execution.events.jsonl').open('a') as out:
out.write(json.dumps(event, sort_keys=True) + '\n'); out.flush(); os.fsync(out.fileno())
return state
def start(self, config, activity='WAITING_LLM'):
config = validate_policy(config)
with self.locked():
if self.path.exists():
raise ExecutionStopped('EXECUTION_EXISTS: reconcile; never reset an existing deadline')
now = time.monotonic()
state = dict(execution_id=uuid.uuid4().hex, epoch=1, owner=process_identity(os.getpid()),
boot_id=boot_id(), started_at=time.time(), started_monotonic=now,
deadline_monotonic=now + config['execution_deadline_s'], config=config,
config_version=config['version'], config_sha256=hashlib.sha256(
json.dumps(config, sort_keys=True).encode()).hexdigest(),
activity=activity, execution_state='ACTIVE', execution_outcome=None,
cleanup_state='NOT_NEEDED', cancel_requested=None, stop_reason=None,
processes=[], token=uuid.uuid4().hex, operations={}, phases={}, stage_sequence=0,
progress={}, health='UNKNOWN', auto_resume_count=0)
self.epoch = state['epoch']
return self._save(state, 'EXECUTION_STARTED')
def attach(self):
self.epoch = self.snapshot()['epoch']
return self
def _reason(self, state, *, admission=False):
if self.epoch is not None and self.epoch != state['epoch']:
return 'OWNER_FENCED'
if state['cancel_requested']:
return 'CANCEL_REQUESTED'
if state['cleanup_state'] == 'QUARANTINED':
return state.get('stop_reason') or 'RESOURCE_QUARANTINED'
if state['execution_state'] != 'ACTIVE' and (admission or state['execution_state'] != 'PAUSED_INFRA'):
return state.get('stop_reason') or state['execution_state']
if state.get('boot_id') != boot_id():
# The old boot's monotonic clock means nothing now; judge a paused execution's deadline by wall time, or
# one already past it is resumed, refused by resume() and left paused, again and again (10-25: 7 rollouts
# looped ~43k times after the Mac mini hang).
budget = state['deadline_monotonic'] - state['started_monotonic']
if state['execution_state'] == 'PAUSED_INFRA' and time.time() >= state['started_at'] + budget + paused_credit(state):
return 'EXECUTION_DEADLINE_EXCEEDED'
return 'BOOT_CHANGED_RECONCILE_REQUIRED'
if time.monotonic() >= state['deadline_monotonic'] + paused_credit(state):
return 'EXECUTION_DEADLINE_EXCEEDED'
owner_status = process_status(state['owner'])
if owner_status == 'DEAD':
return 'OWNER_LOST'
if admission and owner_status != 'ALIVE':
return 'OWNER_STATUS_UNKNOWN'
return None
def stop_reason(self, operation_id=None):
state = self.snapshot()
return self._reason(state) or self._phase_reason(state, operation_id or current_operation())
def _phase_reason(self, state, operation_id):
phase = state.get('phases', {}).get(operation_id)
if not phase or phase['state'] == 'DONE':
return None
if phase['boot_id'] != boot_id():
return 'PHASE_CLOCK_UNKNOWN'
key = 'queue_deadline_monotonic' if phase['state'] == 'QUEUED' else 'execution_deadline_monotonic'
if time.monotonic() >= phase[key]:
return 'QUEUE_DEADLINE_EXCEEDED' if phase['state'] == 'QUEUED' else 'PHASE_EXECUTION_DEADLINE_EXCEEDED'
return None
def begin_phase(self, operation_id, phase, *, queued_monotonic=None):
with self.admission() as state:
if operation_id in state.get('phases', {}):
return state['phases'][operation_id] # Never reset a persisted phase clock.
policy = state['config']['phases'][phase]
started = time.monotonic() if queued_monotonic is None else queued_monotonic
row = dict(phase=phase, state='QUEUED', boot_id=boot_id(), epoch=state['epoch'],
queued_monotonic=started, queue_deadline_monotonic=started + policy['queue_deadline_s'],
execution_started_monotonic=None, execution_deadline_monotonic=None)
state.setdefault('phases', {})[operation_id] = row
self._save(state, 'PHASE_QUEUED', operation_id=operation_id, phase=phase)
return row
def phase_running(self, operation_id):
with self.admission() as state:
phase = state.get('phases', {}).get(operation_id)
if not phase or phase['state'] != 'QUEUED':
return
reason = self._phase_reason(state, operation_id)
if reason:
raise ExecutionStopped(reason)
now = time.monotonic()
phase.update(state='RUNNING', execution_started_monotonic=now,
execution_deadline_monotonic=now + state['config']['phases'][phase['phase']]['execution_deadline_s'])
state['stage_sequence'] = state.get('stage_sequence', 0) + 1
self._save(state, 'PHASE_STARTED', operation_id=operation_id, phase=phase['phase'])
def phase_done_locked(self, state, operation_id):
self.assert_owner(state)
if operation_id in state.get('phases', {}):
state['phases'][operation_id]['state'] = 'DONE'
state['stage_sequence'] = state.get('stage_sequence', 0) + 1
@contextlib.contextmanager
def admission(self):
"""Caller must spawn AND register within this interval; cancel uses this lock."""
with self.locked():
state = self.snapshot()
reason = self._reason(state, admission=True) or self._phase_reason(state, current_operation())
if reason:
raise ExecutionStopped(reason)
yield state
@contextlib.contextmanager
def mutation(self):
with self.locked():
state = self.snapshot()
self.assert_owner(state)
yield state
@contextlib.contextmanager
def scope(self, phase=None, operation_id=None):
old = (current(), current_phase(), getattr(_local, 'operation', None))
_local.execution, _local.phase, _local.operation = self, phase, operation_id
try:
yield self
finally:
_local.execution, _local.phase, _local.operation = old
def policy(self, phase=None):
config = self.snapshot()['config']
return config['phases'][phase] if phase in config.get('phases', {}) else config
def environment(self, env=None):
state = self.snapshot()
return dict(os.environ if env is None else env, BENCH_EXECUTION_DIR=str(self.rd),
BENCH_EXECUTION_EPOCH=str(state['epoch']), BENCH_EXECUTION_TOKEN=state['token'])
def register_locked(self, state, proc, role, scope_token):
identity = getattr(proc, 'leader', None) or process_identity(proc.pid)
state['processes'].append(dict(identity, role=role, scope_token=scope_token, epoch=state['epoch']))
self._save(state, 'PROCESS_STARTED', process=identity, role=role)
def register_scope_locked(self, state, scope, role):
state.setdefault('process_scopes', {})[scope.token] = {
'binding': scope.binding, 'receipt_path': str(scope.path), 'owner': state['owner']}
self.register_locked(state, scope.supervisor, 'supervisor', scope.token)
self.register_locked(state, scope, role, scope.token)
def record_process_set(self, token, leader, identities, unknown):
with self.mutation() as state:
recorded = {process_key(p) for p in state['processes']}
added = [dict(p, role='descendant', scope_token=token, epoch=state['epoch'])
for p in identities if process_key(p) not in recorded]
scope = {'leader': leader, 'unknown': unknown}
previous = state.setdefault('process_sets', {}).get(token)
if added or previous != scope:
state['processes'].extend(added)
state['process_sets'][token] = scope
self._save(state, 'PROCESS_SET_OBSERVED', scope_token=token, added=len(added), unknown=len(unknown))
def activity(self, name, **details):
with self.locked():
state = self.snapshot()
if self.epoch != state['epoch']:
raise ExecutionStopped('OWNER_FENCED')
if state['execution_state'] != 'TERMINAL':
state['activity'] = name
state['activity_details'] = details
return self._save(state, 'ACTIVITY_CHANGED', activity=name, details=details)
def observe_tools(self, events):
"""Persist typed SDK tool boundaries; a timeout fences all new admission."""
with self.locked():
state = self.snapshot()
self.assert_owner(state)
if state['execution_state'] != 'ACTIVE':
return state.get('stop_reason')
calls = state.setdefault('tool_calls', {})
changed = False
for event in events:
if event.get('execution_id') != state['execution_id'] or event.get('epoch') != state['epoch']:
raise ExecutionStopped('TOOL_EVENT_IDENTITY_MISMATCH')
key = event['tool_call_id']
row = calls.get(key)
if row is not None and any(event.get(field) != row[field] for field in
('started_monotonic', 'tool_name', 'cap_s')):
# Provider call IDs are not occurrence identities. Keep the
# first occurrence immutable and reject ambiguous reuse,
# including a late end that belongs to another occurrence.
reason = 'INFRA_TOOL_IDENTITY_CONFLICT'
state.update(execution_state='STOP_REQUESTED', cleanup_state='PENDING',
stop_reason=reason, tool_conflict=dict(event),
activity='RUNNING_TOOL', activity_details=dict(event))
self._save(state, reason)
return reason
if event['state'] == 'RUNNING' and row is None:
started, cap = event['started_monotonic'], event['cap_s']
if (not isinstance(key, str) or not key or not math.isfinite(started) or
not math.isfinite(cap) or not 0 < cap <= (state['config'].get('managed_tool_deadline_s', state['config']['execution_deadline_s']) if event.get('tool_name') == 'terminal_exec' else state['config'].get('ordinary_tool_deadline_s', 300)) or
not state['started_monotonic'] <= started <= time.monotonic() + 1):
raise ExecutionStopped('TOOL_EVENT_INVALID')
row = dict(tool_call_id=key, tool_name=event['tool_name'], state='RUNNING',
started_monotonic=started, deadline_monotonic=started + cap, cap_s=cap)
calls[key] = row
changed = True
elif event['state'] in ('DONE', 'TIMED_OUT') and row is not None:
# A late result cannot move the absolute deadline or erase its expiry.
row['state'] = ('TIMED_OUT' if row['state'] == 'TIMED_OUT' or event['state'] == 'TIMED_OUT' or
event['observed_monotonic'] >= row['deadline_monotonic'] else 'DONE')
changed = True
overdue = next((v for v in calls.values() if v['state'] == 'TIMED_OUT' or
(v['state'] == 'RUNNING' and time.monotonic() >= v['deadline_monotonic'])), None)
active = overdue or next((v for v in calls.values() if v['state'] == 'RUNNING'), None)
if active:
state.update(activity='RUNNING_TOOL', activity_details=dict(active))
elif changed and state['activity'] == 'RUNNING_TOOL':
state.update(activity='WAITING_LLM', activity_details={})
if overdue:
overdue['state'] = 'TIMED_OUT'
state.update(execution_state='STOP_REQUESTED', cleanup_state='PENDING',
stop_reason='TOOL_DEADLINE_EXCEEDED', activity_details=dict(overdue))
if changed or overdue:
self._save(state, 'TOOL_DEADLINE_EXCEEDED' if overdue else 'TOOL_OBSERVED')
return state.get('stop_reason')
def record_progress(self, source, signals, *, phase=None, operation_id=None, role='tool'):
"""One typed sample drives health and the read-only monitor; no log-text heuristics."""
with self.locked():
state = self.snapshot()
self.assert_owner(state)
now, wall = time.monotonic(), time.time()
progress = state.setdefault('progress', {})
previous = progress.setdefault('sources', {}).get(source, {})
old = previous.get('signals', {})
changes = {key: (None if value is None or old.get(key) is None else value != old[key])
for key, value in signals.items()}
unknown = any(value is None for value in signals.values())
changed = any(value is True for value in changes.values())
idle_since = None if unknown or changed else previous.get('idle_since_monotonic') or now
threshold = state['config']['phases'][phase]['suspect_after_s'] if phase in PHASES else state['config']['suspect_after_s']
phase_row = state.get('phases', {}).get(operation_id, {})
waiting = phase_row.get('state') == 'QUEUED' or (not operation_id and state['activity'] in
('QUEUED', 'WAITING_QUOTA', 'WAITING_RESOURCE'))
health = ('WAITING' if waiting else 'UNKNOWN' if unknown else
'SUSPECT' if idle_since is not None and now - idle_since >= threshold else 'OBSERVED')
sample = dict(sampled_at=wall, sampled_monotonic=now, signals=signals, changed=changes,
idle_since_monotonic=idle_since, suspect_after_s=threshold, health=health,
role=role, operation_id=operation_id, phase=phase,
reason='SIGNAL_UNKNOWN' if unknown else 'ALL_SIGNALS_IDLE' if health == 'SUSPECT' else 'PROGRESS_OBSERVED')
progress['sources'][source] = sample
active = {key for key, value in state.get('phases', {}).items() if value['state'] in ('QUEUED', 'RUNNING')}
# Agent silence while a separately owned stage is active is expected.
if role != 'agent' or not active:
progress.update(sampled_at=wall, current_source=source, **{k: sample[k] for k in
('signals', 'changed', 'idle_since_monotonic', 'suspect_after_s', 'reason')})
state['health'] = health
return self._save(state, 'PROGRESS_SAMPLED', source=source, health=health)
def cancel(self, reason):
if not isinstance(reason, str) or not reason.strip():
raise ValueError('cancel requires a reason')
with self.locked():
state = self.snapshot()
if self.epoch is None: # Explicit control-plane cancellation binds under the same lock.
self.epoch = state['epoch']
self.assert_owner(state)
if state['execution_state'] in ('TERMINAL', 'STOP_REQUESTED'):
return dict(state, reason='ALREADY_TERMINAL')
if not state['cancel_requested']:
state['cancel_requested'] = dict(reason=reason.strip(), time=time.time(), epoch=state['epoch'])
state.update(execution_state='CANCEL_REQUESTED', cleanup_state='PENDING', stop_reason='CANCEL_REQUESTED')
self._save(state, 'CANCEL_REQUESTED', reason=reason.strip())
return state
def quarantine(self, reason):
with self.locked():
state = self.snapshot()
self.assert_owner(state)
state.update(cleanup_state='QUARANTINED', stop_reason=reason)
if state['execution_state'] == 'ACTIVE':
state['execution_state'] = 'PAUSED_INFRA'
return self._save(state, 'RESOURCE_QUARANTINED', reason=reason)
def _checkpoint(self, value):
if not isinstance(value, dict) or not (value.get('resumable') is True and value.get('phase') == 'PAUSED_INFRA' or value.get('complete') is True and 'resumable' not in value):
raise ExecutionStopped('CHECKPOINT_INCOMPLETE')
supplied = Path(value.get('path', ''))
path = supplied.resolve()
if supplied.is_symlink() or not path.is_relative_to(self.rd) or not path.is_file():
raise ExecutionStopped('CHECKPOINT_PATH_INVALID')
digest = hashlib.sha256(path.read_bytes()).hexdigest()
if digest != value.get('sha256'):
raise ExecutionStopped('CHECKPOINT_HASH_MISMATCH')
return dict(value, path=str(path))
def pause(self, reason, checkpoint):
if not isinstance(reason, str) or not reason.strip():
raise ValueError('pause requires a reason')
with self.admission() as state:
checkpoint = self._checkpoint(checkpoint)
state.update(execution_state='PAUSED_INFRA', stop_reason=reason,
checkpoint=checkpoint, paused_at=time.time())
return self._save(state, 'EXECUTION_PAUSED', reason=reason)
def resume(self, *, automatic=True, counted=True):
with self.locked():
state = self.snapshot()
if self.epoch is not None and self.epoch != state['epoch']:
raise ExecutionStopped('OWNER_FENCED')
resume_policy(state)
if state['execution_state'] != 'PAUSED_INFRA' or state.get('cancel_requested'):
raise ExecutionStopped('EXECUTION_NOT_RESUMABLE')
if state['cleanup_state'] == 'QUARANTINED':
raise ExecutionStopped('SIDE_EFFECT_UNKNOWN')
if state.get('boot_id') != boot_id():
if not earlier_boot(state['owner']):
raise ExecutionStopped('BOOT_CHANGED_RECONCILE_REQUIRED')
self._rebase_clock(state)
credit = paused_credit(state)
state['deadline_monotonic'] += credit
state['paused_s'] = state.get('paused_s', 0.0) + credit
state.pop('paused_at', None)
if time.monotonic() >= state['deadline_monotonic']:
raise ExecutionStopped('EXECUTION_DEADLINE_EXCEEDED')
self._checkpoint(state.get('checkpoint'))
owner_status = process_status(state['owner'])
same_owner = process_key(state['owner']) == process_key(process_identity(os.getpid()))
if owner_status != 'DEAD' and not same_owner:
raise ExecutionStopped('OWNER_NOT_RELEASABLE')
members = owned_processes(state)
pending = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE']
if members != [] or pending:
raise ExecutionStopped('SIDE_EFFECT_UNKNOWN' if members is None else 'WORK_STILL_ACTIVE')
count = state.get('auto_resume_count', 0)
if automatic and counted and count >= state['config'].get('max_auto_resumes', 3):
raise ExecutionStopped('AUTO_RESUME_BUDGET_EXHAUSTED')
state['epoch'] += 1
self.epoch = state['epoch']
state.update(owner=process_identity(os.getpid()), execution_state='ACTIVE', stop_reason=None,
auto_resume_count=count + int(automatic and counted), resumed_at=time.time(), activity='WAITING_QUOTA')
return self._save(state, 'EXECUTION_RESUMED', automatic=automatic)
@staticmethod
def _rebase_clock(state):
"""Monotonic clocks restart with the host. Carry every deadline over by wall time; the downtime itself is
a pause (paused_at), credited back like any other infrastructure pause."""
offset = (time.monotonic() - time.time()) - (state['started_monotonic'] - state['started_at'])
state['started_monotonic'] += offset
state['deadline_monotonic'] += offset
for row in [*state.get('phases', {}).values(), *state.get('tool_calls', {}).values()]:
for key, value in row.items():
if key.endswith('_monotonic') and isinstance(value, (int, float)):
row[key] = value + offset
if 'boot_id' in row:
row['boot_id'] = boot_id()
state['boot_id'] = boot_id()
def cleanup(self):
"""Bounded cancellation cleanup; unknown remote receipts keep leases isolated."""
with self.locked():
state = self.snapshot()
self.assert_owner(state)
if not state.get('cancel_requested'):
raise ExecutionStopped('CLEANUP_REQUIRES_CANCEL')
if state['execution_state'] == 'TERMINAL':
return state
state['execution_state'] = 'CLEANING'
self._save(state, 'CLEANUP_STARTED')
policy = state['config']
start = time.monotonic()
while True:
members = owned_processes(state)
latest = self.snapshot()
self.assert_owner(latest)
pending = [v for v in latest.get('operations', {}).values() if v.get('state') != 'DONE']
if members == [] and not pending:
break
elapsed = time.monotonic() - start
if members is None or elapsed >= policy['cleanup_deadline_s']:
return self.quarantine('CLEANUP_UNCONFIRMED')
# Each supervisor performs its own bounded TERM/KILL/reap sequence.
# Killing the reaper itself would destroy the complete-set proof.
for scope in latest.get('process_scopes', {}).values():
signal_process(scope['binding']['supervisor'], signal.SIGTERM)
time.sleep(.1)
with self.locked():
latest = self.snapshot()
self.assert_owner(latest)
if latest['cleanup_state'] != 'QUARANTINED':
latest['cleanup_state'] = 'VERIFIED'
return self._save(latest, 'CLEANUP_FINISHED', cleanup_state=latest['cleanup_state'])
def finish(self, exit_code, *, protocol_complete, stop_reason=None, cleanup_state='VERIFIED'):
with self.locked():
state = self.snapshot()
if self.epoch != state['epoch']:
raise ExecutionStopped('OWNER_FENCED')
if state['execution_state'] == 'TERMINAL':
return state
reason = stop_reason or state.get('stop_reason')
members = owned_processes(state) if state['processes'] else []
pending = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE']
known = [process_status(p) for p in state['processes']]
if (members not in ([], None) or pending or 'ALIVE' in known or
(Path('/proc').is_dir() and ('UNKNOWN' in known or members is None))):
cleanup_state = 'QUARANTINED'
reason = reason or 'CLEANUP_UNCONFIRMED'
if state['cleanup_state'] == 'QUARANTINED' or cleanup_state not in ('VERIFIED', 'NOT_NEEDED'):
cleanup_state = 'QUARANTINED'
if state['cancel_requested']:
outcome = 'CANCELLED'
elif reason == 'OWNER_LOST':
outcome = 'LOST'
elif reason and ('DEADLINE' in reason or 'hard_ceiling' in reason):
outcome = 'DEADLINE_EXCEEDED'
elif reason and 'BUDGET' in reason:
outcome = 'BUDGET_EXHAUSTED'
# CLEANUP_UNCONFIRMED: a board or sandbox we could not reach, never the agent (10-27/28: 13 as AGENT_ERROR).
elif reason and any(word in reason for word in ('INFRA', 'UNKNOWN', 'monitor failed', 'CLEANUP_UNCONFIRMED')):
outcome = 'INFRA_ERROR'
elif exit_code == 0 and protocol_complete and cleanup_state == 'VERIFIED':
outcome = 'COMPLETED'
else:
outcome = 'AGENT_ERROR'
state.update(execution_state='TERMINAL', execution_outcome=outcome, cleanup_state=cleanup_state,
stop_reason=reason, exit_code=exit_code, protocol_complete=bool(protocol_complete),
signal=(-exit_code if isinstance(exit_code, int) and exit_code < 0 else
exit_code - 128 if isinstance(exit_code, int) and exit_code > 128 else None),
finished_at=time.time())
return self._save(state, 'EXECUTION_FINISHED', outcome=outcome, reason=reason)
def reconcile(self, crash_checkpoint=None, owner_lost_at=None):
"""Fail closed; never replay an unknown side effect or take a live owner.
crash_checkpoint: the agent's last durable session boundary, frozen by the caller after its owner (the
collector) died with every process proven gone and every operation settled. The rollout then pauses
(CONTROLLER_RESTARTED) and resumes from there instead of being lost.
owner_lost_at: when the dead owner last wrote this state, read before the caller's own saves."""
with self.locked():
state = self.snapshot()
if self.epoch is not None:
self.assert_owner(state)
if state['execution_state'] == 'TERMINAL':
return state
owner = process_status(state['owner'])
members = owned_processes(state)
operations = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE']
if owner == 'ALIVE':
return dict(state, reconciliation='OWNER_ALIVE')
if state['execution_state'] == 'PAUSED_INFRA' and owner == 'DEAD' and members == [] and not operations:
try:
self._checkpoint(state.get('checkpoint'))
except ExecutionStopped:
pass
else:
return dict(state, reconciliation='CHECKPOINT_READY_FOR_RESUME')
state['epoch'] += 1 # Fence every old dispatch path before changing ownership.
self.epoch = state['epoch']
restartable = (owner == 'DEAD' and members == [] and not operations and crash_checkpoint is not None
and state['execution_state'] == 'ACTIVE' and not state.get('cancel_requested'))
if restartable:
checkpoint = self._checkpoint(crash_checkpoint)
for call in state.get('tool_calls', {}).values():
if call['state'] == 'RUNNING':
call['state'] = 'INTERRUPTED' # it died with its owner; the resumed agent is told so
for scope in state.get('process_scopes', {}).values():
scope.setdefault('owner', state['owner']) # scopes registered before owners were recorded
# No collector ran, so none of the downtime was the agent's: hand all of it back now, uncapped (the
# paused-credit cap is for pauses that may never resume). 11-06..08: a 33 h planned stop, then
# updated_at refreshed by the restart's own settling, timed out 10 rollouts the moment they resumed.
downtime = max(0.0, time.time() - (owner_lost_at or state.get('updated_at') or time.time()))
state['deadline_monotonic'] += downtime
state['downtime_s'] = state.get('downtime_s', 0.0) + downtime
state.update(execution_state='PAUSED_INFRA', stop_reason='CONTROLLER_RESTARTED', checkpoint=checkpoint,
paused_at=time.time(), activity='WAITING_QUOTA', activity_details={})
return self._save(state, 'EXECUTION_RECONCILED', owner_status=owner, members=members,
pending_operations=0, reconciliation='CONTROLLER_RESTARTED')
if owner == 'DEAD' and members == [] and not operations:
state.update(execution_state='TERMINAL', execution_outcome='LOST', cleanup_state='VERIFIED',
stop_reason='OWNER_LOST', finished_at=time.time())
else:
state.update(execution_state='PAUSED_INFRA', cleanup_state='QUARANTINED',
stop_reason='SIDE_EFFECT_UNKNOWN' if operations else 'OWNERSHIP_UNKNOWN')
return self._save(state, 'EXECUTION_RECONCILED', owner_status=owner,
members=members, pending_operations=len(operations))
def from_environment():
value = os.environ.get('BENCH_EXECUTION_DIR')
if not value:
return None
result = Execution(value)
result.epoch = int(os.environ['BENCH_EXECUTION_EPOCH'])
return result