A rollout whose collector died (handover, crash) is paused on the next start (CONTROLLER_RESTARTED) with paused_at taken from execution.json's updated_at. But reconcile_run first settles interrupted operations through the stage Worker, which rewrites execution.json, so updated_at was already "now": no downtime was handed back and any stop longer than the remaining budget timed the rollout out the moment it resumed. Those timeouts look like real 24 h runs to the sibling rule, so their requeued attempts were never claimed either (11-06..08: a 33 h stop, 10 of 12 Qwen rollouts lost). reconcile() now takes owner_lost_at (the dead owner's last write, read before the caller's own saves) and adds the whole downtime to the deadline, uncapped: the paused-credit cap is for pauses that may never resume, while a restarted collector resumes at once. downtime_s records it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
788 lines
41 KiB
Python
788 lines
41 KiB
Python
"""Small durable lifecycle ledger. One lock serializes cancel, spawn and finish.
|
||
|
||
Only Linux /proc identities authorize signals. Unknown ownership is quarantined,
|
||
never inferred from a PID or from an old RUNNING label.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import contextlib
|
||
import fcntl
|
||
import hashlib
|
||
import json
|
||
import math
|
||
import os
|
||
import socket
|
||
import signal
|
||
import tempfile
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from pathlib import Path
|
||
|
||
PHASES = ('01-fetch', '02-baseline', '03-export', '04-calibrate', '05-compile',
|
||
'06-hbm', '07-board', '08-eval')
|
||
POLICY_FIELDS = ('queue_deadline_s', 'execution_deadline_s', 'suspect_after_s',
|
||
'cancel_grace_s', 'cleanup_deadline_s')
|
||
_local = threading.local()
|
||
|
||
|
||
class ExecutionStopped(RuntimeError):
|
||
pass
|
||
|
||
|
||
def atomic_json(path: Path, value):
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
fd, name = tempfile.mkstemp(prefix='.' + path.name + '-', dir=path.parent)
|
||
try:
|
||
with os.fdopen(fd, 'w') as out:
|
||
json.dump(value, out, ensure_ascii=False, sort_keys=True)
|
||
out.flush(); os.fsync(out.fileno())
|
||
os.replace(name, path)
|
||
directory = os.open(path.parent, os.O_RDONLY)
|
||
try:
|
||
os.fsync(directory)
|
||
finally:
|
||
os.close(directory)
|
||
finally:
|
||
if os.path.exists(name):
|
||
os.unlink(name)
|
||
|
||
|
||
def boot_id():
|
||
try:
|
||
return Path('/proc/sys/kernel/random/boot_id').read_text().strip()
|
||
except OSError:
|
||
return None
|
||
|
||
|
||
def process_identity(pid: int) -> dict:
|
||
result = dict(host=socket.gethostname(), boot_id=boot_id(), pid=pid)
|
||
try:
|
||
raw = Path(f'/proc/{pid}/stat').read_bytes()
|
||
row = raw.rsplit(b')', 1)[1].split()
|
||
result.update(proc_start_ticks=int(row[19]), ppid=int(row[1]), pgrp=int(row[2]),
|
||
session=int(row[3]), state=row[0].decode('ascii'), cpu_ticks=int(row[11]) + int(row[12]), status='KNOWN')
|
||
except (OSError, ValueError, IndexError):
|
||
result['status'] = 'UNKNOWN'
|
||
return result
|
||
|
||
|
||
def earlier_boot(identity: dict) -> bool:
|
||
"""This host rebooted since the identity was taken: nothing of that boot still runs."""
|
||
return bool(identity.get('host') == socket.gethostname() and identity.get('boot_id') and boot_id()
|
||
and identity['boot_id'] != boot_id())
|
||
|
||
|
||
def process_status(identity: dict) -> str:
|
||
if earlier_boot(identity):
|
||
return 'DEAD'
|
||
if (identity.get('host') != socket.gethostname() or not identity.get('boot_id')
|
||
or identity.get('boot_id') != boot_id() or identity.get('proc_start_ticks') is None):
|
||
return 'UNKNOWN'
|
||
current = process_identity(identity['pid'])
|
||
if current['status'] == 'UNKNOWN':
|
||
return 'DEAD' if not Path(f"/proc/{identity['pid']}").exists() else 'UNKNOWN'
|
||
if current['proc_start_ticks'] != identity['proc_start_ticks'] or current['state'] == 'Z':
|
||
return 'DEAD'
|
||
return 'ALIVE'
|
||
|
||
|
||
def signal_process(identity, sig):
|
||
"""A pidfd closes the identity-check/kill PID-reuse race on supported Linux."""
|
||
if process_status(identity) != 'ALIVE':
|
||
return process_status(identity) == 'DEAD'
|
||
fd = None
|
||
try:
|
||
if hasattr(os, 'pidfd_open') and hasattr(signal, 'pidfd_send_signal'):
|
||
fd = os.pidfd_open(identity['pid'])
|
||
if process_status(identity) != 'ALIVE':
|
||
return True
|
||
signal.pidfd_send_signal(fd, sig)
|
||
else:
|
||
# Refuse rather than authorize a signal from an unverifiable identifier.
|
||
return False
|
||
return True
|
||
except ProcessLookupError:
|
||
return True
|
||
except (PermissionError, OSError):
|
||
return False
|
||
finally:
|
||
if fd is not None:
|
||
os.close(fd)
|
||
|
||
|
||
def process_key(identity):
|
||
return tuple(identity.get(k) for k in ('host', 'boot_id', 'pid', 'proc_start_ticks'))
|
||
|
||
|
||
def marker_processes(key, value):
|
||
"""Enumerate /proc only; an unguessable inherited marker joins detached children."""
|
||
proc = Path('/proc')
|
||
if not proc.is_dir():
|
||
return None
|
||
needle = f'{key}={value}'.encode()
|
||
result = []
|
||
for entry in proc.iterdir():
|
||
if not entry.name.isdigit():
|
||
continue
|
||
try:
|
||
fields = (entry / 'environ').read_bytes().split(b'\0')
|
||
matches = needle in fields
|
||
if key == 'BENCH_PROCESS_ANCESTRY':
|
||
prefix = (key + '=').encode()
|
||
matches = any(value.encode() in field[len(prefix):].split(b',')
|
||
for field in fields if field.startswith(prefix))
|
||
if matches:
|
||
identity = process_identity(int(entry.name))
|
||
if identity['status'] == 'KNOWN' and identity['state'] != 'Z':
|
||
result.append(identity)
|
||
except (OSError, ValueError):
|
||
continue
|
||
return result
|
||
|
||
|
||
def owned_processes(state):
|
||
"""Incomplete containment receipts remain unknown even with no visible PID."""
|
||
members = {}
|
||
for identity in state.get('processes', []):
|
||
if process_status(identity) != 'DEAD':
|
||
members[process_key(identity)] = dict(identity, ownership_verified=True)
|
||
scopes = state.get('process_scopes', {})
|
||
for token in state.get('process_sets', {}):
|
||
if token not in scopes:
|
||
return None # Legacy observations cannot prove a complete owned set.
|
||
for scope in scopes.values():
|
||
if not scope_cleaned(scope, scopes, state.get('owner')) and process_status(scope['binding']['supervisor']) != 'ALIVE':
|
||
return None
|
||
return list(members.values())
|
||
|
||
|
||
def scope_cleaned(scope, scopes, owner=None) -> bool:
|
||
"""One rule for every caller that asks whether a process scope left nothing behind."""
|
||
receipt = scope_receipt(scope)
|
||
if receipt and receipt['cleanup_state'] == 'VERIFIED':
|
||
return True
|
||
supervisor = scope['binding']['supervisor']
|
||
if earlier_boot(supervisor):
|
||
return True
|
||
# The execution's owner (the collector) itself died: crash, kill or planned handover. Its own top-level reaper
|
||
# killed the command and still proved that subtree empty with waitpid(ECHILD). A nested reaper whose parent is
|
||
# another scope's leader keeps needing that parent's receipt below. The owner is the one that started the scope
|
||
# (recorded with it): a resumed execution has a new owner, its earlier scopes keep theirs.
|
||
owner = scope.get('owner') or owner
|
||
if (owner and receipt and receipt.get('all_children_reaped') is True and receipt.get('owner_lost') is True
|
||
and receipt.get('supervisor_error') is None
|
||
and (supervisor.get('ppid'), supervisor.get('host'), supervisor.get('boot_id'))
|
||
== (owner.get('pid'), owner.get('host'), owner.get('boot_id'))):
|
||
return True
|
||
# A nested reaper may lose its parent during cancellation even though
|
||
# both levels finish waitpid(ECHILD). The enclosing verified receipt
|
||
# must name this exact supervisor, including boot/start identity.
|
||
if not (receipt and receipt.get('cleanup_state') == 'QUARANTINED'
|
||
and receipt.get('all_children_reaped') is True
|
||
and receipt.get('owner_lost') is True
|
||
and receipt.get('supervisor_error') is None):
|
||
return False
|
||
supervisor = scope['binding']['supervisor']
|
||
for parent in scopes.values():
|
||
proof = scope_receipt(parent)
|
||
binding = parent['binding']
|
||
if (proof and proof['cleanup_state'] == 'VERIFIED'
|
||
and binding.get('execution_id') == scope['binding'].get('execution_id')
|
||
and binding.get('epoch') == scope['binding'].get('epoch')
|
||
and supervisor.get('ppid') == binding['leader'].get('pid')
|
||
and process_key(supervisor) in {process_key(p) for p in proof.get('members', [])}):
|
||
return True
|
||
return False
|
||
|
||
|
||
def scope_receipt(scope):
|
||
try:
|
||
receipt = json.loads(Path(scope['receipt_path']).read_text())
|
||
if (receipt.get('binding') != scope['binding'] or
|
||
scope['binding'].get('proof') != 'waitpid:ECHILD' or
|
||
not isinstance(receipt.get('exit_code'), int)):
|
||
return None
|
||
if receipt.get('cleanup_state') == 'VERIFIED' and (
|
||
receipt.get('all_children_reaped') is not True or receipt.get('owner_lost') is not False):
|
||
return None
|
||
return receipt
|
||
except (OSError, ValueError, KeyError, TypeError):
|
||
return None
|
||
|
||
|
||
def validate_policy(config):
|
||
if not isinstance(config, dict) or not config.get('version'):
|
||
raise ValueError('LIFECYCLE_CONFIG_INCOMPLETE: version')
|
||
config = json.loads(json.dumps(config))
|
||
config.setdefault('phases', {}).setdefault('terminal', {field: config.get(field) for field in POLICY_FIELDS})
|
||
for name, row in [('terminal', config['phases']['terminal']), ('execution', config), *[(s, config.get('phases', {}).get(s)) for s in PHASES]]:
|
||
if not isinstance(row, dict):
|
||
raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}')
|
||
for field in POLICY_FIELDS:
|
||
value = row.get(field)
|
||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0:
|
||
raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}.{field}')
|
||
if row['cleanup_deadline_s'] < row['cancel_grace_s']:
|
||
raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}.cleanup_deadline_s')
|
||
config = json.loads(json.dumps(config))
|
||
cap = config.setdefault('ordinary_tool_deadline_s', 300)
|
||
if isinstance(cap, bool) or not isinstance(cap, (int, float)) or not math.isfinite(cap) or cap <= 0:
|
||
raise ValueError('LIFECYCLE_CONFIG_INCOMPLETE: ordinary_tool_deadline_s')
|
||
return config
|
||
|
||
|
||
def resume_policy(state):
|
||
"""Reject an implicit policy upgrade before any epoch, quota or DB mutation."""
|
||
cap = state.get('config', {}).get('ordinary_tool_deadline_s')
|
||
if isinstance(cap, bool) or not isinstance(cap, (int, float)) or not math.isfinite(cap) or cap <= 0:
|
||
raise ExecutionStopped('RESUME_TOOL_POLICY_MISSING')
|
||
|
||
|
||
def paused_credit(state):
|
||
"""Time paused for infrastructure (quota, network) is not the agent's: it is handed back on resume, at most one
|
||
execution budget in total so a paused rollout that can never resume still ends. 10-14: three rollouts paused
|
||
7–16 h by GLM outages resumed with < 1 h left and were cut off mid-compile or mid-eval."""
|
||
paused = state.get('paused_at') if state['execution_state'] == 'PAUSED_INFRA' else None
|
||
if not paused:
|
||
return 0.0
|
||
return max(0.0, min(time.time() - paused, state['config']['execution_deadline_s'] - state.get('paused_s', 0.0)))
|
||
|
||
|
||
def current():
|
||
return getattr(_local, 'execution', None)
|
||
|
||
|
||
def current_phase():
|
||
return getattr(_local, 'phase', None)
|
||
|
||
|
||
def current_operation():
|
||
return getattr(_local, 'operation', None) or os.environ.get('BENCH_OPERATION_ID')
|
||
|
||
|
||
class Execution:
|
||
def __init__(self, rd):
|
||
self.rd = Path(rd).resolve()
|
||
self.path = self.rd / 'execution.json'
|
||
self.epoch = None
|
||
|
||
@contextlib.contextmanager
|
||
def locked(self):
|
||
self.rd.mkdir(parents=True, exist_ok=True)
|
||
with (self.rd / 'execution.lock').open('a') as lock:
|
||
fcntl.flock(lock, fcntl.LOCK_EX)
|
||
try:
|
||
yield
|
||
finally:
|
||
fcntl.flock(lock, fcntl.LOCK_UN)
|
||
|
||
def snapshot(self):
|
||
return json.loads(self.path.read_text())
|
||
|
||
def assert_owner(self, state):
|
||
if self.epoch is None or self.epoch != state['epoch']:
|
||
raise ExecutionStopped('OWNER_FENCED')
|
||
|
||
def _save(self, state, event_type, **fields):
|
||
self.assert_owner(state)
|
||
state['sequence'] = state.get('sequence', 0) + 1
|
||
state['updated_at'] = time.time()
|
||
event = dict(event_id=f"{state['execution_id']}:{state['sequence']}",
|
||
execution_id=state['execution_id'], sequence=state['sequence'],
|
||
time=state['updated_at'], type=event_type, epoch=state['epoch'], **fields)
|
||
state['last_event'] = event
|
||
atomic_json(self.path, state)
|
||
with (self.rd / 'execution.events.jsonl').open('a') as out:
|
||
out.write(json.dumps(event, sort_keys=True) + '\n'); out.flush(); os.fsync(out.fileno())
|
||
return state
|
||
|
||
def start(self, config, activity='WAITING_LLM'):
|
||
config = validate_policy(config)
|
||
with self.locked():
|
||
if self.path.exists():
|
||
raise ExecutionStopped('EXECUTION_EXISTS: reconcile; never reset an existing deadline')
|
||
now = time.monotonic()
|
||
state = dict(execution_id=uuid.uuid4().hex, epoch=1, owner=process_identity(os.getpid()),
|
||
boot_id=boot_id(), started_at=time.time(), started_monotonic=now,
|
||
deadline_monotonic=now + config['execution_deadline_s'], config=config,
|
||
config_version=config['version'], config_sha256=hashlib.sha256(
|
||
json.dumps(config, sort_keys=True).encode()).hexdigest(),
|
||
activity=activity, execution_state='ACTIVE', execution_outcome=None,
|
||
cleanup_state='NOT_NEEDED', cancel_requested=None, stop_reason=None,
|
||
processes=[], token=uuid.uuid4().hex, operations={}, phases={}, stage_sequence=0,
|
||
progress={}, health='UNKNOWN', auto_resume_count=0)
|
||
self.epoch = state['epoch']
|
||
return self._save(state, 'EXECUTION_STARTED')
|
||
|
||
def attach(self):
|
||
self.epoch = self.snapshot()['epoch']
|
||
return self
|
||
|
||
def _reason(self, state, *, admission=False):
|
||
if self.epoch is not None and self.epoch != state['epoch']:
|
||
return 'OWNER_FENCED'
|
||
if state['cancel_requested']:
|
||
return 'CANCEL_REQUESTED'
|
||
if state['cleanup_state'] == 'QUARANTINED':
|
||
return state.get('stop_reason') or 'RESOURCE_QUARANTINED'
|
||
if state['execution_state'] != 'ACTIVE' and (admission or state['execution_state'] != 'PAUSED_INFRA'):
|
||
return state.get('stop_reason') or state['execution_state']
|
||
if state.get('boot_id') != boot_id():
|
||
# The old boot's monotonic clock means nothing now; judge a paused execution's deadline by wall time, or
|
||
# one already past it is resumed, refused by resume() and left paused, again and again (10-25: 7 rollouts
|
||
# looped ~43k times after the Mac mini hang).
|
||
budget = state['deadline_monotonic'] - state['started_monotonic']
|
||
if state['execution_state'] == 'PAUSED_INFRA' and time.time() >= state['started_at'] + budget + paused_credit(state):
|
||
return 'EXECUTION_DEADLINE_EXCEEDED'
|
||
return 'BOOT_CHANGED_RECONCILE_REQUIRED'
|
||
if time.monotonic() >= state['deadline_monotonic'] + paused_credit(state):
|
||
return 'EXECUTION_DEADLINE_EXCEEDED'
|
||
owner_status = process_status(state['owner'])
|
||
if owner_status == 'DEAD':
|
||
return 'OWNER_LOST'
|
||
if admission and owner_status != 'ALIVE':
|
||
return 'OWNER_STATUS_UNKNOWN'
|
||
return None
|
||
|
||
def stop_reason(self, operation_id=None):
|
||
state = self.snapshot()
|
||
return self._reason(state) or self._phase_reason(state, operation_id or current_operation())
|
||
|
||
def _phase_reason(self, state, operation_id):
|
||
phase = state.get('phases', {}).get(operation_id)
|
||
if not phase or phase['state'] == 'DONE':
|
||
return None
|
||
if phase['boot_id'] != boot_id():
|
||
return 'PHASE_CLOCK_UNKNOWN'
|
||
key = 'queue_deadline_monotonic' if phase['state'] == 'QUEUED' else 'execution_deadline_monotonic'
|
||
if time.monotonic() >= phase[key]:
|
||
return 'QUEUE_DEADLINE_EXCEEDED' if phase['state'] == 'QUEUED' else 'PHASE_EXECUTION_DEADLINE_EXCEEDED'
|
||
return None
|
||
|
||
def begin_phase(self, operation_id, phase, *, queued_monotonic=None):
|
||
with self.admission() as state:
|
||
if operation_id in state.get('phases', {}):
|
||
return state['phases'][operation_id] # Never reset a persisted phase clock.
|
||
policy = state['config']['phases'][phase]
|
||
started = time.monotonic() if queued_monotonic is None else queued_monotonic
|
||
row = dict(phase=phase, state='QUEUED', boot_id=boot_id(), epoch=state['epoch'],
|
||
queued_monotonic=started, queue_deadline_monotonic=started + policy['queue_deadline_s'],
|
||
execution_started_monotonic=None, execution_deadline_monotonic=None)
|
||
state.setdefault('phases', {})[operation_id] = row
|
||
self._save(state, 'PHASE_QUEUED', operation_id=operation_id, phase=phase)
|
||
return row
|
||
|
||
def phase_running(self, operation_id):
|
||
with self.admission() as state:
|
||
phase = state.get('phases', {}).get(operation_id)
|
||
if not phase or phase['state'] != 'QUEUED':
|
||
return
|
||
reason = self._phase_reason(state, operation_id)
|
||
if reason:
|
||
raise ExecutionStopped(reason)
|
||
now = time.monotonic()
|
||
phase.update(state='RUNNING', execution_started_monotonic=now,
|
||
execution_deadline_monotonic=now + state['config']['phases'][phase['phase']]['execution_deadline_s'])
|
||
state['stage_sequence'] = state.get('stage_sequence', 0) + 1
|
||
self._save(state, 'PHASE_STARTED', operation_id=operation_id, phase=phase['phase'])
|
||
|
||
def phase_done_locked(self, state, operation_id):
|
||
self.assert_owner(state)
|
||
if operation_id in state.get('phases', {}):
|
||
state['phases'][operation_id]['state'] = 'DONE'
|
||
state['stage_sequence'] = state.get('stage_sequence', 0) + 1
|
||
|
||
@contextlib.contextmanager
|
||
def admission(self):
|
||
"""Caller must spawn AND register within this interval; cancel uses this lock."""
|
||
with self.locked():
|
||
state = self.snapshot()
|
||
reason = self._reason(state, admission=True) or self._phase_reason(state, current_operation())
|
||
if reason:
|
||
raise ExecutionStopped(reason)
|
||
yield state
|
||
|
||
@contextlib.contextmanager
|
||
def mutation(self):
|
||
with self.locked():
|
||
state = self.snapshot()
|
||
self.assert_owner(state)
|
||
yield state
|
||
|
||
@contextlib.contextmanager
|
||
def scope(self, phase=None, operation_id=None):
|
||
old = (current(), current_phase(), getattr(_local, 'operation', None))
|
||
_local.execution, _local.phase, _local.operation = self, phase, operation_id
|
||
try:
|
||
yield self
|
||
finally:
|
||
_local.execution, _local.phase, _local.operation = old
|
||
|
||
def policy(self, phase=None):
|
||
config = self.snapshot()['config']
|
||
return config['phases'][phase] if phase in config.get('phases', {}) else config
|
||
|
||
def environment(self, env=None):
|
||
state = self.snapshot()
|
||
return dict(os.environ if env is None else env, BENCH_EXECUTION_DIR=str(self.rd),
|
||
BENCH_EXECUTION_EPOCH=str(state['epoch']), BENCH_EXECUTION_TOKEN=state['token'])
|
||
|
||
def register_locked(self, state, proc, role, scope_token):
|
||
identity = getattr(proc, 'leader', None) or process_identity(proc.pid)
|
||
state['processes'].append(dict(identity, role=role, scope_token=scope_token, epoch=state['epoch']))
|
||
self._save(state, 'PROCESS_STARTED', process=identity, role=role)
|
||
|
||
def register_scope_locked(self, state, scope, role):
|
||
state.setdefault('process_scopes', {})[scope.token] = {
|
||
'binding': scope.binding, 'receipt_path': str(scope.path), 'owner': state['owner']}
|
||
self.register_locked(state, scope.supervisor, 'supervisor', scope.token)
|
||
self.register_locked(state, scope, role, scope.token)
|
||
|
||
def record_process_set(self, token, leader, identities, unknown):
|
||
with self.mutation() as state:
|
||
recorded = {process_key(p) for p in state['processes']}
|
||
added = [dict(p, role='descendant', scope_token=token, epoch=state['epoch'])
|
||
for p in identities if process_key(p) not in recorded]
|
||
scope = {'leader': leader, 'unknown': unknown}
|
||
previous = state.setdefault('process_sets', {}).get(token)
|
||
if added or previous != scope:
|
||
state['processes'].extend(added)
|
||
state['process_sets'][token] = scope
|
||
self._save(state, 'PROCESS_SET_OBSERVED', scope_token=token, added=len(added), unknown=len(unknown))
|
||
|
||
def activity(self, name, **details):
|
||
with self.locked():
|
||
state = self.snapshot()
|
||
if self.epoch != state['epoch']:
|
||
raise ExecutionStopped('OWNER_FENCED')
|
||
if state['execution_state'] != 'TERMINAL':
|
||
state['activity'] = name
|
||
state['activity_details'] = details
|
||
return self._save(state, 'ACTIVITY_CHANGED', activity=name, details=details)
|
||
|
||
def observe_tools(self, events):
|
||
"""Persist typed SDK tool boundaries; a timeout fences all new admission."""
|
||
with self.locked():
|
||
state = self.snapshot()
|
||
self.assert_owner(state)
|
||
if state['execution_state'] != 'ACTIVE':
|
||
return state.get('stop_reason')
|
||
calls = state.setdefault('tool_calls', {})
|
||
changed = False
|
||
for event in events:
|
||
if event.get('execution_id') != state['execution_id'] or event.get('epoch') != state['epoch']:
|
||
raise ExecutionStopped('TOOL_EVENT_IDENTITY_MISMATCH')
|
||
key = event['tool_call_id']
|
||
row = calls.get(key)
|
||
if row is not None and any(event.get(field) != row[field] for field in
|
||
('started_monotonic', 'tool_name', 'cap_s')):
|
||
# Provider call IDs are not occurrence identities. Keep the
|
||
# first occurrence immutable and reject ambiguous reuse,
|
||
# including a late end that belongs to another occurrence.
|
||
reason = 'INFRA_TOOL_IDENTITY_CONFLICT'
|
||
state.update(execution_state='STOP_REQUESTED', cleanup_state='PENDING',
|
||
stop_reason=reason, tool_conflict=dict(event),
|
||
activity='RUNNING_TOOL', activity_details=dict(event))
|
||
self._save(state, reason)
|
||
return reason
|
||
if event['state'] == 'RUNNING' and row is None:
|
||
started, cap = event['started_monotonic'], event['cap_s']
|
||
if (not isinstance(key, str) or not key or not math.isfinite(started) or
|
||
not math.isfinite(cap) or not 0 < cap <= (state['config'].get('managed_tool_deadline_s', state['config']['execution_deadline_s']) if event.get('tool_name') == 'terminal_exec' else state['config'].get('ordinary_tool_deadline_s', 300)) or
|
||
not state['started_monotonic'] <= started <= time.monotonic() + 1):
|
||
raise ExecutionStopped('TOOL_EVENT_INVALID')
|
||
row = dict(tool_call_id=key, tool_name=event['tool_name'], state='RUNNING',
|
||
started_monotonic=started, deadline_monotonic=started + cap, cap_s=cap)
|
||
calls[key] = row
|
||
changed = True
|
||
elif event['state'] in ('DONE', 'TIMED_OUT') and row is not None:
|
||
# A late result cannot move the absolute deadline or erase its expiry.
|
||
row['state'] = ('TIMED_OUT' if row['state'] == 'TIMED_OUT' or event['state'] == 'TIMED_OUT' or
|
||
event['observed_monotonic'] >= row['deadline_monotonic'] else 'DONE')
|
||
changed = True
|
||
overdue = next((v for v in calls.values() if v['state'] == 'TIMED_OUT' or
|
||
(v['state'] == 'RUNNING' and time.monotonic() >= v['deadline_monotonic'])), None)
|
||
active = overdue or next((v for v in calls.values() if v['state'] == 'RUNNING'), None)
|
||
if active:
|
||
state.update(activity='RUNNING_TOOL', activity_details=dict(active))
|
||
elif changed and state['activity'] == 'RUNNING_TOOL':
|
||
state.update(activity='WAITING_LLM', activity_details={})
|
||
if overdue:
|
||
overdue['state'] = 'TIMED_OUT'
|
||
state.update(execution_state='STOP_REQUESTED', cleanup_state='PENDING',
|
||
stop_reason='TOOL_DEADLINE_EXCEEDED', activity_details=dict(overdue))
|
||
if changed or overdue:
|
||
self._save(state, 'TOOL_DEADLINE_EXCEEDED' if overdue else 'TOOL_OBSERVED')
|
||
return state.get('stop_reason')
|
||
|
||
def record_progress(self, source, signals, *, phase=None, operation_id=None, role='tool'):
|
||
"""One typed sample drives health and the read-only monitor; no log-text heuristics."""
|
||
with self.locked():
|
||
state = self.snapshot()
|
||
self.assert_owner(state)
|
||
now, wall = time.monotonic(), time.time()
|
||
progress = state.setdefault('progress', {})
|
||
previous = progress.setdefault('sources', {}).get(source, {})
|
||
old = previous.get('signals', {})
|
||
changes = {key: (None if value is None or old.get(key) is None else value != old[key])
|
||
for key, value in signals.items()}
|
||
unknown = any(value is None for value in signals.values())
|
||
changed = any(value is True for value in changes.values())
|
||
idle_since = None if unknown or changed else previous.get('idle_since_monotonic') or now
|
||
threshold = state['config']['phases'][phase]['suspect_after_s'] if phase in PHASES else state['config']['suspect_after_s']
|
||
phase_row = state.get('phases', {}).get(operation_id, {})
|
||
waiting = phase_row.get('state') == 'QUEUED' or (not operation_id and state['activity'] in
|
||
('QUEUED', 'WAITING_QUOTA', 'WAITING_RESOURCE'))
|
||
health = ('WAITING' if waiting else 'UNKNOWN' if unknown else
|
||
'SUSPECT' if idle_since is not None and now - idle_since >= threshold else 'OBSERVED')
|
||
sample = dict(sampled_at=wall, sampled_monotonic=now, signals=signals, changed=changes,
|
||
idle_since_monotonic=idle_since, suspect_after_s=threshold, health=health,
|
||
role=role, operation_id=operation_id, phase=phase,
|
||
reason='SIGNAL_UNKNOWN' if unknown else 'ALL_SIGNALS_IDLE' if health == 'SUSPECT' else 'PROGRESS_OBSERVED')
|
||
progress['sources'][source] = sample
|
||
active = {key for key, value in state.get('phases', {}).items() if value['state'] in ('QUEUED', 'RUNNING')}
|
||
# Agent silence while a separately owned stage is active is expected.
|
||
if role != 'agent' or not active:
|
||
progress.update(sampled_at=wall, current_source=source, **{k: sample[k] for k in
|
||
('signals', 'changed', 'idle_since_monotonic', 'suspect_after_s', 'reason')})
|
||
state['health'] = health
|
||
return self._save(state, 'PROGRESS_SAMPLED', source=source, health=health)
|
||
|
||
def cancel(self, reason):
|
||
if not isinstance(reason, str) or not reason.strip():
|
||
raise ValueError('cancel requires a reason')
|
||
with self.locked():
|
||
state = self.snapshot()
|
||
if self.epoch is None: # Explicit control-plane cancellation binds under the same lock.
|
||
self.epoch = state['epoch']
|
||
self.assert_owner(state)
|
||
if state['execution_state'] in ('TERMINAL', 'STOP_REQUESTED'):
|
||
return dict(state, reason='ALREADY_TERMINAL')
|
||
if not state['cancel_requested']:
|
||
state['cancel_requested'] = dict(reason=reason.strip(), time=time.time(), epoch=state['epoch'])
|
||
state.update(execution_state='CANCEL_REQUESTED', cleanup_state='PENDING', stop_reason='CANCEL_REQUESTED')
|
||
self._save(state, 'CANCEL_REQUESTED', reason=reason.strip())
|
||
return state
|
||
|
||
def quarantine(self, reason):
|
||
with self.locked():
|
||
state = self.snapshot()
|
||
self.assert_owner(state)
|
||
state.update(cleanup_state='QUARANTINED', stop_reason=reason)
|
||
if state['execution_state'] == 'ACTIVE':
|
||
state['execution_state'] = 'PAUSED_INFRA'
|
||
return self._save(state, 'RESOURCE_QUARANTINED', reason=reason)
|
||
|
||
def _checkpoint(self, value):
|
||
if not isinstance(value, dict) or not (value.get('resumable') is True and value.get('phase') == 'PAUSED_INFRA' or value.get('complete') is True and 'resumable' not in value):
|
||
raise ExecutionStopped('CHECKPOINT_INCOMPLETE')
|
||
supplied = Path(value.get('path', ''))
|
||
path = supplied.resolve()
|
||
if supplied.is_symlink() or not path.is_relative_to(self.rd) or not path.is_file():
|
||
raise ExecutionStopped('CHECKPOINT_PATH_INVALID')
|
||
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
||
if digest != value.get('sha256'):
|
||
raise ExecutionStopped('CHECKPOINT_HASH_MISMATCH')
|
||
return dict(value, path=str(path))
|
||
|
||
def pause(self, reason, checkpoint):
|
||
if not isinstance(reason, str) or not reason.strip():
|
||
raise ValueError('pause requires a reason')
|
||
with self.admission() as state:
|
||
checkpoint = self._checkpoint(checkpoint)
|
||
state.update(execution_state='PAUSED_INFRA', stop_reason=reason,
|
||
checkpoint=checkpoint, paused_at=time.time())
|
||
return self._save(state, 'EXECUTION_PAUSED', reason=reason)
|
||
|
||
def resume(self, *, automatic=True, counted=True):
|
||
with self.locked():
|
||
state = self.snapshot()
|
||
if self.epoch is not None and self.epoch != state['epoch']:
|
||
raise ExecutionStopped('OWNER_FENCED')
|
||
resume_policy(state)
|
||
if state['execution_state'] != 'PAUSED_INFRA' or state.get('cancel_requested'):
|
||
raise ExecutionStopped('EXECUTION_NOT_RESUMABLE')
|
||
if state['cleanup_state'] == 'QUARANTINED':
|
||
raise ExecutionStopped('SIDE_EFFECT_UNKNOWN')
|
||
if state.get('boot_id') != boot_id():
|
||
if not earlier_boot(state['owner']):
|
||
raise ExecutionStopped('BOOT_CHANGED_RECONCILE_REQUIRED')
|
||
self._rebase_clock(state)
|
||
credit = paused_credit(state)
|
||
state['deadline_monotonic'] += credit
|
||
state['paused_s'] = state.get('paused_s', 0.0) + credit
|
||
state.pop('paused_at', None)
|
||
if time.monotonic() >= state['deadline_monotonic']:
|
||
raise ExecutionStopped('EXECUTION_DEADLINE_EXCEEDED')
|
||
self._checkpoint(state.get('checkpoint'))
|
||
owner_status = process_status(state['owner'])
|
||
same_owner = process_key(state['owner']) == process_key(process_identity(os.getpid()))
|
||
if owner_status != 'DEAD' and not same_owner:
|
||
raise ExecutionStopped('OWNER_NOT_RELEASABLE')
|
||
members = owned_processes(state)
|
||
pending = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE']
|
||
if members != [] or pending:
|
||
raise ExecutionStopped('SIDE_EFFECT_UNKNOWN' if members is None else 'WORK_STILL_ACTIVE')
|
||
count = state.get('auto_resume_count', 0)
|
||
if automatic and counted and count >= state['config'].get('max_auto_resumes', 3):
|
||
raise ExecutionStopped('AUTO_RESUME_BUDGET_EXHAUSTED')
|
||
state['epoch'] += 1
|
||
self.epoch = state['epoch']
|
||
state.update(owner=process_identity(os.getpid()), execution_state='ACTIVE', stop_reason=None,
|
||
auto_resume_count=count + int(automatic and counted), resumed_at=time.time(), activity='WAITING_QUOTA')
|
||
return self._save(state, 'EXECUTION_RESUMED', automatic=automatic)
|
||
|
||
@staticmethod
|
||
def _rebase_clock(state):
|
||
"""Monotonic clocks restart with the host. Carry every deadline over by wall time; the downtime itself is
|
||
a pause (paused_at), credited back like any other infrastructure pause."""
|
||
offset = (time.monotonic() - time.time()) - (state['started_monotonic'] - state['started_at'])
|
||
state['started_monotonic'] += offset
|
||
state['deadline_monotonic'] += offset
|
||
for row in [*state.get('phases', {}).values(), *state.get('tool_calls', {}).values()]:
|
||
for key, value in row.items():
|
||
if key.endswith('_monotonic') and isinstance(value, (int, float)):
|
||
row[key] = value + offset
|
||
if 'boot_id' in row:
|
||
row['boot_id'] = boot_id()
|
||
state['boot_id'] = boot_id()
|
||
|
||
def cleanup(self):
|
||
"""Bounded cancellation cleanup; unknown remote receipts keep leases isolated."""
|
||
with self.locked():
|
||
state = self.snapshot()
|
||
self.assert_owner(state)
|
||
if not state.get('cancel_requested'):
|
||
raise ExecutionStopped('CLEANUP_REQUIRES_CANCEL')
|
||
if state['execution_state'] == 'TERMINAL':
|
||
return state
|
||
state['execution_state'] = 'CLEANING'
|
||
self._save(state, 'CLEANUP_STARTED')
|
||
policy = state['config']
|
||
start = time.monotonic()
|
||
while True:
|
||
members = owned_processes(state)
|
||
latest = self.snapshot()
|
||
self.assert_owner(latest)
|
||
pending = [v for v in latest.get('operations', {}).values() if v.get('state') != 'DONE']
|
||
if members == [] and not pending:
|
||
break
|
||
elapsed = time.monotonic() - start
|
||
if members is None or elapsed >= policy['cleanup_deadline_s']:
|
||
return self.quarantine('CLEANUP_UNCONFIRMED')
|
||
# Each supervisor performs its own bounded TERM/KILL/reap sequence.
|
||
# Killing the reaper itself would destroy the complete-set proof.
|
||
for scope in latest.get('process_scopes', {}).values():
|
||
signal_process(scope['binding']['supervisor'], signal.SIGTERM)
|
||
time.sleep(.1)
|
||
with self.locked():
|
||
latest = self.snapshot()
|
||
self.assert_owner(latest)
|
||
if latest['cleanup_state'] != 'QUARANTINED':
|
||
latest['cleanup_state'] = 'VERIFIED'
|
||
return self._save(latest, 'CLEANUP_FINISHED', cleanup_state=latest['cleanup_state'])
|
||
|
||
def finish(self, exit_code, *, protocol_complete, stop_reason=None, cleanup_state='VERIFIED'):
|
||
with self.locked():
|
||
state = self.snapshot()
|
||
if self.epoch != state['epoch']:
|
||
raise ExecutionStopped('OWNER_FENCED')
|
||
if state['execution_state'] == 'TERMINAL':
|
||
return state
|
||
reason = stop_reason or state.get('stop_reason')
|
||
members = owned_processes(state) if state['processes'] else []
|
||
pending = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE']
|
||
known = [process_status(p) for p in state['processes']]
|
||
if (members not in ([], None) or pending or 'ALIVE' in known or
|
||
(Path('/proc').is_dir() and ('UNKNOWN' in known or members is None))):
|
||
cleanup_state = 'QUARANTINED'
|
||
reason = reason or 'CLEANUP_UNCONFIRMED'
|
||
if state['cleanup_state'] == 'QUARANTINED' or cleanup_state not in ('VERIFIED', 'NOT_NEEDED'):
|
||
cleanup_state = 'QUARANTINED'
|
||
if state['cancel_requested']:
|
||
outcome = 'CANCELLED'
|
||
elif reason == 'OWNER_LOST':
|
||
outcome = 'LOST'
|
||
elif reason and ('DEADLINE' in reason or 'hard_ceiling' in reason):
|
||
outcome = 'DEADLINE_EXCEEDED'
|
||
elif reason and 'BUDGET' in reason:
|
||
outcome = 'BUDGET_EXHAUSTED'
|
||
# CLEANUP_UNCONFIRMED: a board or sandbox we could not reach, never the agent (10-27/28: 13 as AGENT_ERROR).
|
||
elif reason and any(word in reason for word in ('INFRA', 'UNKNOWN', 'monitor failed', 'CLEANUP_UNCONFIRMED')):
|
||
outcome = 'INFRA_ERROR'
|
||
elif exit_code == 0 and protocol_complete and cleanup_state == 'VERIFIED':
|
||
outcome = 'COMPLETED'
|
||
else:
|
||
outcome = 'AGENT_ERROR'
|
||
state.update(execution_state='TERMINAL', execution_outcome=outcome, cleanup_state=cleanup_state,
|
||
stop_reason=reason, exit_code=exit_code, protocol_complete=bool(protocol_complete),
|
||
signal=(-exit_code if isinstance(exit_code, int) and exit_code < 0 else
|
||
exit_code - 128 if isinstance(exit_code, int) and exit_code > 128 else None),
|
||
finished_at=time.time())
|
||
return self._save(state, 'EXECUTION_FINISHED', outcome=outcome, reason=reason)
|
||
|
||
def reconcile(self, crash_checkpoint=None, owner_lost_at=None):
|
||
"""Fail closed; never replay an unknown side effect or take a live owner.
|
||
|
||
crash_checkpoint: the agent's last durable session boundary, frozen by the caller after its owner (the
|
||
collector) died with every process proven gone and every operation settled. The rollout then pauses
|
||
(CONTROLLER_RESTARTED) and resumes from there instead of being lost.
|
||
owner_lost_at: when the dead owner last wrote this state, read before the caller's own saves."""
|
||
with self.locked():
|
||
state = self.snapshot()
|
||
if self.epoch is not None:
|
||
self.assert_owner(state)
|
||
if state['execution_state'] == 'TERMINAL':
|
||
return state
|
||
owner = process_status(state['owner'])
|
||
members = owned_processes(state)
|
||
operations = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE']
|
||
if owner == 'ALIVE':
|
||
return dict(state, reconciliation='OWNER_ALIVE')
|
||
if state['execution_state'] == 'PAUSED_INFRA' and owner == 'DEAD' and members == [] and not operations:
|
||
try:
|
||
self._checkpoint(state.get('checkpoint'))
|
||
except ExecutionStopped:
|
||
pass
|
||
else:
|
||
return dict(state, reconciliation='CHECKPOINT_READY_FOR_RESUME')
|
||
state['epoch'] += 1 # Fence every old dispatch path before changing ownership.
|
||
self.epoch = state['epoch']
|
||
restartable = (owner == 'DEAD' and members == [] and not operations and crash_checkpoint is not None
|
||
and state['execution_state'] == 'ACTIVE' and not state.get('cancel_requested'))
|
||
if restartable:
|
||
checkpoint = self._checkpoint(crash_checkpoint)
|
||
for call in state.get('tool_calls', {}).values():
|
||
if call['state'] == 'RUNNING':
|
||
call['state'] = 'INTERRUPTED' # it died with its owner; the resumed agent is told so
|
||
for scope in state.get('process_scopes', {}).values():
|
||
scope.setdefault('owner', state['owner']) # scopes registered before owners were recorded
|
||
# No collector ran, so none of the downtime was the agent's: hand all of it back now, uncapped (the
|
||
# paused-credit cap is for pauses that may never resume). 11-06..08: a 33 h planned stop, then
|
||
# updated_at refreshed by the restart's own settling, timed out 10 rollouts the moment they resumed.
|
||
downtime = max(0.0, time.time() - (owner_lost_at or state.get('updated_at') or time.time()))
|
||
state['deadline_monotonic'] += downtime
|
||
state['downtime_s'] = state.get('downtime_s', 0.0) + downtime
|
||
state.update(execution_state='PAUSED_INFRA', stop_reason='CONTROLLER_RESTARTED', checkpoint=checkpoint,
|
||
paused_at=time.time(), activity='WAITING_QUOTA', activity_details={})
|
||
return self._save(state, 'EXECUTION_RECONCILED', owner_status=owner, members=members,
|
||
pending_operations=0, reconciliation='CONTROLLER_RESTARTED')
|
||
if owner == 'DEAD' and members == [] and not operations:
|
||
state.update(execution_state='TERMINAL', execution_outcome='LOST', cleanup_state='VERIFIED',
|
||
stop_reason='OWNER_LOST', finished_at=time.time())
|
||
else:
|
||
state.update(execution_state='PAUSED_INFRA', cleanup_state='QUARANTINED',
|
||
stop_reason='SIDE_EFFECT_UNKNOWN' if operations else 'OWNERSHIP_UNKNOWN')
|
||
return self._save(state, 'EXECUTION_RECONCILED', owner_status=owner,
|
||
members=members, pending_operations=len(operations))
|
||
|
||
|
||
def from_environment():
|
||
value = os.environ.get('BENCH_EXECUTION_DIR')
|
||
if not value:
|
||
return None
|
||
result = Execution(value)
|
||
result.epoch = int(os.environ['BENCH_EXECUTION_EPOCH'])
|
||
return result
|