GLM 请求失败率约 0.3%,但一次失败就让整条轨迹暂停;3 次续跑次数被网络故障用完后, 轨迹只能挂到 24 小时上限(10-14 有 4 条);回复中途断开的请求过了等待期会被自动放弃, 同时整条执行被取消(10-13/14 共 16 条)。 A. 连接失败、域名解析失败(RETRY_UNSENT,确定什么都没发出去)在 broker 里等一会儿原地重发。 每次重试把等待时间加上一次连接时限计入排队预算 wait_spent,保持 客户端超时 = 排队预算 + 次数×单次时限 的上界;预算用完才照旧暂停。 B. 续跑次数只在最近一次请求可能已经送达时才扣(看账本最近一次 attempt 是否 sent)。 连接/解析失败、因为别的请求结果不明而在准入时被挡下的暂停,续跑不扣次数。 执行记录和账本两处计数同步,调度程序不再按原始次数预先过滤。 C. reconcile 自动放弃时(截止时间再过一个 token 窗口)不再取消执行,而是把这一轮 标成 REQUEST_OUTCOME_UNKNOWN、可以续跑:重发一轮只花一次请求的 token,并计入续跑次数。 人工放弃(operator)仍然永久隔离。续跑资格检查不再被已放弃的 attempt 挡住。 都在调度程序进程里运行,下次重启生效。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
705 lines
35 KiB
Python
705 lines
35 KiB
Python
"""Small durable lifecycle ledger. One lock serializes cancel, spawn and finish.
|
|
|
|
Only Linux /proc identities authorize signals. Unknown ownership is quarantined,
|
|
never inferred from a PID or from an old RUNNING label.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import contextlib
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import socket
|
|
import signal
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
PHASES = ('01-fetch', '02-baseline', '03-export', '04-calibrate', '05-compile',
|
|
'06-hbm', '07-board', '08-eval')
|
|
POLICY_FIELDS = ('queue_deadline_s', 'execution_deadline_s', 'suspect_after_s',
|
|
'cancel_grace_s', 'cleanup_deadline_s')
|
|
_local = threading.local()
|
|
|
|
|
|
class ExecutionStopped(RuntimeError):
|
|
pass
|
|
|
|
|
|
def atomic_json(path: Path, value):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
fd, name = tempfile.mkstemp(prefix='.' + path.name + '-', dir=path.parent)
|
|
try:
|
|
with os.fdopen(fd, 'w') as out:
|
|
json.dump(value, out, ensure_ascii=False, sort_keys=True)
|
|
out.flush(); os.fsync(out.fileno())
|
|
os.replace(name, path)
|
|
directory = os.open(path.parent, os.O_RDONLY)
|
|
try:
|
|
os.fsync(directory)
|
|
finally:
|
|
os.close(directory)
|
|
finally:
|
|
if os.path.exists(name):
|
|
os.unlink(name)
|
|
|
|
|
|
def boot_id():
|
|
try:
|
|
return Path('/proc/sys/kernel/random/boot_id').read_text().strip()
|
|
except OSError:
|
|
return None
|
|
|
|
|
|
def process_identity(pid: int) -> dict:
|
|
result = dict(host=socket.gethostname(), boot_id=boot_id(), pid=pid)
|
|
try:
|
|
raw = Path(f'/proc/{pid}/stat').read_bytes()
|
|
row = raw.rsplit(b')', 1)[1].split()
|
|
result.update(proc_start_ticks=int(row[19]), ppid=int(row[1]), pgrp=int(row[2]),
|
|
session=int(row[3]), state=row[0].decode('ascii'), cpu_ticks=int(row[11]) + int(row[12]), status='KNOWN')
|
|
except (OSError, ValueError, IndexError):
|
|
result['status'] = 'UNKNOWN'
|
|
return result
|
|
|
|
|
|
def process_status(identity: dict) -> str:
|
|
if (identity.get('host') != socket.gethostname() or not identity.get('boot_id')
|
|
or identity.get('boot_id') != boot_id() or identity.get('proc_start_ticks') is None):
|
|
return 'UNKNOWN'
|
|
current = process_identity(identity['pid'])
|
|
if current['status'] == 'UNKNOWN':
|
|
return 'DEAD' if not Path(f"/proc/{identity['pid']}").exists() else 'UNKNOWN'
|
|
if current['proc_start_ticks'] != identity['proc_start_ticks'] or current['state'] == 'Z':
|
|
return 'DEAD'
|
|
return 'ALIVE'
|
|
|
|
|
|
def signal_process(identity, sig):
|
|
"""A pidfd closes the identity-check/kill PID-reuse race on supported Linux."""
|
|
if process_status(identity) != 'ALIVE':
|
|
return process_status(identity) == 'DEAD'
|
|
fd = None
|
|
try:
|
|
if hasattr(os, 'pidfd_open') and hasattr(signal, 'pidfd_send_signal'):
|
|
fd = os.pidfd_open(identity['pid'])
|
|
if process_status(identity) != 'ALIVE':
|
|
return True
|
|
signal.pidfd_send_signal(fd, sig)
|
|
else:
|
|
# Refuse rather than authorize a signal from an unverifiable identifier.
|
|
return False
|
|
return True
|
|
except ProcessLookupError:
|
|
return True
|
|
except (PermissionError, OSError):
|
|
return False
|
|
finally:
|
|
if fd is not None:
|
|
os.close(fd)
|
|
|
|
|
|
def process_key(identity):
|
|
return tuple(identity.get(k) for k in ('host', 'boot_id', 'pid', 'proc_start_ticks'))
|
|
|
|
|
|
def marker_processes(key, value):
|
|
"""Enumerate /proc only; an unguessable inherited marker joins detached children."""
|
|
proc = Path('/proc')
|
|
if not proc.is_dir():
|
|
return None
|
|
needle = f'{key}={value}'.encode()
|
|
result = []
|
|
for entry in proc.iterdir():
|
|
if not entry.name.isdigit():
|
|
continue
|
|
try:
|
|
fields = (entry / 'environ').read_bytes().split(b'\0')
|
|
matches = needle in fields
|
|
if key == 'BENCH_PROCESS_ANCESTRY':
|
|
prefix = (key + '=').encode()
|
|
matches = any(value.encode() in field[len(prefix):].split(b',')
|
|
for field in fields if field.startswith(prefix))
|
|
if matches:
|
|
identity = process_identity(int(entry.name))
|
|
if identity['status'] == 'KNOWN' and identity['state'] != 'Z':
|
|
result.append(identity)
|
|
except (OSError, ValueError):
|
|
continue
|
|
return result
|
|
|
|
|
|
def owned_processes(state):
|
|
"""Incomplete containment receipts remain unknown even with no visible PID."""
|
|
members = {}
|
|
for identity in state.get('processes', []):
|
|
if process_status(identity) != 'DEAD':
|
|
members[process_key(identity)] = dict(identity, ownership_verified=True)
|
|
scopes = state.get('process_scopes', {})
|
|
for token in state.get('process_sets', {}):
|
|
if token not in scopes:
|
|
return None # Legacy observations cannot prove a complete owned set.
|
|
for scope in scopes.values():
|
|
if not scope_cleaned(scope, scopes) and process_status(scope['binding']['supervisor']) != 'ALIVE':
|
|
return None
|
|
return list(members.values())
|
|
|
|
|
|
def scope_cleaned(scope, scopes) -> bool:
|
|
"""One rule for every caller that asks whether a process scope left nothing behind."""
|
|
receipt = scope_receipt(scope)
|
|
if receipt and receipt['cleanup_state'] == 'VERIFIED':
|
|
return True
|
|
# A nested reaper may lose its parent during cancellation even though
|
|
# both levels finish waitpid(ECHILD). The enclosing verified receipt
|
|
# must name this exact supervisor, including boot/start identity.
|
|
if not (receipt and receipt.get('cleanup_state') == 'QUARANTINED'
|
|
and receipt.get('all_children_reaped') is True
|
|
and receipt.get('owner_lost') is True
|
|
and receipt.get('supervisor_error') is None):
|
|
return False
|
|
supervisor = scope['binding']['supervisor']
|
|
for parent in scopes.values():
|
|
proof = scope_receipt(parent)
|
|
binding = parent['binding']
|
|
if (proof and proof['cleanup_state'] == 'VERIFIED'
|
|
and binding.get('execution_id') == scope['binding'].get('execution_id')
|
|
and binding.get('epoch') == scope['binding'].get('epoch')
|
|
and supervisor.get('ppid') == binding['leader'].get('pid')
|
|
and process_key(supervisor) in {process_key(p) for p in proof.get('members', [])}):
|
|
return True
|
|
return False
|
|
|
|
|
|
def scope_receipt(scope):
|
|
try:
|
|
receipt = json.loads(Path(scope['receipt_path']).read_text())
|
|
if (receipt.get('binding') != scope['binding'] or
|
|
scope['binding'].get('proof') != 'waitpid:ECHILD' or
|
|
not isinstance(receipt.get('exit_code'), int)):
|
|
return None
|
|
if receipt.get('cleanup_state') == 'VERIFIED' and (
|
|
receipt.get('all_children_reaped') is not True or receipt.get('owner_lost') is not False):
|
|
return None
|
|
return receipt
|
|
except (OSError, ValueError, KeyError, TypeError):
|
|
return None
|
|
|
|
|
|
def validate_policy(config):
|
|
if not isinstance(config, dict) or not config.get('version'):
|
|
raise ValueError('LIFECYCLE_CONFIG_INCOMPLETE: version')
|
|
config = json.loads(json.dumps(config))
|
|
config.setdefault('phases', {}).setdefault('terminal', {field: config.get(field) for field in POLICY_FIELDS})
|
|
for name, row in [('terminal', config['phases']['terminal']), ('execution', config), *[(s, config.get('phases', {}).get(s)) for s in PHASES]]:
|
|
if not isinstance(row, dict):
|
|
raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}')
|
|
for field in POLICY_FIELDS:
|
|
value = row.get(field)
|
|
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0:
|
|
raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}.{field}')
|
|
if row['cleanup_deadline_s'] < row['cancel_grace_s']:
|
|
raise ValueError(f'LIFECYCLE_CONFIG_INCOMPLETE: {name}.cleanup_deadline_s')
|
|
config = json.loads(json.dumps(config))
|
|
cap = config.setdefault('ordinary_tool_deadline_s', 300)
|
|
if isinstance(cap, bool) or not isinstance(cap, (int, float)) or not math.isfinite(cap) or cap <= 0:
|
|
raise ValueError('LIFECYCLE_CONFIG_INCOMPLETE: ordinary_tool_deadline_s')
|
|
return config
|
|
|
|
|
|
def resume_policy(state):
|
|
"""Reject an implicit policy upgrade before any epoch, quota or DB mutation."""
|
|
cap = state.get('config', {}).get('ordinary_tool_deadline_s')
|
|
if isinstance(cap, bool) or not isinstance(cap, (int, float)) or not math.isfinite(cap) or cap <= 0:
|
|
raise ExecutionStopped('RESUME_TOOL_POLICY_MISSING')
|
|
|
|
|
|
def current():
|
|
return getattr(_local, 'execution', None)
|
|
|
|
|
|
def current_phase():
|
|
return getattr(_local, 'phase', None)
|
|
|
|
|
|
def current_operation():
|
|
return getattr(_local, 'operation', None) or os.environ.get('BENCH_OPERATION_ID')
|
|
|
|
|
|
class Execution:
|
|
def __init__(self, rd):
|
|
self.rd = Path(rd).resolve()
|
|
self.path = self.rd / 'execution.json'
|
|
self.epoch = None
|
|
|
|
@contextlib.contextmanager
|
|
def locked(self):
|
|
self.rd.mkdir(parents=True, exist_ok=True)
|
|
with (self.rd / 'execution.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX)
|
|
try:
|
|
yield
|
|
finally:
|
|
fcntl.flock(lock, fcntl.LOCK_UN)
|
|
|
|
def snapshot(self):
|
|
return json.loads(self.path.read_text())
|
|
|
|
def assert_owner(self, state):
|
|
if self.epoch is None or self.epoch != state['epoch']:
|
|
raise ExecutionStopped('OWNER_FENCED')
|
|
|
|
def _save(self, state, event_type, **fields):
|
|
self.assert_owner(state)
|
|
state['sequence'] = state.get('sequence', 0) + 1
|
|
state['updated_at'] = time.time()
|
|
event = dict(event_id=f"{state['execution_id']}:{state['sequence']}",
|
|
execution_id=state['execution_id'], sequence=state['sequence'],
|
|
time=state['updated_at'], type=event_type, epoch=state['epoch'], **fields)
|
|
state['last_event'] = event
|
|
atomic_json(self.path, state)
|
|
with (self.rd / 'execution.events.jsonl').open('a') as out:
|
|
out.write(json.dumps(event, sort_keys=True) + '\n'); out.flush(); os.fsync(out.fileno())
|
|
return state
|
|
|
|
def start(self, config, activity='WAITING_LLM'):
|
|
config = validate_policy(config)
|
|
with self.locked():
|
|
if self.path.exists():
|
|
raise ExecutionStopped('EXECUTION_EXISTS: reconcile; never reset an existing deadline')
|
|
now = time.monotonic()
|
|
state = dict(execution_id=uuid.uuid4().hex, epoch=1, owner=process_identity(os.getpid()),
|
|
boot_id=boot_id(), started_at=time.time(), started_monotonic=now,
|
|
deadline_monotonic=now + config['execution_deadline_s'], config=config,
|
|
config_version=config['version'], config_sha256=hashlib.sha256(
|
|
json.dumps(config, sort_keys=True).encode()).hexdigest(),
|
|
activity=activity, execution_state='ACTIVE', execution_outcome=None,
|
|
cleanup_state='NOT_NEEDED', cancel_requested=None, stop_reason=None,
|
|
processes=[], token=uuid.uuid4().hex, operations={}, phases={}, stage_sequence=0,
|
|
progress={}, health='UNKNOWN', auto_resume_count=0)
|
|
self.epoch = state['epoch']
|
|
return self._save(state, 'EXECUTION_STARTED')
|
|
|
|
def attach(self):
|
|
self.epoch = self.snapshot()['epoch']
|
|
return self
|
|
|
|
def _reason(self, state, *, admission=False):
|
|
if self.epoch is not None and self.epoch != state['epoch']:
|
|
return 'OWNER_FENCED'
|
|
if state['cancel_requested']:
|
|
return 'CANCEL_REQUESTED'
|
|
if state['cleanup_state'] == 'QUARANTINED':
|
|
return state.get('stop_reason') or 'RESOURCE_QUARANTINED'
|
|
if state['execution_state'] != 'ACTIVE' and (admission or state['execution_state'] != 'PAUSED_INFRA'):
|
|
return state.get('stop_reason') or state['execution_state']
|
|
if state.get('boot_id') != boot_id():
|
|
return 'BOOT_CHANGED_RECONCILE_REQUIRED'
|
|
if time.monotonic() >= state['deadline_monotonic']:
|
|
return 'EXECUTION_DEADLINE_EXCEEDED'
|
|
owner_status = process_status(state['owner'])
|
|
if owner_status == 'DEAD':
|
|
return 'OWNER_LOST'
|
|
if admission and owner_status != 'ALIVE':
|
|
return 'OWNER_STATUS_UNKNOWN'
|
|
return None
|
|
|
|
def stop_reason(self, operation_id=None):
|
|
state = self.snapshot()
|
|
return self._reason(state) or self._phase_reason(state, operation_id or current_operation())
|
|
|
|
def _phase_reason(self, state, operation_id):
|
|
phase = state.get('phases', {}).get(operation_id)
|
|
if not phase or phase['state'] == 'DONE':
|
|
return None
|
|
if phase['boot_id'] != boot_id():
|
|
return 'PHASE_CLOCK_UNKNOWN'
|
|
key = 'queue_deadline_monotonic' if phase['state'] == 'QUEUED' else 'execution_deadline_monotonic'
|
|
if time.monotonic() >= phase[key]:
|
|
return 'QUEUE_DEADLINE_EXCEEDED' if phase['state'] == 'QUEUED' else 'PHASE_EXECUTION_DEADLINE_EXCEEDED'
|
|
return None
|
|
|
|
def begin_phase(self, operation_id, phase, *, queued_monotonic=None):
|
|
with self.admission() as state:
|
|
if operation_id in state.get('phases', {}):
|
|
return state['phases'][operation_id] # Never reset a persisted phase clock.
|
|
policy = state['config']['phases'][phase]
|
|
started = time.monotonic() if queued_monotonic is None else queued_monotonic
|
|
row = dict(phase=phase, state='QUEUED', boot_id=boot_id(), epoch=state['epoch'],
|
|
queued_monotonic=started, queue_deadline_monotonic=started + policy['queue_deadline_s'],
|
|
execution_started_monotonic=None, execution_deadline_monotonic=None)
|
|
state.setdefault('phases', {})[operation_id] = row
|
|
self._save(state, 'PHASE_QUEUED', operation_id=operation_id, phase=phase)
|
|
return row
|
|
|
|
def phase_running(self, operation_id):
|
|
with self.admission() as state:
|
|
phase = state.get('phases', {}).get(operation_id)
|
|
if not phase or phase['state'] != 'QUEUED':
|
|
return
|
|
reason = self._phase_reason(state, operation_id)
|
|
if reason:
|
|
raise ExecutionStopped(reason)
|
|
now = time.monotonic()
|
|
phase.update(state='RUNNING', execution_started_monotonic=now,
|
|
execution_deadline_monotonic=now + state['config']['phases'][phase['phase']]['execution_deadline_s'])
|
|
state['stage_sequence'] = state.get('stage_sequence', 0) + 1
|
|
self._save(state, 'PHASE_STARTED', operation_id=operation_id, phase=phase['phase'])
|
|
|
|
def phase_done_locked(self, state, operation_id):
|
|
self.assert_owner(state)
|
|
if operation_id in state.get('phases', {}):
|
|
state['phases'][operation_id]['state'] = 'DONE'
|
|
state['stage_sequence'] = state.get('stage_sequence', 0) + 1
|
|
|
|
@contextlib.contextmanager
|
|
def admission(self):
|
|
"""Caller must spawn AND register within this interval; cancel uses this lock."""
|
|
with self.locked():
|
|
state = self.snapshot()
|
|
reason = self._reason(state, admission=True) or self._phase_reason(state, current_operation())
|
|
if reason:
|
|
raise ExecutionStopped(reason)
|
|
yield state
|
|
|
|
@contextlib.contextmanager
|
|
def mutation(self):
|
|
with self.locked():
|
|
state = self.snapshot()
|
|
self.assert_owner(state)
|
|
yield state
|
|
|
|
@contextlib.contextmanager
|
|
def scope(self, phase=None, operation_id=None):
|
|
old = (current(), current_phase(), getattr(_local, 'operation', None))
|
|
_local.execution, _local.phase, _local.operation = self, phase, operation_id
|
|
try:
|
|
yield self
|
|
finally:
|
|
_local.execution, _local.phase, _local.operation = old
|
|
|
|
def policy(self, phase=None):
|
|
config = self.snapshot()['config']
|
|
return config['phases'][phase] if phase in config.get('phases', {}) else config
|
|
|
|
def environment(self, env=None):
|
|
state = self.snapshot()
|
|
return dict(os.environ if env is None else env, BENCH_EXECUTION_DIR=str(self.rd),
|
|
BENCH_EXECUTION_EPOCH=str(state['epoch']), BENCH_EXECUTION_TOKEN=state['token'])
|
|
|
|
def register_locked(self, state, proc, role, scope_token):
|
|
identity = getattr(proc, 'leader', None) or process_identity(proc.pid)
|
|
state['processes'].append(dict(identity, role=role, scope_token=scope_token, epoch=state['epoch']))
|
|
self._save(state, 'PROCESS_STARTED', process=identity, role=role)
|
|
|
|
def register_scope_locked(self, state, scope, role):
|
|
state.setdefault('process_scopes', {})[scope.token] = {
|
|
'binding': scope.binding, 'receipt_path': str(scope.path)}
|
|
self.register_locked(state, scope.supervisor, 'supervisor', scope.token)
|
|
self.register_locked(state, scope, role, scope.token)
|
|
|
|
def record_process_set(self, token, leader, identities, unknown):
|
|
with self.mutation() as state:
|
|
recorded = {process_key(p) for p in state['processes']}
|
|
added = [dict(p, role='descendant', scope_token=token, epoch=state['epoch'])
|
|
for p in identities if process_key(p) not in recorded]
|
|
scope = {'leader': leader, 'unknown': unknown}
|
|
previous = state.setdefault('process_sets', {}).get(token)
|
|
if added or previous != scope:
|
|
state['processes'].extend(added)
|
|
state['process_sets'][token] = scope
|
|
self._save(state, 'PROCESS_SET_OBSERVED', scope_token=token, added=len(added), unknown=len(unknown))
|
|
|
|
def activity(self, name, **details):
|
|
with self.locked():
|
|
state = self.snapshot()
|
|
if self.epoch != state['epoch']:
|
|
raise ExecutionStopped('OWNER_FENCED')
|
|
if state['execution_state'] != 'TERMINAL':
|
|
state['activity'] = name
|
|
state['activity_details'] = details
|
|
return self._save(state, 'ACTIVITY_CHANGED', activity=name, details=details)
|
|
|
|
def observe_tools(self, events):
|
|
"""Persist typed SDK tool boundaries; a timeout fences all new admission."""
|
|
with self.locked():
|
|
state = self.snapshot()
|
|
self.assert_owner(state)
|
|
if state['execution_state'] != 'ACTIVE':
|
|
return state.get('stop_reason')
|
|
calls = state.setdefault('tool_calls', {})
|
|
changed = False
|
|
for event in events:
|
|
if event.get('execution_id') != state['execution_id'] or event.get('epoch') != state['epoch']:
|
|
raise ExecutionStopped('TOOL_EVENT_IDENTITY_MISMATCH')
|
|
key = event['tool_call_id']
|
|
row = calls.get(key)
|
|
if row is not None and any(event.get(field) != row[field] for field in
|
|
('started_monotonic', 'tool_name', 'cap_s')):
|
|
# Provider call IDs are not occurrence identities. Keep the
|
|
# first occurrence immutable and reject ambiguous reuse,
|
|
# including a late end that belongs to another occurrence.
|
|
reason = 'INFRA_TOOL_IDENTITY_CONFLICT'
|
|
state.update(execution_state='STOP_REQUESTED', cleanup_state='PENDING',
|
|
stop_reason=reason, tool_conflict=dict(event),
|
|
activity='RUNNING_TOOL', activity_details=dict(event))
|
|
self._save(state, reason)
|
|
return reason
|
|
if event['state'] == 'RUNNING' and row is None:
|
|
started, cap = event['started_monotonic'], event['cap_s']
|
|
if (not isinstance(key, str) or not key or not math.isfinite(started) or
|
|
not math.isfinite(cap) or not 0 < cap <= (state['config'].get('managed_tool_deadline_s', state['config']['execution_deadline_s']) if event.get('tool_name') == 'terminal_exec' else state['config'].get('ordinary_tool_deadline_s', 300)) or
|
|
not state['started_monotonic'] <= started <= time.monotonic() + 1):
|
|
raise ExecutionStopped('TOOL_EVENT_INVALID')
|
|
row = dict(tool_call_id=key, tool_name=event['tool_name'], state='RUNNING',
|
|
started_monotonic=started, deadline_monotonic=started + cap, cap_s=cap)
|
|
calls[key] = row
|
|
changed = True
|
|
elif event['state'] in ('DONE', 'TIMED_OUT') and row is not None:
|
|
# A late result cannot move the absolute deadline or erase its expiry.
|
|
row['state'] = ('TIMED_OUT' if row['state'] == 'TIMED_OUT' or event['state'] == 'TIMED_OUT' or
|
|
event['observed_monotonic'] >= row['deadline_monotonic'] else 'DONE')
|
|
changed = True
|
|
overdue = next((v for v in calls.values() if v['state'] == 'TIMED_OUT' or
|
|
(v['state'] == 'RUNNING' and time.monotonic() >= v['deadline_monotonic'])), None)
|
|
active = overdue or next((v for v in calls.values() if v['state'] == 'RUNNING'), None)
|
|
if active:
|
|
state.update(activity='RUNNING_TOOL', activity_details=dict(active))
|
|
elif changed and state['activity'] == 'RUNNING_TOOL':
|
|
state.update(activity='WAITING_LLM', activity_details={})
|
|
if overdue:
|
|
overdue['state'] = 'TIMED_OUT'
|
|
state.update(execution_state='STOP_REQUESTED', cleanup_state='PENDING',
|
|
stop_reason='TOOL_DEADLINE_EXCEEDED', activity_details=dict(overdue))
|
|
if changed or overdue:
|
|
self._save(state, 'TOOL_DEADLINE_EXCEEDED' if overdue else 'TOOL_OBSERVED')
|
|
return state.get('stop_reason')
|
|
|
|
def record_progress(self, source, signals, *, phase=None, operation_id=None, role='tool'):
|
|
"""One typed sample drives health and the read-only monitor; no log-text heuristics."""
|
|
with self.locked():
|
|
state = self.snapshot()
|
|
self.assert_owner(state)
|
|
now, wall = time.monotonic(), time.time()
|
|
progress = state.setdefault('progress', {})
|
|
previous = progress.setdefault('sources', {}).get(source, {})
|
|
old = previous.get('signals', {})
|
|
changes = {key: (None if value is None or old.get(key) is None else value != old[key])
|
|
for key, value in signals.items()}
|
|
unknown = any(value is None for value in signals.values())
|
|
changed = any(value is True for value in changes.values())
|
|
idle_since = None if unknown or changed else previous.get('idle_since_monotonic') or now
|
|
threshold = state['config']['phases'][phase]['suspect_after_s'] if phase in PHASES else state['config']['suspect_after_s']
|
|
phase_row = state.get('phases', {}).get(operation_id, {})
|
|
waiting = phase_row.get('state') == 'QUEUED' or (not operation_id and state['activity'] in
|
|
('QUEUED', 'WAITING_QUOTA', 'WAITING_RESOURCE'))
|
|
health = ('WAITING' if waiting else 'UNKNOWN' if unknown else
|
|
'SUSPECT' if idle_since is not None and now - idle_since >= threshold else 'OBSERVED')
|
|
sample = dict(sampled_at=wall, sampled_monotonic=now, signals=signals, changed=changes,
|
|
idle_since_monotonic=idle_since, suspect_after_s=threshold, health=health,
|
|
role=role, operation_id=operation_id, phase=phase,
|
|
reason='SIGNAL_UNKNOWN' if unknown else 'ALL_SIGNALS_IDLE' if health == 'SUSPECT' else 'PROGRESS_OBSERVED')
|
|
progress['sources'][source] = sample
|
|
active = {key for key, value in state.get('phases', {}).items() if value['state'] in ('QUEUED', 'RUNNING')}
|
|
# Agent silence while a separately owned stage is active is expected.
|
|
if role != 'agent' or not active:
|
|
progress.update(sampled_at=wall, current_source=source, **{k: sample[k] for k in
|
|
('signals', 'changed', 'idle_since_monotonic', 'suspect_after_s', 'reason')})
|
|
state['health'] = health
|
|
return self._save(state, 'PROGRESS_SAMPLED', source=source, health=health)
|
|
|
|
def cancel(self, reason):
|
|
if not isinstance(reason, str) or not reason.strip():
|
|
raise ValueError('cancel requires a reason')
|
|
with self.locked():
|
|
state = self.snapshot()
|
|
if self.epoch is None: # Explicit control-plane cancellation binds under the same lock.
|
|
self.epoch = state['epoch']
|
|
self.assert_owner(state)
|
|
if state['execution_state'] in ('TERMINAL', 'STOP_REQUESTED'):
|
|
return dict(state, reason='ALREADY_TERMINAL')
|
|
if not state['cancel_requested']:
|
|
state['cancel_requested'] = dict(reason=reason.strip(), time=time.time(), epoch=state['epoch'])
|
|
state.update(execution_state='CANCEL_REQUESTED', cleanup_state='PENDING', stop_reason='CANCEL_REQUESTED')
|
|
self._save(state, 'CANCEL_REQUESTED', reason=reason.strip())
|
|
return state
|
|
|
|
def quarantine(self, reason):
|
|
with self.locked():
|
|
state = self.snapshot()
|
|
self.assert_owner(state)
|
|
state.update(cleanup_state='QUARANTINED', stop_reason=reason)
|
|
if state['execution_state'] == 'ACTIVE':
|
|
state['execution_state'] = 'PAUSED_INFRA'
|
|
return self._save(state, 'RESOURCE_QUARANTINED', reason=reason)
|
|
|
|
def _checkpoint(self, value):
|
|
if not isinstance(value, dict) or not (value.get('resumable') is True and value.get('phase') == 'PAUSED_INFRA' or value.get('complete') is True and 'resumable' not in value):
|
|
raise ExecutionStopped('CHECKPOINT_INCOMPLETE')
|
|
supplied = Path(value.get('path', ''))
|
|
path = supplied.resolve()
|
|
if supplied.is_symlink() or not path.is_relative_to(self.rd) or not path.is_file():
|
|
raise ExecutionStopped('CHECKPOINT_PATH_INVALID')
|
|
digest = hashlib.sha256(path.read_bytes()).hexdigest()
|
|
if digest != value.get('sha256'):
|
|
raise ExecutionStopped('CHECKPOINT_HASH_MISMATCH')
|
|
return dict(value, path=str(path))
|
|
|
|
def pause(self, reason, checkpoint):
|
|
if not isinstance(reason, str) or not reason.strip():
|
|
raise ValueError('pause requires a reason')
|
|
with self.admission() as state:
|
|
checkpoint = self._checkpoint(checkpoint)
|
|
state.update(execution_state='PAUSED_INFRA', stop_reason=reason,
|
|
checkpoint=checkpoint, paused_at=time.time())
|
|
return self._save(state, 'EXECUTION_PAUSED', reason=reason)
|
|
|
|
def resume(self, *, automatic=True, counted=True):
|
|
with self.locked():
|
|
state = self.snapshot()
|
|
if self.epoch is not None and self.epoch != state['epoch']:
|
|
raise ExecutionStopped('OWNER_FENCED')
|
|
resume_policy(state)
|
|
if state['execution_state'] != 'PAUSED_INFRA' or state.get('cancel_requested'):
|
|
raise ExecutionStopped('EXECUTION_NOT_RESUMABLE')
|
|
if state['cleanup_state'] == 'QUARANTINED':
|
|
raise ExecutionStopped('SIDE_EFFECT_UNKNOWN')
|
|
if state.get('boot_id') != boot_id():
|
|
raise ExecutionStopped('BOOT_CHANGED_RECONCILE_REQUIRED')
|
|
if time.monotonic() >= state['deadline_monotonic']:
|
|
raise ExecutionStopped('EXECUTION_DEADLINE_EXCEEDED')
|
|
self._checkpoint(state.get('checkpoint'))
|
|
owner_status = process_status(state['owner'])
|
|
same_owner = process_key(state['owner']) == process_key(process_identity(os.getpid()))
|
|
if owner_status != 'DEAD' and not same_owner:
|
|
raise ExecutionStopped('OWNER_NOT_RELEASABLE')
|
|
members = owned_processes(state)
|
|
pending = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE']
|
|
if members != [] or pending:
|
|
raise ExecutionStopped('SIDE_EFFECT_UNKNOWN' if members is None else 'WORK_STILL_ACTIVE')
|
|
count = state.get('auto_resume_count', 0)
|
|
if automatic and counted and count >= state['config'].get('max_auto_resumes', 3):
|
|
raise ExecutionStopped('AUTO_RESUME_BUDGET_EXHAUSTED')
|
|
state['epoch'] += 1
|
|
self.epoch = state['epoch']
|
|
state.update(owner=process_identity(os.getpid()), execution_state='ACTIVE', stop_reason=None,
|
|
auto_resume_count=count + int(automatic and counted), resumed_at=time.time(), activity='WAITING_QUOTA')
|
|
return self._save(state, 'EXECUTION_RESUMED', automatic=automatic)
|
|
|
|
def cleanup(self):
|
|
"""Bounded cancellation cleanup; unknown remote receipts keep leases isolated."""
|
|
with self.locked():
|
|
state = self.snapshot()
|
|
self.assert_owner(state)
|
|
if not state.get('cancel_requested'):
|
|
raise ExecutionStopped('CLEANUP_REQUIRES_CANCEL')
|
|
if state['execution_state'] == 'TERMINAL':
|
|
return state
|
|
state['execution_state'] = 'CLEANING'
|
|
self._save(state, 'CLEANUP_STARTED')
|
|
policy = state['config']
|
|
start = time.monotonic()
|
|
while True:
|
|
members = owned_processes(state)
|
|
latest = self.snapshot()
|
|
self.assert_owner(latest)
|
|
pending = [v for v in latest.get('operations', {}).values() if v.get('state') != 'DONE']
|
|
if members == [] and not pending:
|
|
break
|
|
elapsed = time.monotonic() - start
|
|
if members is None or elapsed >= policy['cleanup_deadline_s']:
|
|
return self.quarantine('CLEANUP_UNCONFIRMED')
|
|
# Each supervisor performs its own bounded TERM/KILL/reap sequence.
|
|
# Killing the reaper itself would destroy the complete-set proof.
|
|
for scope in latest.get('process_scopes', {}).values():
|
|
signal_process(scope['binding']['supervisor'], signal.SIGTERM)
|
|
time.sleep(.1)
|
|
with self.locked():
|
|
latest = self.snapshot()
|
|
self.assert_owner(latest)
|
|
if latest['cleanup_state'] != 'QUARANTINED':
|
|
latest['cleanup_state'] = 'VERIFIED'
|
|
return self._save(latest, 'CLEANUP_FINISHED', cleanup_state=latest['cleanup_state'])
|
|
|
|
def finish(self, exit_code, *, protocol_complete, stop_reason=None, cleanup_state='VERIFIED'):
|
|
with self.locked():
|
|
state = self.snapshot()
|
|
if self.epoch != state['epoch']:
|
|
raise ExecutionStopped('OWNER_FENCED')
|
|
if state['execution_state'] == 'TERMINAL':
|
|
return state
|
|
reason = stop_reason or state.get('stop_reason')
|
|
members = owned_processes(state) if state['processes'] else []
|
|
pending = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE']
|
|
known = [process_status(p) for p in state['processes']]
|
|
if (members not in ([], None) or pending or 'ALIVE' in known or
|
|
(Path('/proc').is_dir() and ('UNKNOWN' in known or members is None))):
|
|
cleanup_state = 'QUARANTINED'
|
|
reason = reason or 'CLEANUP_UNCONFIRMED'
|
|
if state['cleanup_state'] == 'QUARANTINED' or cleanup_state not in ('VERIFIED', 'NOT_NEEDED'):
|
|
cleanup_state = 'QUARANTINED'
|
|
if state['cancel_requested']:
|
|
outcome = 'CANCELLED'
|
|
elif reason == 'OWNER_LOST':
|
|
outcome = 'LOST'
|
|
elif reason and ('DEADLINE' in reason or 'hard_ceiling' in reason):
|
|
outcome = 'DEADLINE_EXCEEDED'
|
|
elif reason and 'BUDGET' in reason:
|
|
outcome = 'BUDGET_EXHAUSTED'
|
|
elif reason and any(word in reason for word in ('INFRA', 'UNKNOWN', 'monitor failed')):
|
|
outcome = 'INFRA_ERROR'
|
|
elif exit_code == 0 and protocol_complete and cleanup_state == 'VERIFIED':
|
|
outcome = 'COMPLETED'
|
|
else:
|
|
outcome = 'AGENT_ERROR'
|
|
state.update(execution_state='TERMINAL', execution_outcome=outcome, cleanup_state=cleanup_state,
|
|
stop_reason=reason, exit_code=exit_code, protocol_complete=bool(protocol_complete),
|
|
signal=(-exit_code if isinstance(exit_code, int) and exit_code < 0 else
|
|
exit_code - 128 if isinstance(exit_code, int) and exit_code > 128 else None),
|
|
finished_at=time.time())
|
|
return self._save(state, 'EXECUTION_FINISHED', outcome=outcome, reason=reason)
|
|
|
|
def reconcile(self):
|
|
"""Fail closed; never replay an unknown side effect or take a live owner."""
|
|
with self.locked():
|
|
state = self.snapshot()
|
|
if self.epoch is not None:
|
|
self.assert_owner(state)
|
|
if state['execution_state'] == 'TERMINAL':
|
|
return state
|
|
owner = process_status(state['owner'])
|
|
members = owned_processes(state)
|
|
operations = [v for v in state.get('operations', {}).values() if v.get('state') != 'DONE']
|
|
if owner == 'ALIVE':
|
|
return dict(state, reconciliation='OWNER_ALIVE')
|
|
if state['execution_state'] == 'PAUSED_INFRA' and owner == 'DEAD' and members == [] and not operations:
|
|
try:
|
|
self._checkpoint(state.get('checkpoint'))
|
|
except ExecutionStopped:
|
|
pass
|
|
else:
|
|
return dict(state, reconciliation='CHECKPOINT_READY_FOR_RESUME')
|
|
state['epoch'] += 1 # Fence every old dispatch path before changing ownership.
|
|
self.epoch = state['epoch']
|
|
if owner == 'DEAD' and members == [] and not operations:
|
|
state.update(execution_state='TERMINAL', execution_outcome='LOST', cleanup_state='VERIFIED',
|
|
stop_reason='OWNER_LOST', finished_at=time.time())
|
|
else:
|
|
state.update(execution_state='PAUSED_INFRA', cleanup_state='QUARANTINED',
|
|
stop_reason='SIDE_EFFECT_UNKNOWN' if operations else 'OWNERSHIP_UNKNOWN')
|
|
return self._save(state, 'EXECUTION_RECONCILED', owner_status=owner,
|
|
members=members, pending_operations=len(operations))
|
|
|
|
|
|
def from_environment():
|
|
value = os.environ.get('BENCH_EXECUTION_DIR')
|
|
if not value:
|
|
return None
|
|
result = Execution(value)
|
|
result.epoch = int(os.environ['BENCH_EXECUTION_EPOCH'])
|
|
return result
|