The 10-21 03:16 Mac mini reboot left 7 GLM rollouts (05: densenet201,
efficientnet.b2, rtmo.s; 06: rtmpose.x, rtmo.m, hypersim.base, yolo11n.seg)
quarantined OPERATION_STATE_UNKNOWN while still counted RUNNING: each had a
05-compile in flight, its reaper died with the VM before writing a receipt, and
_validate_recovered_receipt demanded one ('scope cleanup unconfirmed') although
scope_cleaned already accepts an earlier boot. Without a receipt an earlier-boot
scope now proves cleanup for a failed result and for the inline 04 whose meta is
hashed; any other success still needs its receipt.
speed_ref: stages/ref of an interrupted run kept bundle.tar, which salty_dev
creates exclusively, so the 3 in-flight references failed with FileExistsError
every 2 minutes and took all 3 workers. The leftover dir is renamed aside.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
695 lines
40 KiB
Python
695 lines
40 KiB
Python
"""Per-rollout outer worker. Only enumerates this rollout's handles directory.
|
|
|
|
Untrusted tickets are copied into a protected queue. Never execute or write
|
|
through a client-supplied path; status publication uses a pinned directory fd.
|
|
"""
|
|
import json
|
|
import hashlib
|
|
import fcntl
|
|
import sys
|
|
import os
|
|
import re
|
|
import stat
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
import yaml
|
|
from pathlib import Path
|
|
from . import stagerun, execution as lifecycle, watchdog
|
|
from .stages import AGENT_RUNNABLE
|
|
|
|
|
|
def read_regular(parent_fd, name, limit=1024 * 1024):
|
|
fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent_fd)
|
|
try:
|
|
if not stat.S_ISREG(os.fstat(fd).st_mode) or (limit is not None and os.fstat(fd).st_size > limit):
|
|
raise ValueError('ticket/config must be a regular file <= 1 MiB')
|
|
if limit is not None:
|
|
return os.read(fd, limit + 1)
|
|
with os.fdopen(os.dup(fd), 'rb') as stream:
|
|
return stream.read()
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def config_bytes(ws, value, limit=1024 * 1024):
|
|
p = Path(value)
|
|
rel = p.relative_to(ws)
|
|
if not rel.parts or '..' in rel.parts:
|
|
raise ValueError('config outside workspace')
|
|
fd = os.open(ws, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
try:
|
|
for part in rel.parts[:-1]:
|
|
nxt = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
|
|
os.close(fd); fd = nxt
|
|
return read_regular(fd, rel.name, limit)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def workspace_relative(ws, value, field):
|
|
"""A path the agent names inside its workspace, kept relative: 04 resolves it in the development
|
|
environment, where files made by `bench dev` live (they may not exist on the controller)."""
|
|
p = Path(value) if isinstance(value, str) and value else None
|
|
if p is not None and p.is_absolute():
|
|
try:
|
|
p = p.relative_to(ws)
|
|
except ValueError:
|
|
p = None
|
|
if p is None or not p.parts or '..' in p.parts:
|
|
raise ValueError(f'{field} must be a path inside the workspace')
|
|
return p.as_posix()
|
|
|
|
|
|
def snapshot_config(ws, value):
|
|
"""Keep path-bearing options from smuggling outer-worker file access."""
|
|
cfg = yaml.safe_load(config_bytes(ws, value)) or {}
|
|
allowed = {
|
|
'input_parameters': {'input_type_rt', 'input_type_train', 'input_layout_train', 'input_layout_rt',
|
|
'input_name', 'input_shape', 'mean_value', 'scale_value', 'std_value'},
|
|
'calibration_parameters': {'calibration_type', 'max_percentile', 'per_channel', 'cal_data_dir',
|
|
'run_on_cpu', 'run_on_bpu', 'optimization', 'quant_config'},
|
|
'compiler_parameters': {'compile_mode', 'optimize_level', 'jobs', 'debug'},
|
|
'verification_parameters': {'hbm_bc_samples'},
|
|
}
|
|
if not isinstance(cfg, dict):
|
|
raise ValueError('config must be a mapping')
|
|
for section, values in cfg.items():
|
|
if section == 'model_parameters':
|
|
# The graph is the agent's (04 checks its inputs/outputs against the platform graph);
|
|
# march, working_dir and output names stay pinned, so other keys are ignored as before.
|
|
if isinstance(values, dict) and 'onnx_model' in values:
|
|
values['onnx_model'] = workspace_relative(ws, values['onnx_model'], 'model_parameters.onnx_model')
|
|
continue
|
|
if section not in allowed or not isinstance(values, dict) or set(values)-allowed[section]:
|
|
extra = sorted(set(values) - allowed[section]) if section in allowed and isinstance(values, dict) else []
|
|
raise ValueError('unsupported config keys: ' + str(section) + (f' {extra}' if extra else ''))
|
|
n = (cfg.get('verification_parameters') or {}).get('hbm_bc_samples', 0)
|
|
if type(n) is not int or not 0 <= n <= 16:
|
|
raise ValueError('verification_parameters.hbm_bc_samples must be an integer in [0, 16]')
|
|
cal = cfg.get('calibration_parameters', {})
|
|
if 'cal_data_dir' in cal:
|
|
cal['cal_data_dir'] = workspace_relative(ws, cal['cal_data_dir'], 'calibration_parameters.cal_data_dir')
|
|
quant = cal.get('quant_config')
|
|
if isinstance(quant, str):
|
|
p = Path(quant)
|
|
if not p.is_absolute():
|
|
p = ws / p
|
|
cal['quant_config'] = json.loads(config_bytes(ws, str(p)))
|
|
if cal.get('quant_config') is not None and not isinstance(cal['quant_config'], dict):
|
|
raise ValueError('quant_config must be an inline mapping or workspace JSON')
|
|
return yaml.safe_dump(cfg, sort_keys=False).encode()
|
|
|
|
|
|
def publish(fd, status):
|
|
# Exclusive temporary creation + rename replaces a malicious status symlink.
|
|
import uuid
|
|
name = '.status-' + uuid.uuid4().hex
|
|
f = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=fd)
|
|
with os.fdopen(f, 'w') as out:
|
|
json.dump(status, out, ensure_ascii=False)
|
|
os.rename(name, 'status.json', src_dir_fd=fd, dst_dir_fd=fd)
|
|
|
|
|
|
class Worker:
|
|
def __init__(self, rd, *, test_noop=False, execution=None):
|
|
self.rd = Path(rd).resolve()
|
|
self.test_noop = test_noop and 'guard-smoke' in self.rd.parts
|
|
self.execution = execution
|
|
if self.execution is None and (self.rd / 'execution.json').exists():
|
|
self.execution = lifecycle.Execution(self.rd).attach()
|
|
self.stop_event = threading.Event()
|
|
self.thread = threading.Thread(target=self.run, name='stage-ticket-worker', daemon=True)
|
|
self.seen = set()
|
|
|
|
def __enter__(self):
|
|
self.thread.start(); return self
|
|
|
|
def __exit__(self, *exc):
|
|
self.stop_event.set() # Drain stops new intake; explicit cancel is separate.
|
|
budget = self.execution.policy()['cleanup_deadline_s'] if self.execution else 60
|
|
if self.execution:
|
|
if exc[0] is not None:
|
|
self.execution.cancel('WORKER_CONTEXT_ERROR')
|
|
# The existing stage watchdog retains its phase and execution clocks.
|
|
# Cleanup time only bounds exit confirmation after a stop, not work.
|
|
while self.thread.is_alive() and not self.execution.stop_reason():
|
|
self.thread.join(timeout=.5)
|
|
else:
|
|
self.thread.join(timeout=budget)
|
|
if self.thread.is_alive():
|
|
self.thread.join(timeout=budget)
|
|
if self.thread.is_alive():
|
|
if self.execution:
|
|
self.execution.quarantine('WORKER_JOIN_UNCONFIRMED')
|
|
raise RuntimeError('worker did not stop within cleanup budget')
|
|
if self.execution:
|
|
# The scanner may have stopped with complete work waiting on SQLite.
|
|
# Spend one shared cleanup budget on registration, never dispatch.
|
|
if not self.reconcile_completed(deadline=time.monotonic() + budget):
|
|
self.execution.quarantine('INFRA_STAGE_REGISTRATION_PENDING')
|
|
|
|
def run(self):
|
|
while not self.stop_event.is_set():
|
|
if self.execution and self.execution.stop_reason():
|
|
return
|
|
self.scan()
|
|
self.stop_event.wait(0.5)
|
|
|
|
def scan(self):
|
|
for hd in sorted((self.rd / 'handles').iterdir()):
|
|
if self.stop_event.is_set():
|
|
return
|
|
if hd.name in self.seen or not re.fullmatch(r'[a-zA-Z0-9_-]+', hd.name):
|
|
continue
|
|
try:
|
|
fd = os.open(hd, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
except OSError:
|
|
continue
|
|
raw_request = b""
|
|
try:
|
|
try:
|
|
raw_request = read_regular(fd, 'request.json')
|
|
req = json.loads(raw_request)
|
|
except FileNotFoundError:
|
|
continue
|
|
self.seen.add(hd.name)
|
|
self.execute(hd.name, fd, req)
|
|
except Exception as e:
|
|
self.seen.add(hd.name)
|
|
queue = self.rd / 'stages' / '_queue' / hd.name
|
|
queue.mkdir(parents=True, exist_ok=True)
|
|
with (queue / 'runner.log').open('a') as log:
|
|
log.write(f'invalid_request: {type(e).__name__}\n')
|
|
reason = str(e) if isinstance(e, lifecycle.ExecutionStopped) else 'invalid_request'
|
|
detail = str(e)[:1000] if isinstance(e, ValueError) else type(e).__name__
|
|
if (queue / 'operation.json').exists():
|
|
# A malformed persisted state is not a new invalid ticket.
|
|
reason = 'OPERATION_STATE_UNKNOWN'
|
|
if self.execution:
|
|
self.execution.quarantine(reason)
|
|
self._publish(fd, dict(self._failed(reason), error_detail=detail))
|
|
finally:
|
|
os.close(fd)
|
|
|
|
def _publish(self, fd, value):
|
|
value = {key: item for key, item in value.items() if key != 'prerequisites'}
|
|
if value.get('db_registered') is False:
|
|
value = {'state': 'RUNNING', 'registration': 'PENDING', 'attempt': value.get('attempt')}
|
|
# A terminal observation must not prompt another model request after
|
|
# cleanup has fenced this execution. Its owner handles the stop reason.
|
|
if value.get('state') == 'DONE' and self.execution and self.execution.stop_reason():
|
|
return
|
|
with self.execution.mutation() if self.execution else stagerun.nullcontext():
|
|
publish(fd, value)
|
|
|
|
def _write(self, path, value):
|
|
with self.execution.mutation() if self.execution else stagerun.nullcontext():
|
|
if path.name == 'status.json':
|
|
operation = json.loads((path.parent / 'operation.json').read_text())
|
|
value = dict(value, **self._receipt_identity(operation))
|
|
lifecycle.atomic_json(path, value)
|
|
|
|
def _board_never_admitted(self, hid):
|
|
"""phase_running fires only while the board lock is held, so a phase
|
|
that never left QUEUED (e.g. QUEUE_DEADLINE_EXCEEDED) touched no board."""
|
|
phase = self.execution.snapshot().get('phases', {}).get(hid) if self.execution else None
|
|
return bool(phase) and phase.get('execution_started_monotonic') is None
|
|
|
|
@staticmethod
|
|
def _receipt_identity(operation):
|
|
return {key: operation.get(key) for key in
|
|
('operation_id', 'payload_sha256', 'execution_id', 'epoch')}
|
|
|
|
def execute(self, hid, fd, req):
|
|
queue = self.rd / 'stages' / '_queue' / hid
|
|
queue.mkdir(parents=True, exist_ok=True)
|
|
# A replay may observe a current receipt but must never race a second dispatch.
|
|
with (queue / 'operation.lock').open('a') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
status = queue / 'status.json'
|
|
value = json.loads(status.read_text()) if status.exists() else {'state': 'QUEUED'}
|
|
self._publish(fd, {'state': 'RUNNING'} if value.get('state') == 'DONE' else value)
|
|
return
|
|
self._execute(hid, fd, req)
|
|
|
|
def _execute(self, hid, fd, req):
|
|
name = req.get('stage')
|
|
if name not in AGENT_RUNNABLE and name != 'terminal' and not (name == 'noop' and self.test_noop):
|
|
raise ValueError('stage not allowed')
|
|
queue = self.rd / 'stages' / '_queue' / hid
|
|
digest = hashlib.sha256(json.dumps(req, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
|
|
queue.mkdir(parents=True, exist_ok=True)
|
|
operation = queue / 'operation.json'
|
|
if operation.exists():
|
|
previous = json.loads(operation.read_text())
|
|
if previous['payload_sha256'] != digest:
|
|
self._publish(fd, self._failed('OPERATION_PAYLOAD_CONFLICT'))
|
|
return
|
|
status = queue / 'status.json'
|
|
if status.exists() and json.loads(status.read_text()).get('state') == 'DONE':
|
|
result = json.loads(status.read_text())
|
|
ledger = self.execution.snapshot().get('operations', {}).get(hid) if self.execution else None
|
|
if (result.get('error_class') != 'SIDE_EFFECT_UNKNOWN' and result.get('db_registered') is not False
|
|
and previous.get('state') == 'DONE' and ledger == previous):
|
|
# Settled before a pause (same skip as reconcile_completed). A later attempt of the
|
|
# rollout may overwrite its outputs (one .hbm path per rollout), so re-proving them
|
|
# on resume quarantined every multi-compile rollout (10-12/13: 19 rollouts).
|
|
self._publish(fd, result)
|
|
return
|
|
if result.get('error_class') == 'SIDE_EFFECT_UNKNOWN':
|
|
if self.execution:
|
|
self.execution.quarantine('SIDE_EFFECT_UNKNOWN')
|
|
else:
|
|
if 'meta_sha256' not in result:
|
|
self._validate_recovered_receipt(hid, name, previous, result)
|
|
result = self._reconcile_stage(hid, name, previous, result)
|
|
if result.get('db_registered') is False:
|
|
self.seen.discard(hid)
|
|
else:
|
|
self._complete_operation(hid, operation, previous)
|
|
self._publish(fd, result)
|
|
return
|
|
if previous.get('executor') and lifecycle.process_status(previous['executor']) == 'ALIVE':
|
|
self._publish(fd, json.loads(status.read_text()) if status.exists() else {'state': 'RUNNING'})
|
|
return
|
|
result = self._failed('SIDE_EFFECT_UNKNOWN')
|
|
self._write(queue / 'status.json', result)
|
|
self._publish(fd, result)
|
|
if self.execution:
|
|
self.execution.quarantine('SIDE_EFFECT_UNKNOWN')
|
|
return
|
|
if self.execution:
|
|
with self.execution.admission() as state:
|
|
previous = dict(operation_id=hid, payload_sha256=digest, state='PREPARING',
|
|
execution_id=state['execution_id'], epoch=state['epoch'],
|
|
received_at=time.time(), received_monotonic=time.monotonic())
|
|
lifecycle.atomic_json(operation, previous)
|
|
state['operations'][hid] = previous
|
|
self.execution._save(state, 'OPERATION_PREPARED', operation_id=hid, payload_sha256=digest)
|
|
else:
|
|
previous = dict(operation_id=hid, payload_sha256=digest, state='PREPARING', received_at=time.time())
|
|
lifecycle.atomic_json(operation, previous)
|
|
try:
|
|
trusted = self._prepare_request(queue, req)
|
|
except Exception as exc:
|
|
# This attempt has not crossed the dispatch boundary. Persist a
|
|
# reproducible rejection, not an ambiguous side-effect outcome.
|
|
with (queue / 'runner.log').open('a') as log:
|
|
log.write(f'invalid_request: {type(exc).__name__}\nrequest: sha256={digest}\n')
|
|
result = dict(self._failed('invalid_request'), error_detail=str(exc)[:1000] if isinstance(exc, ValueError) else type(exc).__name__)
|
|
self._write(queue / 'status.json', result)
|
|
self._publish(fd, result)
|
|
self._complete_operation(hid, operation, previous)
|
|
return
|
|
self._write(queue / 'request.json', trusted)
|
|
if name == 'noop':
|
|
result = {'state': 'DONE', 'result': 'OK', 'exit_code': 0, 'error_class': None,
|
|
'duration_s': 0, 'stdout': str(queue / 'runner.log'), 'stderr': '', 'meta': '',
|
|
'worker_pid': os.getpid()}
|
|
self._write(queue / 'status.json', result)
|
|
self._publish(fd, result)
|
|
else:
|
|
if self.execution:
|
|
self.execution.begin_phase(hid, name, queued_monotonic=previous['received_monotonic'])
|
|
env = dict(os.environ, BENCH_ROOT=str(stagerun.paths.BENCH_ROOT),
|
|
PYTHONPATH=str(stagerun.paths.BENCH_ROOT), BENCH_EXECUTION_PHASE=name,
|
|
BENCH_BOARD_RECEIPT_DIR=str(queue / 'board-receipts'), BENCH_OPERATION_ID=hid)
|
|
py = stagerun.paths.config()['paths'].get('harness_python', sys.executable)
|
|
# A process boundary makes Python/native stages cancellable too.
|
|
def started(proc):
|
|
previous.update(state='DISPATCHED', executor=lifecycle.process_identity(proc.pid))
|
|
if self.execution:
|
|
with self.execution.locked():
|
|
state = self.execution.snapshot()
|
|
self.execution.assert_owner(state)
|
|
lifecycle.atomic_json(operation, previous)
|
|
state['operations'][hid] = previous
|
|
self.execution._save(state, 'OPERATION_DISPATCHED', operation_id=hid)
|
|
else:
|
|
lifecycle.atomic_json(operation, previous)
|
|
last_status = None
|
|
def publish_progress():
|
|
nonlocal last_status
|
|
status_file = queue / 'status.json'
|
|
if status_file.exists():
|
|
status = json.loads(status_file.read_text())
|
|
# The child can finish before watchdog/board cleanup does.
|
|
if status.get('state') != 'DONE' and status != last_status:
|
|
self._publish(fd, status)
|
|
last_status = status
|
|
return None
|
|
policy = self.execution.policy(name) if self.execution else None
|
|
with self.execution.scope(name, hid) if self.execution else stagerun.nullcontext():
|
|
result_run = watchdog.run([py, '-m', 'harness.stagerun', '--exec-protected', str(queue), str(self.rd)],
|
|
cwd=stagerun.paths.BENCH_ROOT, stdout=queue / 'runner.log', stderr=queue / 'runner.stderr.log',
|
|
env=env, execution=self.execution, role='stage', on_start=started,
|
|
stop_check=publish_progress, poll_s=1,
|
|
hard_ceiling_s=(max(.01, trusted['deadline_monotonic'] - time.monotonic()) + policy['cleanup_deadline_s'] if name == 'terminal' else (policy['queue_deadline_s'] + policy['execution_deadline_s']) if policy else 86400))
|
|
publish_progress()
|
|
status_file = queue / 'status.json'
|
|
result = json.loads(status_file.read_text()) if status_file.exists() else {}
|
|
if result.get('state') == 'DONE' and result_run.state != 'OK':
|
|
# The child's own verdict (e.g. TERMINAL_DEADLINE_EXCEEDED) beats a bare executor exit: 10-19 staging,
|
|
# a board command that never got a board reached the agent as STAGE_EXECUTOR_EXIT with no reason.
|
|
result.setdefault('cleanup_state', result_run.cleanup_state)
|
|
self._write(status_file, result)
|
|
elif result.get('state') != 'DONE':
|
|
progress = result
|
|
result = dict(self._failed(result_run.stalled_reason or 'STAGE_EXECUTOR_EXIT', result_run.exit_code),
|
|
**{key: progress[key] for key in ('attempt', 'started_at', 'queued_at', 'pool', 'board_name', 'prerequisites')
|
|
if key in progress})
|
|
result['cleanup_state'] = result_run.cleanup_state
|
|
self._write(status_file, result)
|
|
self._check_board_cleanup(hid, name, trusted, result)
|
|
if self.execution and self.execution.snapshot().get('cancel_requested') and 'meta_sha256' not in result:
|
|
# A completed protected receipt records computation, while the
|
|
# execution retains cancellation and suppresses external DONE.
|
|
result = dict(result, result='CANCELLED', error_class='CANCEL_REQUESTED')
|
|
self._write(status_file, result)
|
|
result = json.loads(status_file.read_text())
|
|
result = self._reconcile_stage(hid, name, previous, result,
|
|
cleanup_state=result_run.cleanup_state)
|
|
self._publish(fd, result)
|
|
if result.get('db_registered') is False:
|
|
self.seen.discard(hid)
|
|
return
|
|
self._complete_operation(hid, operation, previous)
|
|
|
|
def _check_board_cleanup(self, hid, name, trusted, result):
|
|
if self.execution and (name in ('07-board', '08-eval') or (name == 'terminal' and trusted.get('target') == 'board')) and result.get('result') != 'SKIPPED':
|
|
queue = self.rd / 'stages' / '_queue' / hid
|
|
receipts = list((queue / 'board-receipts').glob('*.json'))
|
|
if ((not receipts and not self._board_never_admitted(hid))
|
|
or any(json.loads(p.read_text()).get('cleanup_state') != 'VERIFIED' for p in receipts)):
|
|
self.execution.quarantine('BOARD_CLEANUP_UNCONFIRMED')
|
|
|
|
def settle_interrupted(self):
|
|
"""The collector died (crash, kill, planned handover, reboot): an admitted operation whose executor is gone
|
|
ended with it. Record it exactly like an executor exit under a live collector (a system failure, so a compile
|
|
is free) and tell the agent's handle, so the rollout can resume. Board work without a clean board receipt
|
|
still quarantines. Caller proved every process of the execution gone; nothing here dispatches."""
|
|
for hid, ledger in self.execution.snapshot().get('operations', {}).items():
|
|
queue = self.rd / 'stages' / '_queue' / hid
|
|
status_file = queue / 'status.json'
|
|
progress = json.loads(status_file.read_text()) if status_file.exists() else {}
|
|
if ledger.get('state') == 'DONE' or progress.get('state') == 'DONE':
|
|
continue # settled, or a completed receipt for reconcile_completed
|
|
if ledger.get('executor') and lifecycle.process_status(ledger['executor']) != 'DEAD':
|
|
return False
|
|
request = queue / 'request.json' if (queue / 'request.json').exists() else self.rd / 'handles' / hid / 'request.json'
|
|
trusted = json.loads(request.read_text())
|
|
previous = json.loads((queue / 'operation.json').read_text())
|
|
result = dict(self._failed('STAGE_EXECUTOR_EXIT'),
|
|
note='平台的调度程序重启,打断了这一步;不计编译次数,需要的话重新提交',
|
|
cleanup_state='VERIFIED',
|
|
**{key: progress[key] for key in ('attempt', 'started_at', 'queued_at', 'pool', 'board_name', 'prerequisites')
|
|
if key in progress})
|
|
self._write(status_file, result)
|
|
self._check_board_cleanup(hid, trusted['stage'], trusted, result)
|
|
result = self._reconcile_stage(hid, trusted['stage'], previous, json.loads(status_file.read_text()),
|
|
cleanup_state='VERIFIED')
|
|
if result.get('db_registered') is False:
|
|
return False
|
|
# Not _publish: it withholds DONE from a stopped execution, and this one's owner is dead until it resumes;
|
|
# the resumed agent must find its handle finished, not wait on it forever.
|
|
handle = os.open(self.rd / 'handles' / hid, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
try:
|
|
with self.execution.mutation():
|
|
publish(handle, {key: item for key, item in result.items() if key != 'prerequisites'})
|
|
finally:
|
|
os.close(handle)
|
|
self._complete_operation(hid, queue / 'operation.json', previous)
|
|
return True
|
|
|
|
def _reconcile_stage(self, hid, name, operation, result, *, cleanup_state=None, deadline=None, write_status=True):
|
|
"""Replay one admitted completion after cleanup, without repeating work."""
|
|
deadline = deadline if deadline is not None else time.monotonic() + 1.0
|
|
if result.get('stage', name) != name:
|
|
if name != '05-compile' or result['stage'] != '04-calibrate':
|
|
raise ValueError('completed stage does not match submitted operation')
|
|
name = result['stage'] # Inline calibration failed before compilation.
|
|
for prerequisite in result.get('prerequisites', []):
|
|
if name != '05-compile' or prerequisite.get('stage') != '04-calibrate':
|
|
raise ValueError('unexpected completed prerequisite')
|
|
recovered = self._reconcile_stage(hid, '04-calibrate', operation, prerequisite,
|
|
deadline=deadline, write_status=False)
|
|
if recovered.get('db_registered') is False:
|
|
pending = dict(result, db_registered=False)
|
|
self._write(self.rd / 'stages' / '_queue' / hid / 'status.json', pending)
|
|
return pending
|
|
attempt = result.get('attempt')
|
|
completion = 'meta_sha256' in result
|
|
if completion and (not self.execution or type(attempt) is not int or attempt < 0):
|
|
raise ValueError('completed attempt has no managed execution or valid attempt')
|
|
if (not write_status or result.get('db_registered') is False) and result.get('result') in ('OK', 'SKIPPED') and not completion:
|
|
raise ValueError('completed attempt proof missing')
|
|
if (not self.execution or name not in AGENT_RUNNABLE
|
|
or type(attempt) is not int or attempt < 0
|
|
or (not completion and result.get('result') in ('OK', 'SKIPPED'))
|
|
or (not completion and (cleanup_state or result.get('cleanup_state')) != 'VERIFIED')):
|
|
return result
|
|
identity = self._receipt_identity(operation)
|
|
state = self.execution.snapshot()
|
|
self.execution.assert_owner(state)
|
|
if (identity != self._receipt_identity(state['operations'].get(hid, {}))
|
|
or identity != self._receipt_identity(result) or identity['operation_id'] != hid
|
|
or identity['execution_id'] != state['execution_id']
|
|
or (operation.get('state') != 'DONE' and identity['epoch'] != state['epoch'])):
|
|
raise ValueError('interrupted attempt identity mismatch')
|
|
sd = self.rd / 'stages' / name / f'a{attempt:02d}'
|
|
meta_path = sd / 'meta.json'
|
|
job = json.loads((self.rd.parent.parent / 'job.json').read_text())
|
|
key = (job['run_id'], job['job_id'], int(self.rd.name), name, attempt)
|
|
while True:
|
|
try:
|
|
# Probe availability before hashing large HBM bytes on every
|
|
# scan. Release the transaction before any artifact IO; final
|
|
# registration still rechecks owner and the admitted row.
|
|
if completion:
|
|
with stagerun.db.connect(timeout_s=0) as con:
|
|
con.execute('BEGIN IMMEDIATE')
|
|
self._validate_recovered_receipt(hid, name, operation, result,
|
|
prerequisite=not write_status)
|
|
meta = json.loads(meta_path.read_text()) if meta_path.exists() else None
|
|
if meta and (meta.get('stage') != name or meta.get('attempt') != attempt):
|
|
raise ValueError('interrupted attempt metadata mismatch')
|
|
# No execution lock is held while SQLite is busy. A zero busy
|
|
# timeout bounds setup, BEGIN, UPDATE and COMMIT by one budget.
|
|
with stagerun.db.connect(timeout_s=0) as con:
|
|
con.execute('BEGIN IMMEDIATE')
|
|
self.execution.assert_owner(self.execution.snapshot())
|
|
row = con.execute('SELECT * FROM stages WHERE run_id=? AND job_id=? AND rollout=? AND stage=? AND attempt=?',
|
|
key).fetchone()
|
|
if row is None:
|
|
if completion:
|
|
raise ValueError('completed attempt was not admitted')
|
|
return result
|
|
if meta is not None:
|
|
expected = (meta['state'], meta.get('exit_code'), meta.get('error_class'))
|
|
if row['state'] in ('RUNNING', 'QUEUED'):
|
|
stagerun.db.stage_finish(con, job['run_id'], job['job_id'], name, attempt,
|
|
*expected, rollout=int(self.rd.name))
|
|
elif tuple(row[k] for k in ('state', 'exit_code', 'error_class')) != expected:
|
|
raise ValueError('completed attempt conflicts with terminal database state')
|
|
elif row['state'] not in ('RUNNING', 'QUEUED'):
|
|
return result
|
|
if meta is not None:
|
|
break
|
|
# Persist interrupted metadata outside the write transaction.
|
|
sd.mkdir(parents=True, exist_ok=True)
|
|
meta = dict(stage=name, attempt=attempt, rollout=int(self.rd.name), state='FAIL',
|
|
exit_code=result.get('exit_code'), error_class=result.get('error_class'),
|
|
started_at=row['started_at'], finished_at=result['finished_at'],
|
|
duration_s=max(0, result['finished_at'] - row['started_at']),
|
|
measurements={}, interrupted=True, **identity)
|
|
lifecycle.atomic_json(meta_path, meta)
|
|
(sd / 'exit_code').write_text(str(meta['exit_code']))
|
|
except sqlite3.OperationalError as exc:
|
|
if not stagerun.db.is_busy(exc):
|
|
raise
|
|
if time.monotonic() >= deadline:
|
|
pending = dict(result, db_registered=False)
|
|
if write_status:
|
|
self._write(self.rd / 'stages' / '_queue' / hid / 'status.json', pending)
|
|
return pending
|
|
time.sleep(min(.05, max(0, deadline - time.monotonic())))
|
|
latest = sd.parent / 'meta.json'
|
|
if not latest.exists() or json.loads(latest.read_text()).get('attempt', -1) <= attempt:
|
|
if not latest.exists() or latest.read_bytes() != meta_path.read_bytes():
|
|
lifecycle.atomic_json(latest, meta)
|
|
result = dict(result, stdout=str(sd / 'stdout.log'), stderr=str(sd / 'stderr.log'),
|
|
meta=str(meta_path), cleanup_state='VERIFIED', db_registered=True)
|
|
if write_status:
|
|
self._write(self.rd / 'stages' / '_queue' / hid / 'status.json', result)
|
|
return result
|
|
|
|
def reconcile_completed(self, *, deadline=None):
|
|
"""Register protected completions only; never accept or dispatch a ticket."""
|
|
for hid, ledger in self.execution.snapshot().get('operations', {}).items():
|
|
queue = self.rd / 'stages' / '_queue' / hid
|
|
status_file = queue / 'status.json'
|
|
if not status_file.exists():
|
|
continue
|
|
result = json.loads(status_file.read_text())
|
|
if result.get('state') != 'DONE' or result.get('error_class') == 'SIDE_EFFECT_UNKNOWN':
|
|
continue
|
|
operation = queue / 'operation.json'
|
|
previous = json.loads(operation.read_text())
|
|
if result.get('db_registered') is not False and previous.get('state') == 'DONE' and ledger == previous:
|
|
continue
|
|
with (queue / 'operation.lock').open('a') as lock:
|
|
try:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
return False
|
|
previous = json.loads(operation.read_text())
|
|
name = json.loads((queue / 'request.json').read_text())['stage']
|
|
if 'meta_sha256' not in result:
|
|
self._validate_recovered_receipt(hid, name, previous, result)
|
|
result = self._reconcile_stage(hid, name, previous, result, deadline=deadline)
|
|
if result.get('db_registered') is False:
|
|
return False
|
|
self._complete_operation(hid, operation, previous)
|
|
return True
|
|
|
|
def _complete_operation(self, hid, operation, previous):
|
|
# Preserve a committed receipt byte-for-byte, including its timestamp.
|
|
# The operation file and execution ledger have separate atomic commits;
|
|
# either may already be complete when recovery reaches this boundary.
|
|
if previous.get('state') != 'DONE':
|
|
previous.update(state='DONE', receipt_id=hid, finished_at=time.time())
|
|
if self.execution:
|
|
with self.execution.locked():
|
|
state = self.execution.snapshot()
|
|
self.execution.assert_owner(state)
|
|
if state['operations'].get(hid) == previous:
|
|
return
|
|
lifecycle.atomic_json(operation, previous)
|
|
state['operations'][hid] = previous
|
|
self.execution.phase_done_locked(state, hid)
|
|
self.execution._save(state, 'OPERATION_RECEIPT', operation_id=hid, receipt_id=hid)
|
|
else:
|
|
lifecycle.atomic_json(operation, previous)
|
|
|
|
def _validate_recovered_receipt(self, hid, name, previous, result, *, prerequisite=False):
|
|
"""A stage status proves its result; kernel receipts prove its cleanup."""
|
|
if previous.get('operation_id') != hid:
|
|
raise ValueError('receipt operation mismatch')
|
|
if not self.execution:
|
|
return
|
|
state = self.execution.snapshot()
|
|
self.execution.assert_owner(state)
|
|
identity = self._receipt_identity(previous)
|
|
ledger = state['operations'].get(hid, {})
|
|
if (identity != self._receipt_identity(result) or identity != self._receipt_identity(ledger)
|
|
or identity['execution_id'] != state['execution_id']
|
|
or (previous.get('state') != 'DONE' and identity['epoch'] != state['epoch'])):
|
|
raise ValueError('receipt identity mismatch')
|
|
scopes = [scope for scope in state.get('process_scopes', {}).values()
|
|
if scope['binding'].get('operation_id') == hid]
|
|
executor = previous.get('executor')
|
|
if not executor:
|
|
if scopes or not ((name == 'noop' and self.test_noop) or
|
|
result.get('error_class') == 'invalid_request'):
|
|
raise ValueError('receipt has no dispatch proof')
|
|
return
|
|
leader_verified = False
|
|
# An inline prerequisite can finish before this executor's later work is cancelled.
|
|
completed_prerequisite = (prerequisite and name == '04-calibrate'
|
|
and 'meta_sha256' in result and bool(result.get('outputs')))
|
|
for scope in scopes:
|
|
binding = scope['binding']
|
|
receipt = lifecycle.scope_receipt(scope)
|
|
# The host rebooted under it: no reaper lived to write a receipt, and nothing of that boot runs. Enough
|
|
# for a failed result or an inline 04 whose meta proves it (checked below), never for another success
|
|
# (10-21 03:16 reboot: 7 rollouts with a 05-compile in flight were quarantined OPERATION_STATE_UNKNOWN
|
|
# and held their slots).
|
|
rebooted = (receipt is None and lifecycle.earlier_boot(binding['supervisor'])
|
|
and (result.get('result') not in ('OK', 'SKIPPED') or completed_prerequisite))
|
|
if (binding.get('execution_id') != identity['execution_id'] or binding.get('epoch') != identity['epoch']
|
|
or not (receipt or rebooted)
|
|
or not lifecycle.scope_cleaned(scope, state.get('process_scopes', {}), state.get('owner'))):
|
|
raise ValueError('scope cleanup unconfirmed')
|
|
if lifecycle.process_key(binding['leader']) == lifecycle.process_key(executor):
|
|
if result.get('result') in ('OK', 'SKIPPED') and (((receipt or {}).get('exit_code', 0) != 0 and not completed_prerequisite) or
|
|
lifecycle.process_key(result.get('process', {})) != lifecycle.process_key(executor)):
|
|
raise ValueError('successful receipt has no matching executor')
|
|
leader_verified = True
|
|
if not leader_verified:
|
|
raise ValueError('executor scope missing')
|
|
if 'meta_sha256' in result:
|
|
from .stages.base import recipe_output
|
|
attempt = result.get('attempt')
|
|
if type(attempt) is not int or attempt < 0:
|
|
raise ValueError('receipt attempt mismatch')
|
|
meta_path = self.rd / 'stages' / name / f'a{attempt:02d}' / 'meta.json'
|
|
raw = meta_path.read_bytes()
|
|
meta = json.loads(raw)
|
|
if (result.get('meta') != str(meta_path) or hashlib.sha256(raw).hexdigest() != result['meta_sha256']
|
|
or meta.get('stage') != name or meta.get('attempt') != attempt
|
|
or meta.get('rollout') != int(self.rd.name)
|
|
or self._receipt_identity(meta) != identity
|
|
or (meta.get('state'), meta.get('exit_code'), meta.get('error_class')) !=
|
|
(result.get('result'), result.get('exit_code'), result.get('error_class'))):
|
|
raise ValueError('completed attempt metadata mismatch')
|
|
outputs = result.get('outputs')
|
|
if not isinstance(outputs, dict) or (meta.get('recipe_outputs') and outputs != meta['recipe_outputs']):
|
|
raise ValueError('completed attempt output proof missing')
|
|
if name == '05-compile' and result.get('result') == 'OK' and not outputs:
|
|
raise ValueError('completed compile has no artifact proof')
|
|
for descriptor in outputs.values():
|
|
if recipe_output(Path(descriptor['path'])) != descriptor:
|
|
raise ValueError('completed artifact changed')
|
|
if (name in ('07-board', '08-eval') or (name == 'terminal' and json.loads((self.rd / 'stages' / '_queue' / hid / 'request.json').read_text()).get('target') == 'board')) and result.get('result') != 'SKIPPED':
|
|
receipts = list((self.rd / 'stages' / '_queue' / hid / 'board-receipts').glob('*.json'))
|
|
if not receipts and not self._board_never_admitted(hid):
|
|
raise ValueError('board cleanup receipt missing')
|
|
for path in receipts:
|
|
receipt = json.loads(path.read_text())
|
|
if (receipt.get('operation_id') != path.stem or receipt.get('guard_version') != 4
|
|
or receipt.get('cleanup_state') != 'VERIFIED' or receipt.get('all_children_reaped') is not True
|
|
or receipt.get('proof') not in ('waitpid:ECHILD', 'not_started')):
|
|
raise ValueError('board cleanup unconfirmed')
|
|
|
|
def _prepare_request(self, queue, req):
|
|
if req.get('stage') == 'terminal':
|
|
from .terminal import prepare
|
|
return prepare(self.rd, queue, req)
|
|
config = None
|
|
if req.get('config'):
|
|
config = queue / 'config.yaml'
|
|
config.write_bytes(snapshot_config(self.rd / 'workspace', req['config']))
|
|
recipe = None
|
|
if req.get('recipe') and req.get('submission'):
|
|
raise ValueError('recipe and submission are mutually exclusive')
|
|
if req.get('recipe'):
|
|
from . import recipes
|
|
job = json.loads((self.rd.parent.parent / 'job.json').read_text())
|
|
recipes.snapshot(self.rd / 'workspace', req['recipe'], queue / 'snapshot', job)
|
|
recipe = queue / 'snapshot' / 'recipe.json'
|
|
elif req.get('submission'):
|
|
handle = req['submission']
|
|
if not isinstance(handle, str) or not re.fullmatch(r'[a-zA-Z0-9_-]+', handle):
|
|
raise ValueError('submission must be a handle from this rollout')
|
|
# Only protected queue records are authoritative, never client status.
|
|
source = self.rd / 'stages' / '_queue' / handle
|
|
source_fd = os.open(source, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
try:
|
|
source_request = json.loads(read_regular(source_fd, 'request.json'))
|
|
finally:
|
|
os.close(source_fd)
|
|
recipe = Path(source_request['recipe']).resolve()
|
|
if not recipe.is_relative_to(self.rd / 'stages' / '_queue'):
|
|
raise ValueError('submission outside protected rollout queue')
|
|
return {'stage': req['stage'], 'config': str(config) if config else None,
|
|
'recipe': str(recipe) if recipe else None, 'received_at': time.time()}
|
|
|
|
@staticmethod
|
|
def _failed(reason, exit_code=2):
|
|
return {'state': 'DONE', 'result': 'FAIL', 'exit_code': exit_code,
|
|
'error_class': reason, 'duration_s': 0, 'stdout': '', 'stderr': '', 'meta': '',
|
|
'finished_at': time.time()}
|