data-gen-bpu/harness/stageworker.py
bench 44f3142cfa Reboot recovery: a reboot-killed scope proves a failed step; speed_ref sets aside an interrupted run
The 10-21 03:16 Mac mini reboot left 7 GLM rollouts (05: densenet201,
efficientnet.b2, rtmo.s; 06: rtmpose.x, rtmo.m, hypersim.base, yolo11n.seg)
quarantined OPERATION_STATE_UNKNOWN while still counted RUNNING: each had a
05-compile in flight, its reaper died with the VM before writing a receipt, and
_validate_recovered_receipt demanded one ('scope cleanup unconfirmed') although
scope_cleaned already accepts an earlier boot. Without a receipt an earlier-boot
scope now proves cleanup for a failed result and for the inline 04 whose meta is
hashed; any other success still needs its receipt.

speed_ref: stages/ref of an interrupted run kept bundle.tar, which salty_dev
creates exclusively, so the 3 in-flight references failed with FileExistsError
every 2 minutes and took all 3 workers. The leftover dir is renamed aside.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-21 07:38:06 +08:00

695 lines
40 KiB
Python

"""Per-rollout outer worker. Only enumerates this rollout's handles directory.
Untrusted tickets are copied into a protected queue. Never execute or write
through a client-supplied path; status publication uses a pinned directory fd.
"""
import json
import hashlib
import fcntl
import sys
import os
import re
import stat
import sqlite3
import threading
import time
import yaml
from pathlib import Path
from . import stagerun, execution as lifecycle, watchdog
from .stages import AGENT_RUNNABLE
def read_regular(parent_fd, name, limit=1024 * 1024):
fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent_fd)
try:
if not stat.S_ISREG(os.fstat(fd).st_mode) or (limit is not None and os.fstat(fd).st_size > limit):
raise ValueError('ticket/config must be a regular file <= 1 MiB')
if limit is not None:
return os.read(fd, limit + 1)
with os.fdopen(os.dup(fd), 'rb') as stream:
return stream.read()
finally:
os.close(fd)
def config_bytes(ws, value, limit=1024 * 1024):
p = Path(value)
rel = p.relative_to(ws)
if not rel.parts or '..' in rel.parts:
raise ValueError('config outside workspace')
fd = os.open(ws, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
for part in rel.parts[:-1]:
nxt = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
os.close(fd); fd = nxt
return read_regular(fd, rel.name, limit)
finally:
os.close(fd)
def workspace_relative(ws, value, field):
"""A path the agent names inside its workspace, kept relative: 04 resolves it in the development
environment, where files made by `bench dev` live (they may not exist on the controller)."""
p = Path(value) if isinstance(value, str) and value else None
if p is not None and p.is_absolute():
try:
p = p.relative_to(ws)
except ValueError:
p = None
if p is None or not p.parts or '..' in p.parts:
raise ValueError(f'{field} must be a path inside the workspace')
return p.as_posix()
def snapshot_config(ws, value):
"""Keep path-bearing options from smuggling outer-worker file access."""
cfg = yaml.safe_load(config_bytes(ws, value)) or {}
allowed = {
'input_parameters': {'input_type_rt', 'input_type_train', 'input_layout_train', 'input_layout_rt',
'input_name', 'input_shape', 'mean_value', 'scale_value', 'std_value'},
'calibration_parameters': {'calibration_type', 'max_percentile', 'per_channel', 'cal_data_dir',
'run_on_cpu', 'run_on_bpu', 'optimization', 'quant_config'},
'compiler_parameters': {'compile_mode', 'optimize_level', 'jobs', 'debug'},
'verification_parameters': {'hbm_bc_samples'},
}
if not isinstance(cfg, dict):
raise ValueError('config must be a mapping')
for section, values in cfg.items():
if section == 'model_parameters':
# The graph is the agent's (04 checks its inputs/outputs against the platform graph);
# march, working_dir and output names stay pinned, so other keys are ignored as before.
if isinstance(values, dict) and 'onnx_model' in values:
values['onnx_model'] = workspace_relative(ws, values['onnx_model'], 'model_parameters.onnx_model')
continue
if section not in allowed or not isinstance(values, dict) or set(values)-allowed[section]:
extra = sorted(set(values) - allowed[section]) if section in allowed and isinstance(values, dict) else []
raise ValueError('unsupported config keys: ' + str(section) + (f' {extra}' if extra else ''))
n = (cfg.get('verification_parameters') or {}).get('hbm_bc_samples', 0)
if type(n) is not int or not 0 <= n <= 16:
raise ValueError('verification_parameters.hbm_bc_samples must be an integer in [0, 16]')
cal = cfg.get('calibration_parameters', {})
if 'cal_data_dir' in cal:
cal['cal_data_dir'] = workspace_relative(ws, cal['cal_data_dir'], 'calibration_parameters.cal_data_dir')
quant = cal.get('quant_config')
if isinstance(quant, str):
p = Path(quant)
if not p.is_absolute():
p = ws / p
cal['quant_config'] = json.loads(config_bytes(ws, str(p)))
if cal.get('quant_config') is not None and not isinstance(cal['quant_config'], dict):
raise ValueError('quant_config must be an inline mapping or workspace JSON')
return yaml.safe_dump(cfg, sort_keys=False).encode()
def publish(fd, status):
# Exclusive temporary creation + rename replaces a malicious status symlink.
import uuid
name = '.status-' + uuid.uuid4().hex
f = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=fd)
with os.fdopen(f, 'w') as out:
json.dump(status, out, ensure_ascii=False)
os.rename(name, 'status.json', src_dir_fd=fd, dst_dir_fd=fd)
class Worker:
def __init__(self, rd, *, test_noop=False, execution=None):
self.rd = Path(rd).resolve()
self.test_noop = test_noop and 'guard-smoke' in self.rd.parts
self.execution = execution
if self.execution is None and (self.rd / 'execution.json').exists():
self.execution = lifecycle.Execution(self.rd).attach()
self.stop_event = threading.Event()
self.thread = threading.Thread(target=self.run, name='stage-ticket-worker', daemon=True)
self.seen = set()
def __enter__(self):
self.thread.start(); return self
def __exit__(self, *exc):
self.stop_event.set() # Drain stops new intake; explicit cancel is separate.
budget = self.execution.policy()['cleanup_deadline_s'] if self.execution else 60
if self.execution:
if exc[0] is not None:
self.execution.cancel('WORKER_CONTEXT_ERROR')
# The existing stage watchdog retains its phase and execution clocks.
# Cleanup time only bounds exit confirmation after a stop, not work.
while self.thread.is_alive() and not self.execution.stop_reason():
self.thread.join(timeout=.5)
else:
self.thread.join(timeout=budget)
if self.thread.is_alive():
self.thread.join(timeout=budget)
if self.thread.is_alive():
if self.execution:
self.execution.quarantine('WORKER_JOIN_UNCONFIRMED')
raise RuntimeError('worker did not stop within cleanup budget')
if self.execution:
# The scanner may have stopped with complete work waiting on SQLite.
# Spend one shared cleanup budget on registration, never dispatch.
if not self.reconcile_completed(deadline=time.monotonic() + budget):
self.execution.quarantine('INFRA_STAGE_REGISTRATION_PENDING')
def run(self):
while not self.stop_event.is_set():
if self.execution and self.execution.stop_reason():
return
self.scan()
self.stop_event.wait(0.5)
def scan(self):
for hd in sorted((self.rd / 'handles').iterdir()):
if self.stop_event.is_set():
return
if hd.name in self.seen or not re.fullmatch(r'[a-zA-Z0-9_-]+', hd.name):
continue
try:
fd = os.open(hd, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
except OSError:
continue
raw_request = b""
try:
try:
raw_request = read_regular(fd, 'request.json')
req = json.loads(raw_request)
except FileNotFoundError:
continue
self.seen.add(hd.name)
self.execute(hd.name, fd, req)
except Exception as e:
self.seen.add(hd.name)
queue = self.rd / 'stages' / '_queue' / hd.name
queue.mkdir(parents=True, exist_ok=True)
with (queue / 'runner.log').open('a') as log:
log.write(f'invalid_request: {type(e).__name__}\n')
reason = str(e) if isinstance(e, lifecycle.ExecutionStopped) else 'invalid_request'
detail = str(e)[:1000] if isinstance(e, ValueError) else type(e).__name__
if (queue / 'operation.json').exists():
# A malformed persisted state is not a new invalid ticket.
reason = 'OPERATION_STATE_UNKNOWN'
if self.execution:
self.execution.quarantine(reason)
self._publish(fd, dict(self._failed(reason), error_detail=detail))
finally:
os.close(fd)
def _publish(self, fd, value):
value = {key: item for key, item in value.items() if key != 'prerequisites'}
if value.get('db_registered') is False:
value = {'state': 'RUNNING', 'registration': 'PENDING', 'attempt': value.get('attempt')}
# A terminal observation must not prompt another model request after
# cleanup has fenced this execution. Its owner handles the stop reason.
if value.get('state') == 'DONE' and self.execution and self.execution.stop_reason():
return
with self.execution.mutation() if self.execution else stagerun.nullcontext():
publish(fd, value)
def _write(self, path, value):
with self.execution.mutation() if self.execution else stagerun.nullcontext():
if path.name == 'status.json':
operation = json.loads((path.parent / 'operation.json').read_text())
value = dict(value, **self._receipt_identity(operation))
lifecycle.atomic_json(path, value)
def _board_never_admitted(self, hid):
"""phase_running fires only while the board lock is held, so a phase
that never left QUEUED (e.g. QUEUE_DEADLINE_EXCEEDED) touched no board."""
phase = self.execution.snapshot().get('phases', {}).get(hid) if self.execution else None
return bool(phase) and phase.get('execution_started_monotonic') is None
@staticmethod
def _receipt_identity(operation):
return {key: operation.get(key) for key in
('operation_id', 'payload_sha256', 'execution_id', 'epoch')}
def execute(self, hid, fd, req):
queue = self.rd / 'stages' / '_queue' / hid
queue.mkdir(parents=True, exist_ok=True)
# A replay may observe a current receipt but must never race a second dispatch.
with (queue / 'operation.lock').open('a') as lock:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
status = queue / 'status.json'
value = json.loads(status.read_text()) if status.exists() else {'state': 'QUEUED'}
self._publish(fd, {'state': 'RUNNING'} if value.get('state') == 'DONE' else value)
return
self._execute(hid, fd, req)
def _execute(self, hid, fd, req):
name = req.get('stage')
if name not in AGENT_RUNNABLE and name != 'terminal' and not (name == 'noop' and self.test_noop):
raise ValueError('stage not allowed')
queue = self.rd / 'stages' / '_queue' / hid
digest = hashlib.sha256(json.dumps(req, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
queue.mkdir(parents=True, exist_ok=True)
operation = queue / 'operation.json'
if operation.exists():
previous = json.loads(operation.read_text())
if previous['payload_sha256'] != digest:
self._publish(fd, self._failed('OPERATION_PAYLOAD_CONFLICT'))
return
status = queue / 'status.json'
if status.exists() and json.loads(status.read_text()).get('state') == 'DONE':
result = json.loads(status.read_text())
ledger = self.execution.snapshot().get('operations', {}).get(hid) if self.execution else None
if (result.get('error_class') != 'SIDE_EFFECT_UNKNOWN' and result.get('db_registered') is not False
and previous.get('state') == 'DONE' and ledger == previous):
# Settled before a pause (same skip as reconcile_completed). A later attempt of the
# rollout may overwrite its outputs (one .hbm path per rollout), so re-proving them
# on resume quarantined every multi-compile rollout (10-12/13: 19 rollouts).
self._publish(fd, result)
return
if result.get('error_class') == 'SIDE_EFFECT_UNKNOWN':
if self.execution:
self.execution.quarantine('SIDE_EFFECT_UNKNOWN')
else:
if 'meta_sha256' not in result:
self._validate_recovered_receipt(hid, name, previous, result)
result = self._reconcile_stage(hid, name, previous, result)
if result.get('db_registered') is False:
self.seen.discard(hid)
else:
self._complete_operation(hid, operation, previous)
self._publish(fd, result)
return
if previous.get('executor') and lifecycle.process_status(previous['executor']) == 'ALIVE':
self._publish(fd, json.loads(status.read_text()) if status.exists() else {'state': 'RUNNING'})
return
result = self._failed('SIDE_EFFECT_UNKNOWN')
self._write(queue / 'status.json', result)
self._publish(fd, result)
if self.execution:
self.execution.quarantine('SIDE_EFFECT_UNKNOWN')
return
if self.execution:
with self.execution.admission() as state:
previous = dict(operation_id=hid, payload_sha256=digest, state='PREPARING',
execution_id=state['execution_id'], epoch=state['epoch'],
received_at=time.time(), received_monotonic=time.monotonic())
lifecycle.atomic_json(operation, previous)
state['operations'][hid] = previous
self.execution._save(state, 'OPERATION_PREPARED', operation_id=hid, payload_sha256=digest)
else:
previous = dict(operation_id=hid, payload_sha256=digest, state='PREPARING', received_at=time.time())
lifecycle.atomic_json(operation, previous)
try:
trusted = self._prepare_request(queue, req)
except Exception as exc:
# This attempt has not crossed the dispatch boundary. Persist a
# reproducible rejection, not an ambiguous side-effect outcome.
with (queue / 'runner.log').open('a') as log:
log.write(f'invalid_request: {type(exc).__name__}\nrequest: sha256={digest}\n')
result = dict(self._failed('invalid_request'), error_detail=str(exc)[:1000] if isinstance(exc, ValueError) else type(exc).__name__)
self._write(queue / 'status.json', result)
self._publish(fd, result)
self._complete_operation(hid, operation, previous)
return
self._write(queue / 'request.json', trusted)
if name == 'noop':
result = {'state': 'DONE', 'result': 'OK', 'exit_code': 0, 'error_class': None,
'duration_s': 0, 'stdout': str(queue / 'runner.log'), 'stderr': '', 'meta': '',
'worker_pid': os.getpid()}
self._write(queue / 'status.json', result)
self._publish(fd, result)
else:
if self.execution:
self.execution.begin_phase(hid, name, queued_monotonic=previous['received_monotonic'])
env = dict(os.environ, BENCH_ROOT=str(stagerun.paths.BENCH_ROOT),
PYTHONPATH=str(stagerun.paths.BENCH_ROOT), BENCH_EXECUTION_PHASE=name,
BENCH_BOARD_RECEIPT_DIR=str(queue / 'board-receipts'), BENCH_OPERATION_ID=hid)
py = stagerun.paths.config()['paths'].get('harness_python', sys.executable)
# A process boundary makes Python/native stages cancellable too.
def started(proc):
previous.update(state='DISPATCHED', executor=lifecycle.process_identity(proc.pid))
if self.execution:
with self.execution.locked():
state = self.execution.snapshot()
self.execution.assert_owner(state)
lifecycle.atomic_json(operation, previous)
state['operations'][hid] = previous
self.execution._save(state, 'OPERATION_DISPATCHED', operation_id=hid)
else:
lifecycle.atomic_json(operation, previous)
last_status = None
def publish_progress():
nonlocal last_status
status_file = queue / 'status.json'
if status_file.exists():
status = json.loads(status_file.read_text())
# The child can finish before watchdog/board cleanup does.
if status.get('state') != 'DONE' and status != last_status:
self._publish(fd, status)
last_status = status
return None
policy = self.execution.policy(name) if self.execution else None
with self.execution.scope(name, hid) if self.execution else stagerun.nullcontext():
result_run = watchdog.run([py, '-m', 'harness.stagerun', '--exec-protected', str(queue), str(self.rd)],
cwd=stagerun.paths.BENCH_ROOT, stdout=queue / 'runner.log', stderr=queue / 'runner.stderr.log',
env=env, execution=self.execution, role='stage', on_start=started,
stop_check=publish_progress, poll_s=1,
hard_ceiling_s=(max(.01, trusted['deadline_monotonic'] - time.monotonic()) + policy['cleanup_deadline_s'] if name == 'terminal' else (policy['queue_deadline_s'] + policy['execution_deadline_s']) if policy else 86400))
publish_progress()
status_file = queue / 'status.json'
result = json.loads(status_file.read_text()) if status_file.exists() else {}
if result.get('state') == 'DONE' and result_run.state != 'OK':
# The child's own verdict (e.g. TERMINAL_DEADLINE_EXCEEDED) beats a bare executor exit: 10-19 staging,
# a board command that never got a board reached the agent as STAGE_EXECUTOR_EXIT with no reason.
result.setdefault('cleanup_state', result_run.cleanup_state)
self._write(status_file, result)
elif result.get('state') != 'DONE':
progress = result
result = dict(self._failed(result_run.stalled_reason or 'STAGE_EXECUTOR_EXIT', result_run.exit_code),
**{key: progress[key] for key in ('attempt', 'started_at', 'queued_at', 'pool', 'board_name', 'prerequisites')
if key in progress})
result['cleanup_state'] = result_run.cleanup_state
self._write(status_file, result)
self._check_board_cleanup(hid, name, trusted, result)
if self.execution and self.execution.snapshot().get('cancel_requested') and 'meta_sha256' not in result:
# A completed protected receipt records computation, while the
# execution retains cancellation and suppresses external DONE.
result = dict(result, result='CANCELLED', error_class='CANCEL_REQUESTED')
self._write(status_file, result)
result = json.loads(status_file.read_text())
result = self._reconcile_stage(hid, name, previous, result,
cleanup_state=result_run.cleanup_state)
self._publish(fd, result)
if result.get('db_registered') is False:
self.seen.discard(hid)
return
self._complete_operation(hid, operation, previous)
def _check_board_cleanup(self, hid, name, trusted, result):
if self.execution and (name in ('07-board', '08-eval') or (name == 'terminal' and trusted.get('target') == 'board')) and result.get('result') != 'SKIPPED':
queue = self.rd / 'stages' / '_queue' / hid
receipts = list((queue / 'board-receipts').glob('*.json'))
if ((not receipts and not self._board_never_admitted(hid))
or any(json.loads(p.read_text()).get('cleanup_state') != 'VERIFIED' for p in receipts)):
self.execution.quarantine('BOARD_CLEANUP_UNCONFIRMED')
def settle_interrupted(self):
"""The collector died (crash, kill, planned handover, reboot): an admitted operation whose executor is gone
ended with it. Record it exactly like an executor exit under a live collector (a system failure, so a compile
is free) and tell the agent's handle, so the rollout can resume. Board work without a clean board receipt
still quarantines. Caller proved every process of the execution gone; nothing here dispatches."""
for hid, ledger in self.execution.snapshot().get('operations', {}).items():
queue = self.rd / 'stages' / '_queue' / hid
status_file = queue / 'status.json'
progress = json.loads(status_file.read_text()) if status_file.exists() else {}
if ledger.get('state') == 'DONE' or progress.get('state') == 'DONE':
continue # settled, or a completed receipt for reconcile_completed
if ledger.get('executor') and lifecycle.process_status(ledger['executor']) != 'DEAD':
return False
request = queue / 'request.json' if (queue / 'request.json').exists() else self.rd / 'handles' / hid / 'request.json'
trusted = json.loads(request.read_text())
previous = json.loads((queue / 'operation.json').read_text())
result = dict(self._failed('STAGE_EXECUTOR_EXIT'),
note='平台的调度程序重启,打断了这一步;不计编译次数,需要的话重新提交',
cleanup_state='VERIFIED',
**{key: progress[key] for key in ('attempt', 'started_at', 'queued_at', 'pool', 'board_name', 'prerequisites')
if key in progress})
self._write(status_file, result)
self._check_board_cleanup(hid, trusted['stage'], trusted, result)
result = self._reconcile_stage(hid, trusted['stage'], previous, json.loads(status_file.read_text()),
cleanup_state='VERIFIED')
if result.get('db_registered') is False:
return False
# Not _publish: it withholds DONE from a stopped execution, and this one's owner is dead until it resumes;
# the resumed agent must find its handle finished, not wait on it forever.
handle = os.open(self.rd / 'handles' / hid, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
with self.execution.mutation():
publish(handle, {key: item for key, item in result.items() if key != 'prerequisites'})
finally:
os.close(handle)
self._complete_operation(hid, queue / 'operation.json', previous)
return True
def _reconcile_stage(self, hid, name, operation, result, *, cleanup_state=None, deadline=None, write_status=True):
"""Replay one admitted completion after cleanup, without repeating work."""
deadline = deadline if deadline is not None else time.monotonic() + 1.0
if result.get('stage', name) != name:
if name != '05-compile' or result['stage'] != '04-calibrate':
raise ValueError('completed stage does not match submitted operation')
name = result['stage'] # Inline calibration failed before compilation.
for prerequisite in result.get('prerequisites', []):
if name != '05-compile' or prerequisite.get('stage') != '04-calibrate':
raise ValueError('unexpected completed prerequisite')
recovered = self._reconcile_stage(hid, '04-calibrate', operation, prerequisite,
deadline=deadline, write_status=False)
if recovered.get('db_registered') is False:
pending = dict(result, db_registered=False)
self._write(self.rd / 'stages' / '_queue' / hid / 'status.json', pending)
return pending
attempt = result.get('attempt')
completion = 'meta_sha256' in result
if completion and (not self.execution or type(attempt) is not int or attempt < 0):
raise ValueError('completed attempt has no managed execution or valid attempt')
if (not write_status or result.get('db_registered') is False) and result.get('result') in ('OK', 'SKIPPED') and not completion:
raise ValueError('completed attempt proof missing')
if (not self.execution or name not in AGENT_RUNNABLE
or type(attempt) is not int or attempt < 0
or (not completion and result.get('result') in ('OK', 'SKIPPED'))
or (not completion and (cleanup_state or result.get('cleanup_state')) != 'VERIFIED')):
return result
identity = self._receipt_identity(operation)
state = self.execution.snapshot()
self.execution.assert_owner(state)
if (identity != self._receipt_identity(state['operations'].get(hid, {}))
or identity != self._receipt_identity(result) or identity['operation_id'] != hid
or identity['execution_id'] != state['execution_id']
or (operation.get('state') != 'DONE' and identity['epoch'] != state['epoch'])):
raise ValueError('interrupted attempt identity mismatch')
sd = self.rd / 'stages' / name / f'a{attempt:02d}'
meta_path = sd / 'meta.json'
job = json.loads((self.rd.parent.parent / 'job.json').read_text())
key = (job['run_id'], job['job_id'], int(self.rd.name), name, attempt)
while True:
try:
# Probe availability before hashing large HBM bytes on every
# scan. Release the transaction before any artifact IO; final
# registration still rechecks owner and the admitted row.
if completion:
with stagerun.db.connect(timeout_s=0) as con:
con.execute('BEGIN IMMEDIATE')
self._validate_recovered_receipt(hid, name, operation, result,
prerequisite=not write_status)
meta = json.loads(meta_path.read_text()) if meta_path.exists() else None
if meta and (meta.get('stage') != name or meta.get('attempt') != attempt):
raise ValueError('interrupted attempt metadata mismatch')
# No execution lock is held while SQLite is busy. A zero busy
# timeout bounds setup, BEGIN, UPDATE and COMMIT by one budget.
with stagerun.db.connect(timeout_s=0) as con:
con.execute('BEGIN IMMEDIATE')
self.execution.assert_owner(self.execution.snapshot())
row = con.execute('SELECT * FROM stages WHERE run_id=? AND job_id=? AND rollout=? AND stage=? AND attempt=?',
key).fetchone()
if row is None:
if completion:
raise ValueError('completed attempt was not admitted')
return result
if meta is not None:
expected = (meta['state'], meta.get('exit_code'), meta.get('error_class'))
if row['state'] in ('RUNNING', 'QUEUED'):
stagerun.db.stage_finish(con, job['run_id'], job['job_id'], name, attempt,
*expected, rollout=int(self.rd.name))
elif tuple(row[k] for k in ('state', 'exit_code', 'error_class')) != expected:
raise ValueError('completed attempt conflicts with terminal database state')
elif row['state'] not in ('RUNNING', 'QUEUED'):
return result
if meta is not None:
break
# Persist interrupted metadata outside the write transaction.
sd.mkdir(parents=True, exist_ok=True)
meta = dict(stage=name, attempt=attempt, rollout=int(self.rd.name), state='FAIL',
exit_code=result.get('exit_code'), error_class=result.get('error_class'),
started_at=row['started_at'], finished_at=result['finished_at'],
duration_s=max(0, result['finished_at'] - row['started_at']),
measurements={}, interrupted=True, **identity)
lifecycle.atomic_json(meta_path, meta)
(sd / 'exit_code').write_text(str(meta['exit_code']))
except sqlite3.OperationalError as exc:
if not stagerun.db.is_busy(exc):
raise
if time.monotonic() >= deadline:
pending = dict(result, db_registered=False)
if write_status:
self._write(self.rd / 'stages' / '_queue' / hid / 'status.json', pending)
return pending
time.sleep(min(.05, max(0, deadline - time.monotonic())))
latest = sd.parent / 'meta.json'
if not latest.exists() or json.loads(latest.read_text()).get('attempt', -1) <= attempt:
if not latest.exists() or latest.read_bytes() != meta_path.read_bytes():
lifecycle.atomic_json(latest, meta)
result = dict(result, stdout=str(sd / 'stdout.log'), stderr=str(sd / 'stderr.log'),
meta=str(meta_path), cleanup_state='VERIFIED', db_registered=True)
if write_status:
self._write(self.rd / 'stages' / '_queue' / hid / 'status.json', result)
return result
def reconcile_completed(self, *, deadline=None):
"""Register protected completions only; never accept or dispatch a ticket."""
for hid, ledger in self.execution.snapshot().get('operations', {}).items():
queue = self.rd / 'stages' / '_queue' / hid
status_file = queue / 'status.json'
if not status_file.exists():
continue
result = json.loads(status_file.read_text())
if result.get('state') != 'DONE' or result.get('error_class') == 'SIDE_EFFECT_UNKNOWN':
continue
operation = queue / 'operation.json'
previous = json.loads(operation.read_text())
if result.get('db_registered') is not False and previous.get('state') == 'DONE' and ledger == previous:
continue
with (queue / 'operation.lock').open('a') as lock:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
return False
previous = json.loads(operation.read_text())
name = json.loads((queue / 'request.json').read_text())['stage']
if 'meta_sha256' not in result:
self._validate_recovered_receipt(hid, name, previous, result)
result = self._reconcile_stage(hid, name, previous, result, deadline=deadline)
if result.get('db_registered') is False:
return False
self._complete_operation(hid, operation, previous)
return True
def _complete_operation(self, hid, operation, previous):
# Preserve a committed receipt byte-for-byte, including its timestamp.
# The operation file and execution ledger have separate atomic commits;
# either may already be complete when recovery reaches this boundary.
if previous.get('state') != 'DONE':
previous.update(state='DONE', receipt_id=hid, finished_at=time.time())
if self.execution:
with self.execution.locked():
state = self.execution.snapshot()
self.execution.assert_owner(state)
if state['operations'].get(hid) == previous:
return
lifecycle.atomic_json(operation, previous)
state['operations'][hid] = previous
self.execution.phase_done_locked(state, hid)
self.execution._save(state, 'OPERATION_RECEIPT', operation_id=hid, receipt_id=hid)
else:
lifecycle.atomic_json(operation, previous)
def _validate_recovered_receipt(self, hid, name, previous, result, *, prerequisite=False):
"""A stage status proves its result; kernel receipts prove its cleanup."""
if previous.get('operation_id') != hid:
raise ValueError('receipt operation mismatch')
if not self.execution:
return
state = self.execution.snapshot()
self.execution.assert_owner(state)
identity = self._receipt_identity(previous)
ledger = state['operations'].get(hid, {})
if (identity != self._receipt_identity(result) or identity != self._receipt_identity(ledger)
or identity['execution_id'] != state['execution_id']
or (previous.get('state') != 'DONE' and identity['epoch'] != state['epoch'])):
raise ValueError('receipt identity mismatch')
scopes = [scope for scope in state.get('process_scopes', {}).values()
if scope['binding'].get('operation_id') == hid]
executor = previous.get('executor')
if not executor:
if scopes or not ((name == 'noop' and self.test_noop) or
result.get('error_class') == 'invalid_request'):
raise ValueError('receipt has no dispatch proof')
return
leader_verified = False
# An inline prerequisite can finish before this executor's later work is cancelled.
completed_prerequisite = (prerequisite and name == '04-calibrate'
and 'meta_sha256' in result and bool(result.get('outputs')))
for scope in scopes:
binding = scope['binding']
receipt = lifecycle.scope_receipt(scope)
# The host rebooted under it: no reaper lived to write a receipt, and nothing of that boot runs. Enough
# for a failed result or an inline 04 whose meta proves it (checked below), never for another success
# (10-21 03:16 reboot: 7 rollouts with a 05-compile in flight were quarantined OPERATION_STATE_UNKNOWN
# and held their slots).
rebooted = (receipt is None and lifecycle.earlier_boot(binding['supervisor'])
and (result.get('result') not in ('OK', 'SKIPPED') or completed_prerequisite))
if (binding.get('execution_id') != identity['execution_id'] or binding.get('epoch') != identity['epoch']
or not (receipt or rebooted)
or not lifecycle.scope_cleaned(scope, state.get('process_scopes', {}), state.get('owner'))):
raise ValueError('scope cleanup unconfirmed')
if lifecycle.process_key(binding['leader']) == lifecycle.process_key(executor):
if result.get('result') in ('OK', 'SKIPPED') and (((receipt or {}).get('exit_code', 0) != 0 and not completed_prerequisite) or
lifecycle.process_key(result.get('process', {})) != lifecycle.process_key(executor)):
raise ValueError('successful receipt has no matching executor')
leader_verified = True
if not leader_verified:
raise ValueError('executor scope missing')
if 'meta_sha256' in result:
from .stages.base import recipe_output
attempt = result.get('attempt')
if type(attempt) is not int or attempt < 0:
raise ValueError('receipt attempt mismatch')
meta_path = self.rd / 'stages' / name / f'a{attempt:02d}' / 'meta.json'
raw = meta_path.read_bytes()
meta = json.loads(raw)
if (result.get('meta') != str(meta_path) or hashlib.sha256(raw).hexdigest() != result['meta_sha256']
or meta.get('stage') != name or meta.get('attempt') != attempt
or meta.get('rollout') != int(self.rd.name)
or self._receipt_identity(meta) != identity
or (meta.get('state'), meta.get('exit_code'), meta.get('error_class')) !=
(result.get('result'), result.get('exit_code'), result.get('error_class'))):
raise ValueError('completed attempt metadata mismatch')
outputs = result.get('outputs')
if not isinstance(outputs, dict) or (meta.get('recipe_outputs') and outputs != meta['recipe_outputs']):
raise ValueError('completed attempt output proof missing')
if name == '05-compile' and result.get('result') == 'OK' and not outputs:
raise ValueError('completed compile has no artifact proof')
for descriptor in outputs.values():
if recipe_output(Path(descriptor['path'])) != descriptor:
raise ValueError('completed artifact changed')
if (name in ('07-board', '08-eval') or (name == 'terminal' and json.loads((self.rd / 'stages' / '_queue' / hid / 'request.json').read_text()).get('target') == 'board')) and result.get('result') != 'SKIPPED':
receipts = list((self.rd / 'stages' / '_queue' / hid / 'board-receipts').glob('*.json'))
if not receipts and not self._board_never_admitted(hid):
raise ValueError('board cleanup receipt missing')
for path in receipts:
receipt = json.loads(path.read_text())
if (receipt.get('operation_id') != path.stem or receipt.get('guard_version') != 4
or receipt.get('cleanup_state') != 'VERIFIED' or receipt.get('all_children_reaped') is not True
or receipt.get('proof') not in ('waitpid:ECHILD', 'not_started')):
raise ValueError('board cleanup unconfirmed')
def _prepare_request(self, queue, req):
if req.get('stage') == 'terminal':
from .terminal import prepare
return prepare(self.rd, queue, req)
config = None
if req.get('config'):
config = queue / 'config.yaml'
config.write_bytes(snapshot_config(self.rd / 'workspace', req['config']))
recipe = None
if req.get('recipe') and req.get('submission'):
raise ValueError('recipe and submission are mutually exclusive')
if req.get('recipe'):
from . import recipes
job = json.loads((self.rd.parent.parent / 'job.json').read_text())
recipes.snapshot(self.rd / 'workspace', req['recipe'], queue / 'snapshot', job)
recipe = queue / 'snapshot' / 'recipe.json'
elif req.get('submission'):
handle = req['submission']
if not isinstance(handle, str) or not re.fullmatch(r'[a-zA-Z0-9_-]+', handle):
raise ValueError('submission must be a handle from this rollout')
# Only protected queue records are authoritative, never client status.
source = self.rd / 'stages' / '_queue' / handle
source_fd = os.open(source, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
source_request = json.loads(read_regular(source_fd, 'request.json'))
finally:
os.close(source_fd)
recipe = Path(source_request['recipe']).resolve()
if not recipe.is_relative_to(self.rd / 'stages' / '_queue'):
raise ValueError('submission outside protected rollout queue')
return {'stage': req['stage'], 'config': str(config) if config else None,
'recipe': str(recipe) if recipe else None, 'received_at': time.time()}
@staticmethod
def _failed(reason, exit_code=2):
return {'state': 'DONE', 'result': 'FAIL', 'exit_code': exit_code,
'error_class': reason, 'duration_s': 0, 'stdout': '', 'stderr': '', 'meta': '',
'finished_at': time.time()}