data-gen-bpu/harness/stageworker.py
bench f74fe00aaf hbm/bc 模拟器对比改为 agent 按需开启,只作诊断
阶段配置新增 verification_parameters.hbm_bc_samples(0–16,默认 0)。agent 用
bench stage run hbm --config <yaml> 指定 N 张校准样本,用 x86 模拟器对比 HBM 与 BC,
结果写在 G1.hbm_bc_cos_min,不改 verifier_ok:开了不会被扣分,真正把关的仍是 08-eval 板端实测。
编译配置只取三段,所以同一个 yaml 拿去编译也不受影响。适用任何有 .bc 的后端;跑失败记 None。
s06_hbm 随阶段打包,部署后马上生效(默认不跑);配置校验在 stageworker、说明在 SYSTEM.md,
都要下次重启时一起上线,否则按说明提交会被旧校验拒绝。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 12:48:17 +08:00

617 lines
35 KiB
Python

"""Per-rollout outer worker. Only enumerates this rollout's handles directory.
Untrusted tickets are copied into a protected queue. Never execute or write
through a client-supplied path; status publication uses a pinned directory fd.
"""
import json
import hashlib
import fcntl
import sys
import os
import re
import stat
import sqlite3
import threading
import time
import yaml
from pathlib import Path
from . import stagerun, execution as lifecycle, watchdog
from .stages import AGENT_RUNNABLE
def read_regular(parent_fd, name):
fd = os.open(name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=parent_fd)
try:
if not stat.S_ISREG(os.fstat(fd).st_mode) or os.fstat(fd).st_size > 1024 * 1024:
raise ValueError('ticket/config must be a regular file <= 1 MiB')
return os.read(fd, 1024 * 1024 + 1)
finally:
os.close(fd)
def config_bytes(ws, value):
p = Path(value)
rel = p.relative_to(ws)
if not rel.parts or '..' in rel.parts:
raise ValueError('config outside workspace')
fd = os.open(ws, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
for part in rel.parts[:-1]:
nxt = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
os.close(fd); fd = nxt
return read_regular(fd, rel.name)
finally:
os.close(fd)
def snapshot_config(ws, value):
"""Keep path-bearing options from smuggling outer-worker file access."""
cfg = yaml.safe_load(config_bytes(ws, value)) or {}
allowed = {
'input_parameters': {'input_type_rt', 'input_type_train', 'input_layout_train', 'input_layout_rt',
'input_name', 'input_shape', 'mean_value', 'scale_value', 'std_value'},
'calibration_parameters': {'calibration_type', 'max_percentile', 'per_channel',
'run_on_cpu', 'run_on_bpu', 'optimization', 'quant_config'},
'compiler_parameters': {'compile_mode', 'optimize_level', 'jobs', 'debug'},
'verification_parameters': {'hbm_bc_samples'},
}
if not isinstance(cfg, dict):
raise ValueError('config must be a mapping')
for section, values in cfg.items():
if section == 'model_parameters':
continue # Existing backend ignores/pins this section.
if section not in allowed or not isinstance(values, dict) or set(values)-allowed[section]:
raise ValueError('unsupported config keys: ' + str(section))
n = (cfg.get('verification_parameters') or {}).get('hbm_bc_samples', 0)
if type(n) is not int or not 0 <= n <= 16:
raise ValueError('verification_parameters.hbm_bc_samples must be an integer in [0, 16]')
cal = cfg.get('calibration_parameters', {})
quant = cal.get('quant_config')
if isinstance(quant, str):
p = Path(quant)
if not p.is_absolute():
p = ws / p
cal['quant_config'] = json.loads(config_bytes(ws, str(p)))
if cal.get('quant_config') is not None and not isinstance(cal['quant_config'], dict):
raise ValueError('quant_config must be an inline mapping or workspace JSON')
return yaml.safe_dump(cfg, sort_keys=False).encode()
def publish(fd, status):
# Exclusive temporary creation + rename replaces a malicious status symlink.
import uuid
name = '.status-' + uuid.uuid4().hex
f = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=fd)
with os.fdopen(f, 'w') as out:
json.dump(status, out, ensure_ascii=False)
os.rename(name, 'status.json', src_dir_fd=fd, dst_dir_fd=fd)
class Worker:
def __init__(self, rd, *, test_noop=False, execution=None):
self.rd = Path(rd).resolve()
self.test_noop = test_noop and 'guard-smoke' in self.rd.parts
self.execution = execution
if self.execution is None and (self.rd / 'execution.json').exists():
self.execution = lifecycle.Execution(self.rd).attach()
self.stop_event = threading.Event()
self.thread = threading.Thread(target=self.run, name='stage-ticket-worker', daemon=True)
self.seen = set()
def __enter__(self):
self.thread.start(); return self
def __exit__(self, *exc):
self.stop_event.set() # Drain stops new intake; explicit cancel is separate.
budget = self.execution.policy()['cleanup_deadline_s'] if self.execution else 60
if self.execution:
if exc[0] is not None:
self.execution.cancel('WORKER_CONTEXT_ERROR')
# The existing stage watchdog retains its phase and execution clocks.
# Cleanup time only bounds exit confirmation after a stop, not work.
while self.thread.is_alive() and not self.execution.stop_reason():
self.thread.join(timeout=.5)
else:
self.thread.join(timeout=budget)
if self.thread.is_alive():
self.thread.join(timeout=budget)
if self.thread.is_alive():
if self.execution:
self.execution.quarantine('WORKER_JOIN_UNCONFIRMED')
raise RuntimeError('worker did not stop within cleanup budget')
if self.execution:
# The scanner may have stopped with complete work waiting on SQLite.
# Spend one shared cleanup budget on registration, never dispatch.
if not self.reconcile_completed(deadline=time.monotonic() + budget):
self.execution.quarantine('INFRA_STAGE_REGISTRATION_PENDING')
def run(self):
while not self.stop_event.is_set():
if self.execution and self.execution.stop_reason():
return
self.scan()
self.stop_event.wait(0.5)
def scan(self):
for hd in sorted((self.rd / 'handles').iterdir()):
if self.stop_event.is_set():
return
if hd.name in self.seen or not re.fullmatch(r'[a-zA-Z0-9_-]+', hd.name):
continue
try:
fd = os.open(hd, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
except OSError:
continue
raw_request = b""
try:
try:
raw_request = read_regular(fd, 'request.json')
req = json.loads(raw_request)
except FileNotFoundError:
continue
self.seen.add(hd.name)
self.execute(hd.name, fd, req)
except Exception as e:
self.seen.add(hd.name)
queue = self.rd / 'stages' / '_queue' / hd.name
queue.mkdir(parents=True, exist_ok=True)
with (queue / 'runner.log').open('a') as log:
log.write(f'invalid_request: {type(e).__name__}\n')
reason = str(e) if isinstance(e, lifecycle.ExecutionStopped) else 'invalid_request'
detail = str(e)[:1000] if isinstance(e, ValueError) else type(e).__name__
if (queue / 'operation.json').exists():
# A malformed persisted state is not a new invalid ticket.
reason = 'OPERATION_STATE_UNKNOWN'
if self.execution:
self.execution.quarantine(reason)
self._publish(fd, dict(self._failed(reason), error_detail=detail))
finally:
os.close(fd)
def _publish(self, fd, value):
value = {key: item for key, item in value.items() if key != 'prerequisites'}
if value.get('db_registered') is False:
value = {'state': 'RUNNING', 'registration': 'PENDING', 'attempt': value.get('attempt')}
# A terminal observation must not prompt another model request after
# cleanup has fenced this execution. Its owner handles the stop reason.
if value.get('state') == 'DONE' and self.execution and self.execution.stop_reason():
return
with self.execution.mutation() if self.execution else stagerun.nullcontext():
publish(fd, value)
def _write(self, path, value):
with self.execution.mutation() if self.execution else stagerun.nullcontext():
if path.name == 'status.json':
operation = json.loads((path.parent / 'operation.json').read_text())
value = dict(value, **self._receipt_identity(operation))
lifecycle.atomic_json(path, value)
def _board_never_admitted(self, hid):
"""phase_running fires only while the board lock is held, so a phase
that never left QUEUED (e.g. QUEUE_DEADLINE_EXCEEDED) touched no board."""
phase = self.execution.snapshot().get('phases', {}).get(hid) if self.execution else None
return bool(phase) and phase.get('execution_started_monotonic') is None
@staticmethod
def _receipt_identity(operation):
return {key: operation.get(key) for key in
('operation_id', 'payload_sha256', 'execution_id', 'epoch')}
def execute(self, hid, fd, req):
queue = self.rd / 'stages' / '_queue' / hid
queue.mkdir(parents=True, exist_ok=True)
# A replay may observe a current receipt but must never race a second dispatch.
with (queue / 'operation.lock').open('a') as lock:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
status = queue / 'status.json'
value = json.loads(status.read_text()) if status.exists() else {'state': 'QUEUED'}
self._publish(fd, {'state': 'RUNNING'} if value.get('state') == 'DONE' else value)
return
self._execute(hid, fd, req)
def _execute(self, hid, fd, req):
name = req.get('stage')
if name not in AGENT_RUNNABLE and name != 'terminal' and not (name == 'noop' and self.test_noop):
raise ValueError('stage not allowed')
queue = self.rd / 'stages' / '_queue' / hid
digest = hashlib.sha256(json.dumps(req, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
queue.mkdir(parents=True, exist_ok=True)
operation = queue / 'operation.json'
if operation.exists():
previous = json.loads(operation.read_text())
if previous['payload_sha256'] != digest:
self._publish(fd, self._failed('OPERATION_PAYLOAD_CONFLICT'))
return
status = queue / 'status.json'
if status.exists() and json.loads(status.read_text()).get('state') == 'DONE':
result = json.loads(status.read_text())
ledger = self.execution.snapshot().get('operations', {}).get(hid) if self.execution else None
if (result.get('error_class') != 'SIDE_EFFECT_UNKNOWN' and result.get('db_registered') is not False
and previous.get('state') == 'DONE' and ledger == previous):
# Settled before a pause (same skip as reconcile_completed). A later attempt of the
# rollout may overwrite its outputs (one .hbm path per rollout), so re-proving them
# on resume quarantined every multi-compile rollout (10-12/13: 19 rollouts).
self._publish(fd, result)
return
if result.get('error_class') == 'SIDE_EFFECT_UNKNOWN':
if self.execution:
self.execution.quarantine('SIDE_EFFECT_UNKNOWN')
else:
if 'meta_sha256' not in result:
self._validate_recovered_receipt(hid, name, previous, result)
result = self._reconcile_stage(hid, name, previous, result)
if result.get('db_registered') is False:
self.seen.discard(hid)
else:
self._complete_operation(hid, operation, previous)
self._publish(fd, result)
return
if previous.get('executor') and lifecycle.process_status(previous['executor']) == 'ALIVE':
self._publish(fd, json.loads(status.read_text()) if status.exists() else {'state': 'RUNNING'})
return
result = self._failed('SIDE_EFFECT_UNKNOWN')
self._write(queue / 'status.json', result)
self._publish(fd, result)
if self.execution:
self.execution.quarantine('SIDE_EFFECT_UNKNOWN')
return
if self.execution:
with self.execution.admission() as state:
previous = dict(operation_id=hid, payload_sha256=digest, state='PREPARING',
execution_id=state['execution_id'], epoch=state['epoch'],
received_at=time.time(), received_monotonic=time.monotonic())
lifecycle.atomic_json(operation, previous)
state['operations'][hid] = previous
self.execution._save(state, 'OPERATION_PREPARED', operation_id=hid, payload_sha256=digest)
else:
previous = dict(operation_id=hid, payload_sha256=digest, state='PREPARING', received_at=time.time())
lifecycle.atomic_json(operation, previous)
try:
trusted = self._prepare_request(queue, req)
except Exception as exc:
# This attempt has not crossed the dispatch boundary. Persist a
# reproducible rejection, not an ambiguous side-effect outcome.
with (queue / 'runner.log').open('a') as log:
log.write(f'invalid_request: {type(exc).__name__}\nrequest: sha256={digest}\n')
result = dict(self._failed('invalid_request'), error_detail=str(exc)[:1000] if isinstance(exc, ValueError) else type(exc).__name__)
self._write(queue / 'status.json', result)
self._publish(fd, result)
self._complete_operation(hid, operation, previous)
return
self._write(queue / 'request.json', trusted)
if name == 'noop':
result = {'state': 'DONE', 'result': 'OK', 'exit_code': 0, 'error_class': None,
'duration_s': 0, 'stdout': str(queue / 'runner.log'), 'stderr': '', 'meta': '',
'worker_pid': os.getpid()}
self._write(queue / 'status.json', result)
self._publish(fd, result)
else:
if self.execution:
self.execution.begin_phase(hid, name, queued_monotonic=previous['received_monotonic'])
env = dict(os.environ, BENCH_ROOT=str(stagerun.paths.BENCH_ROOT),
PYTHONPATH=str(stagerun.paths.BENCH_ROOT), BENCH_EXECUTION_PHASE=name,
BENCH_BOARD_RECEIPT_DIR=str(queue / 'board-receipts'), BENCH_OPERATION_ID=hid)
py = stagerun.paths.config()['paths'].get('harness_python', sys.executable)
# A process boundary makes Python/native stages cancellable too.
def started(proc):
previous.update(state='DISPATCHED', executor=lifecycle.process_identity(proc.pid))
if self.execution:
with self.execution.locked():
state = self.execution.snapshot()
self.execution.assert_owner(state)
lifecycle.atomic_json(operation, previous)
state['operations'][hid] = previous
self.execution._save(state, 'OPERATION_DISPATCHED', operation_id=hid)
else:
lifecycle.atomic_json(operation, previous)
last_status = None
def publish_progress():
nonlocal last_status
status_file = queue / 'status.json'
if status_file.exists():
status = json.loads(status_file.read_text())
# The child can finish before watchdog/board cleanup does.
if status.get('state') != 'DONE' and status != last_status:
self._publish(fd, status)
last_status = status
return None
policy = self.execution.policy(name) if self.execution else None
with self.execution.scope(name, hid) if self.execution else stagerun.nullcontext():
result_run = watchdog.run([py, '-m', 'harness.stagerun', '--exec-protected', str(queue), str(self.rd)],
cwd=stagerun.paths.BENCH_ROOT, stdout=queue / 'runner.log', stderr=queue / 'runner.stderr.log',
env=env, execution=self.execution, role='stage', on_start=started,
stop_check=publish_progress, poll_s=1,
hard_ceiling_s=(max(.01, trusted['deadline_monotonic'] - time.monotonic()) + policy['cleanup_deadline_s'] if name == 'terminal' else (policy['queue_deadline_s'] + policy['execution_deadline_s']) if policy else 86400))
publish_progress()
status_file = queue / 'status.json'
result = json.loads(status_file.read_text()) if status_file.exists() else {}
if result_run.state != 'OK' or result.get('state') != 'DONE':
progress = result
result = dict(self._failed(result_run.stalled_reason or 'STAGE_EXECUTOR_EXIT', result_run.exit_code),
**{key: progress[key] for key in ('attempt', 'started_at', 'queued_at', 'pool', 'board_name', 'prerequisites')
if key in progress})
result['cleanup_state'] = result_run.cleanup_state
self._write(status_file, result)
if self.execution and (name in ('07-board', '08-eval') or (name == 'terminal' and trusted.get('target') == 'board')) and result.get('result') != 'SKIPPED':
receipts = list((queue / 'board-receipts').glob('*.json'))
if ((not receipts and not self._board_never_admitted(hid))
or any(json.loads(p.read_text()).get('cleanup_state') != 'VERIFIED' for p in receipts)):
self.execution.quarantine('BOARD_CLEANUP_UNCONFIRMED')
if self.execution and self.execution.snapshot().get('cancel_requested') and 'meta_sha256' not in result:
# A completed protected receipt records computation, while the
# execution retains cancellation and suppresses external DONE.
result = dict(result, result='CANCELLED', error_class='CANCEL_REQUESTED')
self._write(status_file, result)
result = json.loads(status_file.read_text())
result = self._reconcile_stage(hid, name, previous, result,
cleanup_state=result_run.cleanup_state)
self._publish(fd, result)
if result.get('db_registered') is False:
self.seen.discard(hid)
return
self._complete_operation(hid, operation, previous)
def _reconcile_stage(self, hid, name, operation, result, *, cleanup_state=None, deadline=None, write_status=True):
"""Replay one admitted completion after cleanup, without repeating work."""
deadline = deadline if deadline is not None else time.monotonic() + 1.0
if result.get('stage', name) != name:
if name != '05-compile' or result['stage'] != '04-calibrate':
raise ValueError('completed stage does not match submitted operation')
name = result['stage'] # Inline calibration failed before compilation.
for prerequisite in result.get('prerequisites', []):
if name != '05-compile' or prerequisite.get('stage') != '04-calibrate':
raise ValueError('unexpected completed prerequisite')
recovered = self._reconcile_stage(hid, '04-calibrate', operation, prerequisite,
deadline=deadline, write_status=False)
if recovered.get('db_registered') is False:
pending = dict(result, db_registered=False)
self._write(self.rd / 'stages' / '_queue' / hid / 'status.json', pending)
return pending
attempt = result.get('attempt')
completion = 'meta_sha256' in result
if completion and (not self.execution or type(attempt) is not int or attempt < 0):
raise ValueError('completed attempt has no managed execution or valid attempt')
if (not write_status or result.get('db_registered') is False) and result.get('result') in ('OK', 'SKIPPED') and not completion:
raise ValueError('completed attempt proof missing')
if (not self.execution or name not in AGENT_RUNNABLE
or type(attempt) is not int or attempt < 0
or (not completion and result.get('result') in ('OK', 'SKIPPED'))
or (not completion and (cleanup_state or result.get('cleanup_state')) != 'VERIFIED')):
return result
identity = self._receipt_identity(operation)
state = self.execution.snapshot()
self.execution.assert_owner(state)
if (identity != self._receipt_identity(state['operations'].get(hid, {}))
or identity != self._receipt_identity(result) or identity['operation_id'] != hid
or identity['execution_id'] != state['execution_id']
or (operation.get('state') != 'DONE' and identity['epoch'] != state['epoch'])):
raise ValueError('interrupted attempt identity mismatch')
sd = self.rd / 'stages' / name / f'a{attempt:02d}'
meta_path = sd / 'meta.json'
job = json.loads((self.rd.parent.parent / 'job.json').read_text())
key = (job['run_id'], job['job_id'], int(self.rd.name), name, attempt)
while True:
try:
# Probe availability before hashing large HBM bytes on every
# scan. Release the transaction before any artifact IO; final
# registration still rechecks owner and the admitted row.
if completion:
with stagerun.db.connect(timeout_s=0) as con:
con.execute('BEGIN IMMEDIATE')
self._validate_recovered_receipt(hid, name, operation, result,
prerequisite=not write_status)
meta = json.loads(meta_path.read_text()) if meta_path.exists() else None
if meta and (meta.get('stage') != name or meta.get('attempt') != attempt):
raise ValueError('interrupted attempt metadata mismatch')
# No execution lock is held while SQLite is busy. A zero busy
# timeout bounds setup, BEGIN, UPDATE and COMMIT by one budget.
with stagerun.db.connect(timeout_s=0) as con:
con.execute('BEGIN IMMEDIATE')
self.execution.assert_owner(self.execution.snapshot())
row = con.execute('SELECT * FROM stages WHERE run_id=? AND job_id=? AND rollout=? AND stage=? AND attempt=?',
key).fetchone()
if row is None:
if completion:
raise ValueError('completed attempt was not admitted')
return result
if meta is not None:
expected = (meta['state'], meta.get('exit_code'), meta.get('error_class'))
if row['state'] in ('RUNNING', 'QUEUED'):
stagerun.db.stage_finish(con, job['run_id'], job['job_id'], name, attempt,
*expected, rollout=int(self.rd.name))
elif tuple(row[k] for k in ('state', 'exit_code', 'error_class')) != expected:
raise ValueError('completed attempt conflicts with terminal database state')
elif row['state'] not in ('RUNNING', 'QUEUED'):
return result
if meta is not None:
break
# Persist interrupted metadata outside the write transaction.
sd.mkdir(parents=True, exist_ok=True)
meta = dict(stage=name, attempt=attempt, rollout=int(self.rd.name), state='FAIL',
exit_code=result.get('exit_code'), error_class=result.get('error_class'),
started_at=row['started_at'], finished_at=result['finished_at'],
duration_s=max(0, result['finished_at'] - row['started_at']),
measurements={}, interrupted=True, **identity)
lifecycle.atomic_json(meta_path, meta)
(sd / 'exit_code').write_text(str(meta['exit_code']))
except sqlite3.OperationalError as exc:
if not stagerun.db.is_busy(exc):
raise
if time.monotonic() >= deadline:
pending = dict(result, db_registered=False)
if write_status:
self._write(self.rd / 'stages' / '_queue' / hid / 'status.json', pending)
return pending
time.sleep(min(.05, max(0, deadline - time.monotonic())))
latest = sd.parent / 'meta.json'
if not latest.exists() or json.loads(latest.read_text()).get('attempt', -1) <= attempt:
if not latest.exists() or latest.read_bytes() != meta_path.read_bytes():
lifecycle.atomic_json(latest, meta)
result = dict(result, stdout=str(sd / 'stdout.log'), stderr=str(sd / 'stderr.log'),
meta=str(meta_path), cleanup_state='VERIFIED', db_registered=True)
if write_status:
self._write(self.rd / 'stages' / '_queue' / hid / 'status.json', result)
return result
def reconcile_completed(self, *, deadline=None):
"""Register protected completions only; never accept or dispatch a ticket."""
for hid, ledger in self.execution.snapshot().get('operations', {}).items():
queue = self.rd / 'stages' / '_queue' / hid
status_file = queue / 'status.json'
if not status_file.exists():
continue
result = json.loads(status_file.read_text())
if result.get('state') != 'DONE' or result.get('error_class') == 'SIDE_EFFECT_UNKNOWN':
continue
operation = queue / 'operation.json'
previous = json.loads(operation.read_text())
if result.get('db_registered') is not False and previous.get('state') == 'DONE' and ledger == previous:
continue
with (queue / 'operation.lock').open('a') as lock:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
return False
previous = json.loads(operation.read_text())
name = json.loads((queue / 'request.json').read_text())['stage']
if 'meta_sha256' not in result:
self._validate_recovered_receipt(hid, name, previous, result)
result = self._reconcile_stage(hid, name, previous, result, deadline=deadline)
if result.get('db_registered') is False:
return False
self._complete_operation(hid, operation, previous)
return True
def _complete_operation(self, hid, operation, previous):
# Preserve a committed receipt byte-for-byte, including its timestamp.
# The operation file and execution ledger have separate atomic commits;
# either may already be complete when recovery reaches this boundary.
if previous.get('state') != 'DONE':
previous.update(state='DONE', receipt_id=hid, finished_at=time.time())
if self.execution:
with self.execution.locked():
state = self.execution.snapshot()
self.execution.assert_owner(state)
if state['operations'].get(hid) == previous:
return
lifecycle.atomic_json(operation, previous)
state['operations'][hid] = previous
self.execution.phase_done_locked(state, hid)
self.execution._save(state, 'OPERATION_RECEIPT', operation_id=hid, receipt_id=hid)
else:
lifecycle.atomic_json(operation, previous)
def _validate_recovered_receipt(self, hid, name, previous, result, *, prerequisite=False):
"""A stage status proves its result; kernel receipts prove its cleanup."""
if previous.get('operation_id') != hid:
raise ValueError('receipt operation mismatch')
if not self.execution:
return
state = self.execution.snapshot()
self.execution.assert_owner(state)
identity = self._receipt_identity(previous)
ledger = state['operations'].get(hid, {})
if (identity != self._receipt_identity(result) or identity != self._receipt_identity(ledger)
or identity['execution_id'] != state['execution_id']
or (previous.get('state') != 'DONE' and identity['epoch'] != state['epoch'])):
raise ValueError('receipt identity mismatch')
scopes = [scope for scope in state.get('process_scopes', {}).values()
if scope['binding'].get('operation_id') == hid]
executor = previous.get('executor')
if not executor:
if scopes or not ((name == 'noop' and self.test_noop) or
result.get('error_class') == 'invalid_request'):
raise ValueError('receipt has no dispatch proof')
return
leader_verified = False
# An inline prerequisite can finish before this executor's later work is cancelled.
completed_prerequisite = (prerequisite and name == '04-calibrate'
and 'meta_sha256' in result and bool(result.get('outputs')))
for scope in scopes:
binding = scope['binding']
receipt = lifecycle.scope_receipt(scope)
if (binding.get('execution_id') != identity['execution_id'] or binding.get('epoch') != identity['epoch']
or not receipt or not lifecycle.scope_cleaned(scope, state.get('process_scopes', {}))):
raise ValueError('scope cleanup unconfirmed')
if lifecycle.process_key(binding['leader']) == lifecycle.process_key(executor):
if result.get('result') in ('OK', 'SKIPPED') and ((receipt['exit_code'] != 0 and not completed_prerequisite) or
lifecycle.process_key(result.get('process', {})) != lifecycle.process_key(executor)):
raise ValueError('successful receipt has no matching executor')
leader_verified = True
if not leader_verified:
raise ValueError('executor scope missing')
if 'meta_sha256' in result:
from .stages.base import recipe_output
attempt = result.get('attempt')
if type(attempt) is not int or attempt < 0:
raise ValueError('receipt attempt mismatch')
meta_path = self.rd / 'stages' / name / f'a{attempt:02d}' / 'meta.json'
raw = meta_path.read_bytes()
meta = json.loads(raw)
if (result.get('meta') != str(meta_path) or hashlib.sha256(raw).hexdigest() != result['meta_sha256']
or meta.get('stage') != name or meta.get('attempt') != attempt
or meta.get('rollout') != int(self.rd.name)
or self._receipt_identity(meta) != identity
or (meta.get('state'), meta.get('exit_code'), meta.get('error_class')) !=
(result.get('result'), result.get('exit_code'), result.get('error_class'))):
raise ValueError('completed attempt metadata mismatch')
outputs = result.get('outputs')
if not isinstance(outputs, dict) or (meta.get('recipe_outputs') and outputs != meta['recipe_outputs']):
raise ValueError('completed attempt output proof missing')
if name == '05-compile' and result.get('result') == 'OK' and not outputs:
raise ValueError('completed compile has no artifact proof')
for descriptor in outputs.values():
if recipe_output(Path(descriptor['path'])) != descriptor:
raise ValueError('completed artifact changed')
if (name in ('07-board', '08-eval') or (name == 'terminal' and json.loads((self.rd / 'stages' / '_queue' / hid / 'request.json').read_text()).get('target') == 'board')) and result.get('result') != 'SKIPPED':
receipts = list((self.rd / 'stages' / '_queue' / hid / 'board-receipts').glob('*.json'))
if not receipts and not self._board_never_admitted(hid):
raise ValueError('board cleanup receipt missing')
for path in receipts:
receipt = json.loads(path.read_text())
if (receipt.get('operation_id') != path.stem or receipt.get('guard_version') != 4
or receipt.get('cleanup_state') != 'VERIFIED' or receipt.get('all_children_reaped') is not True
or receipt.get('proof') not in ('waitpid:ECHILD', 'not_started')):
raise ValueError('board cleanup unconfirmed')
def _prepare_request(self, queue, req):
if req.get('stage') == 'terminal':
from .terminal import prepare
return prepare(self.rd, queue, req)
config = None
if req.get('config'):
config = queue / 'config.yaml'
config.write_bytes(snapshot_config(self.rd / 'workspace', req['config']))
recipe = None
if req.get('recipe') and req.get('submission'):
raise ValueError('recipe and submission are mutually exclusive')
if req.get('recipe'):
from . import recipes
job = json.loads((self.rd.parent.parent / 'job.json').read_text())
recipes.snapshot(self.rd / 'workspace', req['recipe'], queue / 'snapshot', job)
recipe = queue / 'snapshot' / 'recipe.json'
elif req.get('submission'):
handle = req['submission']
if not isinstance(handle, str) or not re.fullmatch(r'[a-zA-Z0-9_-]+', handle):
raise ValueError('submission must be a handle from this rollout')
# Only protected queue records are authoritative, never client status.
source = self.rd / 'stages' / '_queue' / handle
source_fd = os.open(source, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
try:
source_request = json.loads(read_regular(source_fd, 'request.json'))
finally:
os.close(source_fd)
recipe = Path(source_request['recipe']).resolve()
if not recipe.is_relative_to(self.rd / 'stages' / '_queue'):
raise ValueError('submission outside protected rollout queue')
return {'stage': req['stage'], 'config': str(config) if config else None,
'recipe': str(recipe) if recipe else None, 'received_at': time.time()}
@staticmethod
def _failed(reason, exit_code=2):
return {'state': 'DONE', 'result': 'FAIL', 'exit_code': exit_code,
'error_class': reason, 'duration_s': 0, 'stdout': '', 'stderr': '', 'meta': '',
'finished_at': time.time()}