fix: preserve completed calibration when later compilation is cancelled

This commit is contained in:
ming01.wang 2026-09-29 19:04:50 +08:00
parent 13f1e99aee
commit 10663a800f
2 changed files with 140 additions and 5 deletions

View File

@ -354,7 +354,7 @@ class Worker:
completion = 'meta_sha256' in result
if completion and (not self.execution or type(attempt) is not int or attempt < 0):
raise ValueError('completed attempt has no managed execution or valid attempt')
if result.get('db_registered') is False and result.get('result') in ('OK', 'SKIPPED') and not completion:
if (not write_status or result.get('db_registered') is False) and result.get('result') in ('OK', 'SKIPPED') and not completion:
raise ValueError('completed attempt proof missing')
if (not self.execution or name not in AGENT_RUNNABLE
or type(attempt) is not int or attempt < 0
@ -381,7 +381,8 @@ class Worker:
if completion:
with stagerun.db.connect(timeout_s=0) as con:
con.execute('BEGIN IMMEDIATE')
self._validate_recovered_receipt(hid, name, operation, result)
self._validate_recovered_receipt(hid, name, operation, result,
prerequisite=not write_status)
meta = json.loads(meta_path.read_text()) if meta_path.exists() else None
if meta and (meta.get('stage') != name or meta.get('attempt') != attempt):
raise ValueError('interrupted attempt metadata mismatch')
@ -483,7 +484,7 @@ class Worker:
else:
lifecycle.atomic_json(operation, previous)
def _validate_recovered_receipt(self, hid, name, previous, result):
def _validate_recovered_receipt(self, hid, name, previous, result, *, prerequisite=False):
"""A stage status proves its result; kernel receipts prove its cleanup."""
if previous.get('operation_id') != hid:
raise ValueError('receipt operation mismatch')
@ -506,6 +507,9 @@ class Worker:
raise ValueError('receipt has no dispatch proof')
return
leader_verified = False
# An inline prerequisite can finish before this executor's later work is cancelled.
completed_prerequisite = (prerequisite and name == '04-calibrate'
and 'meta_sha256' in result and bool(result.get('outputs')))
for scope in scopes:
binding = scope['binding']
receipt = lifecycle.scope_receipt(scope)
@ -513,7 +517,7 @@ class Worker:
or not receipt or receipt.get('cleanup_state') != 'VERIFIED'):
raise ValueError('scope cleanup unconfirmed')
if lifecycle.process_key(binding['leader']) == lifecycle.process_key(executor):
if result.get('result') in ('OK', 'SKIPPED') and (receipt['exit_code'] != 0 or
if result.get('result') in ('OK', 'SKIPPED') and ((receipt['exit_code'] != 0 and not completed_prerequisite) or
lifecycle.process_key(result.get('process', {})) != lifecycle.process_key(executor)):
raise ValueError('successful receipt has no matching executor')
leader_verified = True

View File

@ -20,7 +20,7 @@ ROOT = Path(__file__).resolve().parents[1]
CONNECT = db.connect
READONLY = db.readonly
CHILD = r'''
import sys
import sys, time
from pathlib import Path
from contextlib import nullcontext
from functools import partial
@ -34,6 +34,9 @@ class Once(Stage):
name = '05-compile'
def native(self, ctx, sd):
with (rd / 'effects').open('a') as f: f.write('one\n')
if outcome == 'BLOCK':
(rd / 'compile-started').touch()
while True: time.sleep(.1)
(work_dir(ctx) / 'model.hbm').write_bytes(b'actual-compiled-artifact')
return StageOutcome(outcome, 2 if outcome == 'FAIL' else 0,
'fixture_failure' if outcome == 'FAIL' else None)
@ -529,6 +532,134 @@ stageworker.Worker(rd, execution=owner).scan()
self.assertEqual((self.rd / 'cal-effects').read_text(), 'one\n')
self.assertEqual((self.rd / 'effects').read_text(), 'one\n')
def cancel_after_inline_calibration(self, fault=None):
job_path = self.rd.parent.parent / 'job.json'
job = json.loads(job_path.read_text()); job['backend'] = 'ptq'
job_path.write_text(json.dumps(job))
cal_meta = self.rd / 'stages/04-calibrate/a00/meta.json'
cal_output = self.rd / 'stages/04-calibrate/compile.yaml'
def run(command, **kw):
self.dispatches += 1
original_check = kw['stop_check']
def cancel_when_compiling():
if (self.rd / 'compile-started').exists() and not self.owner.snapshot().get('cancel_requested'):
self.cal_proof = json.loads((self.queue / 'status.json').read_text())['prerequisites'][0]
self.cal_before = cal_meta.read_bytes(), cal_output.read_bytes()
self.owner.cancel('TEST_WINDOW_ENDED')
return original_check()
kw.update(stop_check=cancel_when_compiling, poll_s=.01, hard_ceiling_s=5)
result = self.run([sys.executable, '-c', CHILD, str(self.queue), str(self.rd),
str(self.database), 'BLOCK'], **kw)
self.assertEqual((result.exit_code, result.cleanup_state), (-15, 'VERIFIED'))
self.assertEqual(execution.owned_processes(self.owner.snapshot()), [])
status = json.loads((self.queue / 'status.json').read_text())
proof = status['prerequisites'][0]
if fault == 'identity': proof['execution_id'] = 'other'
elif fault == 'process': proof['process']['proc_start_ticks'] += 1
elif fault == 'proof': proof.pop('meta_sha256')
elif fault == 'proof_without_pending':
proof.pop('meta_sha256'); proof['db_registered'] = True
elif fault == 'outputs': proof['outputs'] = {}
elif fault == 'meta': cal_meta.write_text('{}')
elif fault == 'artifact': cal_output.write_text('changed')
elif fault == 'scope':
with self.owner.mutation() as state:
state['process_scopes'] = {}
self.owner._save(state, 'TEST_REMOVE_SCOPE')
elif fault == 'cleanup':
scope = next(iter(self.owner.snapshot()['process_scopes'].values()))
path = Path(scope['receipt_path']); receipt = json.loads(path.read_text())
receipt['cleanup_state'] = 'UNKNOWN'; path.write_text(json.dumps(receipt))
if fault: (self.queue / 'status.json').write_text(json.dumps(status))
return result
with patch.object(stageworker.watchdog, 'run', side_effect=run):
self.worker.scan()
def test_cancel_after_inline_calibration_preserves_proof_and_settles_once(self):
from harness import collection_runtime, paths, pipeline, recovery
self.cancel_after_inline_calibration()
status = json.loads((self.queue / 'status.json').read_text())
self.assertEqual((status['result'], status['cleanup_state']), ('CANCELLED', 'VERIFIED'))
self.assertEqual(status['prerequisites'][0], self.cal_proof)
with self.connect(self.database) as con:
rows = [tuple(row) for row in con.execute('SELECT stage,state,exit_code,error_class FROM stages ORDER BY stage')]
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='CANCELLED')
self.assertEqual(rows, [('04-calibrate', 'OK', 0, None), ('05-compile', 'FAIL', -15, 'CANCEL_REQUESTED')])
self.assertEqual(self.owner.snapshot()['operations']['h1']['state'], 'DONE')
self.assertEqual(self.owner.snapshot()['phases']['h1']['state'], 'DONE')
operation = json.loads((self.queue / 'operation.json').read_text())
with self.assertRaisesRegex(ValueError, 'successful receipt has no matching executor'):
self.worker._validate_recovered_receipt('h1', '04-calibrate', operation, self.cal_proof)
self.owner.cleanup()
state = self.owner.finish(-15, protocol_complete=False)
self.assertEqual((state['execution_outcome'], state['cleanup_state']), ('CANCELLED', 'VERIFIED'))
before = (self.queue / 'operation.json').read_bytes(), (self.queue / 'status.json').read_bytes()
with patch.object(db, 'readonly', lambda: READONLY(self.database)), \
patch.object(paths, 'RUNS', self.root / 'runs'), \
patch.object(paths, 'STATE', self.root / 'lease-state'), \
patch.object(stageworker.watchdog, 'run', side_effect=AssertionError('recovery dispatched work')):
for _ in range(2):
self.assertTrue(recovery.reconcile_run('r', attempts=[('j', 0)])['ready'])
paths.STATE.mkdir(parents=True, exist_ok=True)
pipeline.collect_drain_file('r').touch()
messages = []
with patch.object(collection_runtime, 'config', return_value={}), \
patch.object(paths, 'config', return_value={'pools': {'agent': {'workers': 1}}}):
self.assertEqual(pipeline.collect_rollouts('r', log=messages.append), {})
self.assertTrue(any('drained' in message for message in messages))
self.assertEqual(((self.queue / 'operation.json').read_bytes(), (self.queue / 'status.json').read_bytes()), before)
self.assertEqual(((self.rd / 'stages/04-calibrate/a00/meta.json').read_bytes(),
(self.rd / 'stages/04-calibrate/compile.yaml').read_bytes()), self.cal_before)
self.assertEqual((self.rd / 'cal-effects').read_text(), 'one\n')
self.assertEqual((self.rd / 'effects').read_text(), 'one\n')
self.assertEqual(self.dispatches, 1)
def test_legacy_cancel_quarantine_reconciles_without_erasing_history(self):
from harness import paths, recovery
validate = self.worker._validate_recovered_receipt
def legacy_validator(hid, name, operation, result, **kw):
if name == '04-calibrate':
raise ValueError('successful receipt has no matching executor')
return validate(hid, name, operation, result, **kw)
# Reproduce only the old rejection; child cancellation and cleanup receipts are real.
with patch.object(self.worker, '_validate_recovered_receipt', side_effect=legacy_validator):
self.cancel_after_inline_calibration()
state = self.owner.finish(-15, protocol_complete=False)
self.assertEqual((state['execution_outcome'], state['cleanup_state']), ('CANCELLED', 'QUARANTINED'))
self.assertEqual(state['operations']['h1']['state'], 'DISPATCHED')
with self.connect(self.database) as con:
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='CANCELLED')
with patch.object(db, 'readonly', lambda: READONLY(self.database)), \
patch.object(paths, 'RUNS', self.root / 'runs'), \
patch.object(paths, 'STATE', self.root / 'lease-state'), \
patch.object(stageworker.watchdog, 'run', side_effect=AssertionError('recovery dispatched work')):
for _ in range(2):
self.assertTrue(recovery.reconcile_run('r', attempts=[('j', 0)])['ready'])
state = self.owner.snapshot()
self.assertEqual((state['cleanup_state'], state['stop_reason']), ('QUARANTINED', 'OPERATION_STATE_UNKNOWN'))
self.assertEqual(state['operations']['h1']['state'], 'DONE')
self.assertEqual(state['phases']['h1']['state'], 'DONE')
with self.connect(self.database) as con:
self.assertEqual([tuple(r) for r in con.execute('SELECT stage,state FROM stages ORDER BY stage')],
[('04-calibrate', 'OK'), ('05-compile', 'FAIL')])
self.assertEqual(((self.rd / 'stages/04-calibrate/a00/meta.json').read_bytes(),
(self.rd / 'stages/04-calibrate/compile.yaml').read_bytes()), self.cal_before)
self.assertEqual(self.dispatches, 1)
def test_cancelled_compile_rejects_incomplete_or_changed_calibration_proof(self):
for fault in ('identity', 'process', 'proof', 'proof_without_pending', 'outputs', 'meta', 'artifact', 'scope', 'cleanup'):
with self.subTest(fault=fault):
other = StageRegistration(); other.setUp()
try:
other.cancel_after_inline_calibration(fault)
self.assertEqual(other.owner.snapshot()['cleanup_state'], 'QUARANTINED')
self.assertEqual(other.owner.snapshot()['stop_reason'], 'OPERATION_STATE_UNKNOWN')
with other.connect(other.database) as con:
self.assertEqual([r[0] for r in con.execute('SELECT state FROM stages')], ['RUNNING', 'RUNNING'])
self.assertEqual(other.dispatches, 1)
finally:
other.doCleanups()
def test_short_lock_releases_inside_registration_budget(self):
self.dispatch(release_after=.15)
self.assertEqual(self.lock_observations, [True])