fix: preserve completed calibration when later compilation is cancelled
This commit is contained in:
parent
13f1e99aee
commit
10663a800f
@ -354,7 +354,7 @@ class Worker:
|
||||
completion = 'meta_sha256' in result
|
||||
if completion and (not self.execution or type(attempt) is not int or attempt < 0):
|
||||
raise ValueError('completed attempt has no managed execution or valid attempt')
|
||||
if result.get('db_registered') is False and result.get('result') in ('OK', 'SKIPPED') and not completion:
|
||||
if (not write_status or result.get('db_registered') is False) and result.get('result') in ('OK', 'SKIPPED') and not completion:
|
||||
raise ValueError('completed attempt proof missing')
|
||||
if (not self.execution or name not in AGENT_RUNNABLE
|
||||
or type(attempt) is not int or attempt < 0
|
||||
@ -381,7 +381,8 @@ class Worker:
|
||||
if completion:
|
||||
with stagerun.db.connect(timeout_s=0) as con:
|
||||
con.execute('BEGIN IMMEDIATE')
|
||||
self._validate_recovered_receipt(hid, name, operation, result)
|
||||
self._validate_recovered_receipt(hid, name, operation, result,
|
||||
prerequisite=not write_status)
|
||||
meta = json.loads(meta_path.read_text()) if meta_path.exists() else None
|
||||
if meta and (meta.get('stage') != name or meta.get('attempt') != attempt):
|
||||
raise ValueError('interrupted attempt metadata mismatch')
|
||||
@ -483,7 +484,7 @@ class Worker:
|
||||
else:
|
||||
lifecycle.atomic_json(operation, previous)
|
||||
|
||||
def _validate_recovered_receipt(self, hid, name, previous, result):
|
||||
def _validate_recovered_receipt(self, hid, name, previous, result, *, prerequisite=False):
|
||||
"""A stage status proves its result; kernel receipts prove its cleanup."""
|
||||
if previous.get('operation_id') != hid:
|
||||
raise ValueError('receipt operation mismatch')
|
||||
@ -506,6 +507,9 @@ class Worker:
|
||||
raise ValueError('receipt has no dispatch proof')
|
||||
return
|
||||
leader_verified = False
|
||||
# An inline prerequisite can finish before this executor's later work is cancelled.
|
||||
completed_prerequisite = (prerequisite and name == '04-calibrate'
|
||||
and 'meta_sha256' in result and bool(result.get('outputs')))
|
||||
for scope in scopes:
|
||||
binding = scope['binding']
|
||||
receipt = lifecycle.scope_receipt(scope)
|
||||
@ -513,7 +517,7 @@ class Worker:
|
||||
or not receipt or receipt.get('cleanup_state') != 'VERIFIED'):
|
||||
raise ValueError('scope cleanup unconfirmed')
|
||||
if lifecycle.process_key(binding['leader']) == lifecycle.process_key(executor):
|
||||
if result.get('result') in ('OK', 'SKIPPED') and (receipt['exit_code'] != 0 or
|
||||
if result.get('result') in ('OK', 'SKIPPED') and ((receipt['exit_code'] != 0 and not completed_prerequisite) or
|
||||
lifecycle.process_key(result.get('process', {})) != lifecycle.process_key(executor)):
|
||||
raise ValueError('successful receipt has no matching executor')
|
||||
leader_verified = True
|
||||
|
||||
@ -20,7 +20,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
CONNECT = db.connect
|
||||
READONLY = db.readonly
|
||||
CHILD = r'''
|
||||
import sys
|
||||
import sys, time
|
||||
from pathlib import Path
|
||||
from contextlib import nullcontext
|
||||
from functools import partial
|
||||
@ -34,6 +34,9 @@ class Once(Stage):
|
||||
name = '05-compile'
|
||||
def native(self, ctx, sd):
|
||||
with (rd / 'effects').open('a') as f: f.write('one\n')
|
||||
if outcome == 'BLOCK':
|
||||
(rd / 'compile-started').touch()
|
||||
while True: time.sleep(.1)
|
||||
(work_dir(ctx) / 'model.hbm').write_bytes(b'actual-compiled-artifact')
|
||||
return StageOutcome(outcome, 2 if outcome == 'FAIL' else 0,
|
||||
'fixture_failure' if outcome == 'FAIL' else None)
|
||||
@ -529,6 +532,134 @@ stageworker.Worker(rd, execution=owner).scan()
|
||||
self.assertEqual((self.rd / 'cal-effects').read_text(), 'one\n')
|
||||
self.assertEqual((self.rd / 'effects').read_text(), 'one\n')
|
||||
|
||||
def cancel_after_inline_calibration(self, fault=None):
|
||||
job_path = self.rd.parent.parent / 'job.json'
|
||||
job = json.loads(job_path.read_text()); job['backend'] = 'ptq'
|
||||
job_path.write_text(json.dumps(job))
|
||||
cal_meta = self.rd / 'stages/04-calibrate/a00/meta.json'
|
||||
cal_output = self.rd / 'stages/04-calibrate/compile.yaml'
|
||||
def run(command, **kw):
|
||||
self.dispatches += 1
|
||||
original_check = kw['stop_check']
|
||||
def cancel_when_compiling():
|
||||
if (self.rd / 'compile-started').exists() and not self.owner.snapshot().get('cancel_requested'):
|
||||
self.cal_proof = json.loads((self.queue / 'status.json').read_text())['prerequisites'][0]
|
||||
self.cal_before = cal_meta.read_bytes(), cal_output.read_bytes()
|
||||
self.owner.cancel('TEST_WINDOW_ENDED')
|
||||
return original_check()
|
||||
kw.update(stop_check=cancel_when_compiling, poll_s=.01, hard_ceiling_s=5)
|
||||
result = self.run([sys.executable, '-c', CHILD, str(self.queue), str(self.rd),
|
||||
str(self.database), 'BLOCK'], **kw)
|
||||
self.assertEqual((result.exit_code, result.cleanup_state), (-15, 'VERIFIED'))
|
||||
self.assertEqual(execution.owned_processes(self.owner.snapshot()), [])
|
||||
status = json.loads((self.queue / 'status.json').read_text())
|
||||
proof = status['prerequisites'][0]
|
||||
if fault == 'identity': proof['execution_id'] = 'other'
|
||||
elif fault == 'process': proof['process']['proc_start_ticks'] += 1
|
||||
elif fault == 'proof': proof.pop('meta_sha256')
|
||||
elif fault == 'proof_without_pending':
|
||||
proof.pop('meta_sha256'); proof['db_registered'] = True
|
||||
elif fault == 'outputs': proof['outputs'] = {}
|
||||
elif fault == 'meta': cal_meta.write_text('{}')
|
||||
elif fault == 'artifact': cal_output.write_text('changed')
|
||||
elif fault == 'scope':
|
||||
with self.owner.mutation() as state:
|
||||
state['process_scopes'] = {}
|
||||
self.owner._save(state, 'TEST_REMOVE_SCOPE')
|
||||
elif fault == 'cleanup':
|
||||
scope = next(iter(self.owner.snapshot()['process_scopes'].values()))
|
||||
path = Path(scope['receipt_path']); receipt = json.loads(path.read_text())
|
||||
receipt['cleanup_state'] = 'UNKNOWN'; path.write_text(json.dumps(receipt))
|
||||
if fault: (self.queue / 'status.json').write_text(json.dumps(status))
|
||||
return result
|
||||
with patch.object(stageworker.watchdog, 'run', side_effect=run):
|
||||
self.worker.scan()
|
||||
|
||||
def test_cancel_after_inline_calibration_preserves_proof_and_settles_once(self):
|
||||
from harness import collection_runtime, paths, pipeline, recovery
|
||||
self.cancel_after_inline_calibration()
|
||||
status = json.loads((self.queue / 'status.json').read_text())
|
||||
self.assertEqual((status['result'], status['cleanup_state']), ('CANCELLED', 'VERIFIED'))
|
||||
self.assertEqual(status['prerequisites'][0], self.cal_proof)
|
||||
with self.connect(self.database) as con:
|
||||
rows = [tuple(row) for row in con.execute('SELECT stage,state,exit_code,error_class FROM stages ORDER BY stage')]
|
||||
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='CANCELLED')
|
||||
self.assertEqual(rows, [('04-calibrate', 'OK', 0, None), ('05-compile', 'FAIL', -15, 'CANCEL_REQUESTED')])
|
||||
self.assertEqual(self.owner.snapshot()['operations']['h1']['state'], 'DONE')
|
||||
self.assertEqual(self.owner.snapshot()['phases']['h1']['state'], 'DONE')
|
||||
operation = json.loads((self.queue / 'operation.json').read_text())
|
||||
with self.assertRaisesRegex(ValueError, 'successful receipt has no matching executor'):
|
||||
self.worker._validate_recovered_receipt('h1', '04-calibrate', operation, self.cal_proof)
|
||||
self.owner.cleanup()
|
||||
state = self.owner.finish(-15, protocol_complete=False)
|
||||
self.assertEqual((state['execution_outcome'], state['cleanup_state']), ('CANCELLED', 'VERIFIED'))
|
||||
before = (self.queue / 'operation.json').read_bytes(), (self.queue / 'status.json').read_bytes()
|
||||
with patch.object(db, 'readonly', lambda: READONLY(self.database)), \
|
||||
patch.object(paths, 'RUNS', self.root / 'runs'), \
|
||||
patch.object(paths, 'STATE', self.root / 'lease-state'), \
|
||||
patch.object(stageworker.watchdog, 'run', side_effect=AssertionError('recovery dispatched work')):
|
||||
for _ in range(2):
|
||||
self.assertTrue(recovery.reconcile_run('r', attempts=[('j', 0)])['ready'])
|
||||
paths.STATE.mkdir(parents=True, exist_ok=True)
|
||||
pipeline.collect_drain_file('r').touch()
|
||||
messages = []
|
||||
with patch.object(collection_runtime, 'config', return_value={}), \
|
||||
patch.object(paths, 'config', return_value={'pools': {'agent': {'workers': 1}}}):
|
||||
self.assertEqual(pipeline.collect_rollouts('r', log=messages.append), {})
|
||||
self.assertTrue(any('drained' in message for message in messages))
|
||||
self.assertEqual(((self.queue / 'operation.json').read_bytes(), (self.queue / 'status.json').read_bytes()), before)
|
||||
self.assertEqual(((self.rd / 'stages/04-calibrate/a00/meta.json').read_bytes(),
|
||||
(self.rd / 'stages/04-calibrate/compile.yaml').read_bytes()), self.cal_before)
|
||||
self.assertEqual((self.rd / 'cal-effects').read_text(), 'one\n')
|
||||
self.assertEqual((self.rd / 'effects').read_text(), 'one\n')
|
||||
self.assertEqual(self.dispatches, 1)
|
||||
|
||||
def test_legacy_cancel_quarantine_reconciles_without_erasing_history(self):
|
||||
from harness import paths, recovery
|
||||
validate = self.worker._validate_recovered_receipt
|
||||
def legacy_validator(hid, name, operation, result, **kw):
|
||||
if name == '04-calibrate':
|
||||
raise ValueError('successful receipt has no matching executor')
|
||||
return validate(hid, name, operation, result, **kw)
|
||||
# Reproduce only the old rejection; child cancellation and cleanup receipts are real.
|
||||
with patch.object(self.worker, '_validate_recovered_receipt', side_effect=legacy_validator):
|
||||
self.cancel_after_inline_calibration()
|
||||
state = self.owner.finish(-15, protocol_complete=False)
|
||||
self.assertEqual((state['execution_outcome'], state['cleanup_state']), ('CANCELLED', 'QUARANTINED'))
|
||||
self.assertEqual(state['operations']['h1']['state'], 'DISPATCHED')
|
||||
with self.connect(self.database) as con:
|
||||
db.rollout_upsert(con, 'r', 'j', 0, 'e2e', 'teacher', state='CANCELLED')
|
||||
with patch.object(db, 'readonly', lambda: READONLY(self.database)), \
|
||||
patch.object(paths, 'RUNS', self.root / 'runs'), \
|
||||
patch.object(paths, 'STATE', self.root / 'lease-state'), \
|
||||
patch.object(stageworker.watchdog, 'run', side_effect=AssertionError('recovery dispatched work')):
|
||||
for _ in range(2):
|
||||
self.assertTrue(recovery.reconcile_run('r', attempts=[('j', 0)])['ready'])
|
||||
state = self.owner.snapshot()
|
||||
self.assertEqual((state['cleanup_state'], state['stop_reason']), ('QUARANTINED', 'OPERATION_STATE_UNKNOWN'))
|
||||
self.assertEqual(state['operations']['h1']['state'], 'DONE')
|
||||
self.assertEqual(state['phases']['h1']['state'], 'DONE')
|
||||
with self.connect(self.database) as con:
|
||||
self.assertEqual([tuple(r) for r in con.execute('SELECT stage,state FROM stages ORDER BY stage')],
|
||||
[('04-calibrate', 'OK'), ('05-compile', 'FAIL')])
|
||||
self.assertEqual(((self.rd / 'stages/04-calibrate/a00/meta.json').read_bytes(),
|
||||
(self.rd / 'stages/04-calibrate/compile.yaml').read_bytes()), self.cal_before)
|
||||
self.assertEqual(self.dispatches, 1)
|
||||
|
||||
def test_cancelled_compile_rejects_incomplete_or_changed_calibration_proof(self):
|
||||
for fault in ('identity', 'process', 'proof', 'proof_without_pending', 'outputs', 'meta', 'artifact', 'scope', 'cleanup'):
|
||||
with self.subTest(fault=fault):
|
||||
other = StageRegistration(); other.setUp()
|
||||
try:
|
||||
other.cancel_after_inline_calibration(fault)
|
||||
self.assertEqual(other.owner.snapshot()['cleanup_state'], 'QUARANTINED')
|
||||
self.assertEqual(other.owner.snapshot()['stop_reason'], 'OPERATION_STATE_UNKNOWN')
|
||||
with other.connect(other.database) as con:
|
||||
self.assertEqual([r[0] for r in con.execute('SELECT state FROM stages')], ['RUNNING', 'RUNNING'])
|
||||
self.assertEqual(other.dispatches, 1)
|
||||
finally:
|
||||
other.doCleanups()
|
||||
|
||||
def test_short_lock_releases_inside_registration_budget(self):
|
||||
self.dispatch(release_after=.15)
|
||||
self.assertEqual(self.lock_observations, [True])
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user