data-gen-bpu/scripts/test_execution.py

396 lines
21 KiB
Python

"""Lifecycle contracts using only temporary state and owned test processes."""
import os
import signal
import subprocess
import sys
import tempfile
import threading
import json
import hashlib
import fcntl
import time
import unittest
from pathlib import Path
from unittest.mock import patch
from harness import execution, scheduler, watchdog
def policy():
base = dict(queue_deadline_s=.15, execution_deadline_s=10, suspect_after_s=.1,
cancel_grace_s=.1, cleanup_deadline_s=2)
return dict(base, version='test-v1', phases={name: dict(base) for name in execution.PHASES})
class ExecutionTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.rd = Path(self.tmp.name)
self.ex = execution.Execution(self.rd)
self.ex.start(policy())
if not Path('/proc').is_dir():
# Production admission requires Linux identity; mock only this known
# test controller for portable pure state-machine checks.
original = execution.process_status
owner = execution.process_key(self.ex.snapshot()['owner'])
override = patch.object(execution, 'process_status', side_effect=lambda identity:
'ALIVE' if execution.process_key(identity) == owner else original(identity))
override.start(); self.addCleanup(override.stop)
def test_cancel_fences_spawn_and_cannot_be_completed(self):
self.ex.cancel('test cancel')
with self.assertRaises(execution.ExecutionStopped):
with self.ex.admission():
self.fail('cancelled admission opened')
result = self.ex.finish(0, protocol_complete=True)
self.assertEqual(result['execution_outcome'], 'CANCELLED')
self.assertEqual(result['cleanup_state'], 'VERIFIED')
def test_exit_and_protocol_are_independent(self):
self.assertEqual(self.ex.finish(143, protocol_complete=True)['execution_outcome'], 'AGENT_ERROR')
other = execution.Execution(self.rd / 'other'); other.start(policy())
self.assertEqual(other.finish(0, protocol_complete=False)['execution_outcome'], 'AGENT_ERROR')
def test_late_cancel_does_not_overwrite_completion(self):
self.ex.finish(0, protocol_complete=True)
self.assertEqual(self.ex.cancel('late')['reason'], 'ALREADY_TERMINAL')
self.assertEqual(self.ex.snapshot()['execution_outcome'], 'COMPLETED')
def test_old_epoch_is_fenced(self):
stale = execution.Execution(self.rd)
stale.epoch = self.ex.epoch - 1
with self.assertRaises(execution.ExecutionStopped):
with stale.admission():
self.fail('old owner admitted')
def test_queue_wait_is_bounded(self):
with patch.object(scheduler, '_try_slots', return_value=(None, None)), \
patch.object(scheduler, 'pool_size', return_value=1), \
patch.object(scheduler, 'pool_reserve', return_value=0), self.ex.scope('05-compile'):
before = time.monotonic()
with self.assertRaises(execution.ExecutionStopped):
with scheduler.acquire('cpu', poll_s=.01):
self.fail('unavailable slot acquired')
self.assertLess(time.monotonic() - before, 1)
def test_pid_reuse_is_not_alive(self):
ident = execution.process_identity(os.getpid())
if ident.get('status') == 'UNKNOWN':
self.skipTest('Linux proc identity required')
ident['proc_start_ticks'] += 1
self.assertEqual(execution.process_status(ident), 'DEAD')
def test_restart_does_not_reset_deadline_or_replay(self):
state = self.ex.snapshot()
duplicate = execution.Execution(self.rd)
with self.assertRaises(execution.ExecutionStopped):
duplicate.start(policy())
self.assertEqual(duplicate.snapshot()['deadline_monotonic'], state['deadline_monotonic'])
def test_completed_requires_verified_cleanup(self):
result = self.ex.finish(0, protocol_complete=True, cleanup_state='UNKNOWN')
self.assertEqual(result['cleanup_state'], 'QUARANTINED')
self.assertNotEqual(result['execution_outcome'], 'COMPLETED')
def test_cancel_serializes_with_spawn_admission(self):
started, committed = threading.Event(), threading.Event()
def cancel():
started.set()
self.ex.cancel('race fixture')
committed.set()
with self.ex.admission():
thread = threading.Thread(target=cancel)
thread.start(); started.wait(1)
self.assertFalse(committed.wait(.03))
thread.join(1)
self.assertTrue(committed.is_set())
with self.assertRaises(execution.ExecutionStopped):
with self.ex.admission():
self.fail('post-cancel spawn admitted')
def test_crashed_holder_lease_is_not_reallocated(self):
with patch.object(scheduler.paths, 'STATE', self.rd / 'state'), self.ex.scope('05-compile'):
lease, _ = scheduler._try_slots('cpu', [0])
self.assertIsNotNone(lease)
fcntl.flock(lease, fcntl.LOCK_UN); lease.close() # Simulate kernel lock release after a crash.
next_lease, _ = scheduler._try_slots('cpu', [0])
self.assertIsNone(next_lease)
self.assertTrue((self.rd / 'state/locks/cpu-0.lease.json').exists())
def test_unknown_telemetry_does_not_disable_deadline(self):
with patch.object(watchdog, '_proc_tree_stat', return_value=None):
before = time.monotonic()
result = watchdog.run([sys.executable, '-c', 'import time; time.sleep(60)'], cwd=self.rd,
stdout=self.rd/'out', stderr=self.rd/'err', execution=self.ex, poll_s=.01, hard_ceiling_s=.1)
self.assertLess(time.monotonic() - before, 3)
self.assertIn('DEADLINE', result.stalled_reason)
self.assertEqual(result.cleanup_state, 'VERIFIED')
def test_expired_start_bookkeeping_never_releases_command_gate(self):
marker = self.rd / 'must-not-start'
result = watchdog.run([sys.executable, '-c',
"from pathlib import Path;Path('must-not-start').touch()"], cwd=self.rd,
stdout=self.rd/'gate.out', stderr=self.rd/'gate.err', execution=self.ex,
poll_s=.01, hard_ceiling_s=.1, on_start=lambda _: time.sleep(.2))
self.assertIn('DEADLINE', result.stalled_reason)
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertFalse(marker.exists())
self.assertLess(result.duration_s, 3)
def test_predeadline_channel_loss_remains_infrastructure_error(self):
def stop_supervisor(proc):
execution.signal_process(proc.supervisor_identity, signal.SIGTERM)
proc.supervisor.wait(timeout=3)
result = watchdog.run([sys.executable, '-c',
"from pathlib import Path;Path('must-not-start').touch()"], cwd=self.rd,
stdout=self.rd/'closed.out', stderr=self.rd/'closed.err', execution=self.ex,
poll_s=.01, hard_ceiling_s=5, on_start=stop_supervisor)
self.assertIn('runner monitor failed:', result.stalled_reason)
self.assertNotIn('DEADLINE', result.stalled_reason)
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertFalse((self.rd/'must-not-start').exists())
def test_suspect_is_advisory_not_an_idle_kill(self):
with patch.object(watchdog, '_proc_tree_stat', return_value=(100, 0)), \
patch.object(watchdog, '_io_bytes', return_value=0):
result = watchdog.run([sys.executable, '-c', 'import time; time.sleep(.25)'], cwd=self.rd,
stdout=self.rd/'out', stderr=self.rd/'err', execution=self.ex, poll_s=.01)
self.assertEqual(result.state, 'OK')
self.assertTrue(result.suspect)
def worker_fixture(self):
from harness.stageworker import Worker
rd = self.rd / 'guard-smoke' / 'rollout'
(rd / 'handles/h1').mkdir(parents=True)
(rd / 'workspace').mkdir()
owner = execution.Execution(rd); owner.start(policy())
request = {'stage': 'noop', 'submitted_at': 1}
(rd / 'handles/h1/request.json').write_text(json.dumps(request))
return rd, owner, Worker(rd, test_noop=True, execution=owner)
def test_worker_restart_reuses_receipt_without_dispatch(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
worker.scan()
operation = rd / 'stages/_queue/h1/operation.json'
before = operation.read_bytes()
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(operation.read_bytes(), before)
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['result'], 'OK')
def test_worker_payload_conflict_preserves_original_receipt(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
worker.scan()
receipt = rd / 'stages/_queue/h1/status.json'
before = receipt.read_bytes()
(rd / 'handles/h1/request.json').write_text('{"stage":"noop","submitted_at":2}')
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(receipt.read_bytes(), before)
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['error_class'],
'OPERATION_PAYLOAD_CONFLICT')
def test_unknown_operation_is_quarantined_not_replayed(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
worker.scan()
(rd/'stages/_queue/h1/status.json').unlink()
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(owner.snapshot()['cleanup_state'], 'QUARANTINED')
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['error_class'],
'SIDE_EFFECT_UNKNOWN')
def checkpoint(self):
path = self.rd / 'session.jsonl'
path.write_text('{"message_id":"m1"}\n')
return {'complete': True, 'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}
def test_pause_resume_preserves_identity_deadline_and_fences_previous_epoch(self):
before = self.ex.snapshot()
stale = execution.Execution(self.rd).attach()
self.ex.pause('QUOTA_WAIT_BUDGET_EXHAUSTED', self.checkpoint())
with self.assertRaises(execution.ExecutionStopped):
with self.ex.admission():
self.fail('paused execution dispatched')
with patch.object(execution, 'marker_processes', return_value=[]):
state = self.ex.resume()
self.assertEqual(state['execution_id'], before['execution_id'])
self.assertEqual(state['deadline_monotonic'], before['deadline_monotonic'])
self.assertEqual(state['epoch'], before['epoch'] + 1)
self.assertEqual(state['auto_resume_count'], 1)
with self.assertRaises(execution.ExecutionStopped):
with stale.admission():
self.fail('old epoch survived takeover')
def test_incomplete_or_changed_checkpoint_cannot_resume(self):
with self.assertRaisesRegex(execution.ExecutionStopped, 'INCOMPLETE'):
self.ex.pause('quota', {})
checkpoint = self.checkpoint()
self.ex.pause('quota', checkpoint)
Path(checkpoint['path']).write_text('changed')
with self.assertRaisesRegex(execution.ExecutionStopped, 'HASH_MISMATCH'):
self.ex.resume()
def test_auto_resume_budget_survives_new_controller_objects(self):
checkpoint = self.checkpoint()
with patch.object(execution, 'marker_processes', return_value=[]):
for _ in range(3):
self.ex.pause('quota', checkpoint)
self.ex = execution.Execution(self.rd).attach()
self.ex.resume()
self.ex.pause('quota', checkpoint)
with self.assertRaisesRegex(execution.ExecutionStopped, 'AUTO_RESUME_BUDGET_EXHAUSTED'):
execution.Execution(self.rd).resume()
@unittest.skipUnless(Path('/proc').is_dir(), 'Actual worker requires Linux owner identity')
def test_worker_executes_real_protected_stage_and_reuses_receipt(self):
from harness import stageworker, stagerun
import yaml
root = self.rd / 'fixture-project'
root.mkdir()
source = Path(__file__).resolve().parents[1]
(root/'harness').symlink_to(source/'harness', target_is_directory=True)
(root/'registry').symlink_to(source/'registry', target_is_directory=True)
cfg = {'paths': {'harness_python': sys.executable}, 'pools': {'cpu': {'workers': 1}}}
(root/'config.yaml').write_text(yaml.safe_dump(cfg))
weights = root/'cache/weights/fixture-sha'
weights.mkdir(parents=True)
(weights.parent/'index.json').write_text('{"fixture":{"sha256":"fixture-sha"}}')
rd = root/'runs/r/j/rollouts/00'
(rd/'handles/h1').mkdir(parents=True)
(rd/'workspace').mkdir()
(rd.parent.parent/'job.json').write_text(json.dumps({'run_id':'r', 'job_id':'j',
'model':'fixture', 'task':'cls', 'march':'nash-p', 'backend':'ptq'}))
(rd/'handles/h1/request.json').write_text('{"stage":"01-fetch"}')
owner = execution.Execution(rd)
fixture_policy = policy(); fixture_policy['phases']['01-fetch']['queue_deadline_s'] = 2
owner.start(fixture_policy)
with patch.object(stagerun.paths, 'BENCH_ROOT', root), \
patch.object(stagerun.paths, 'config', return_value=cfg), \
patch.dict(os.environ, {'BENCH_CONFIG': str(root/'config.yaml'), 'BENCH_STATE_DIR': str(root/'state')}):
worker = stageworker.Worker(rd, execution=owner)
worker.scan()
status = json.loads((rd/'handles/h1/status.json').read_text())
self.assertEqual(status['result'], 'SKIPPED', status)
self.assertEqual(owner.snapshot()['operations']['h1']['state'], 'DONE')
with patch.object(stageworker.watchdog, 'run') as replay:
stageworker.Worker(rd, execution=owner).scan()
replay.assert_not_called()
self.assertFalse(list((root/'state/locks').glob('*.lease.json')))
def test_receipt_cannot_bind_reused_supervisor_pid(self):
binding = dict(proof='waitpid:ECHILD', supervisor=dict(pid=123, proc_start_ticks=1))
receipt = self.rd/'receipt.json'
execution.atomic_json(receipt, dict(binding=dict(binding, supervisor=dict(pid=123, proc_start_ticks=2)),
cleanup_state='VERIFIED', all_children_reaped=True, owner_lost=False, exit_code=0))
self.assertIsNone(execution.scope_receipt(dict(binding=binding, receipt_path=str(receipt))))
def test_board_exception_preserves_lease_and_quarantines_before_release(self):
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock
from harness import stagerun
hd = self.rd/'stages/_queue/h1'; hd.mkdir(parents=True)
(hd/'request.json').write_text('{"stage":"07-board"}')
class Board:
name, pool, level = '07-board', 'board', 'job'
def run(self, ctx):
raise RuntimeError('transport failed after dispatch')
ctx = SimpleNamespace(job={'model':'m','march':'nash-p','job_id':'j'},extra={},
run_id='r',rollout=0,lane='normal',agent_config=None)
with patch.object(stagerun, '_ctx_from_rollout', return_value=ctx), \
patch.dict(stagerun.BY_NAME, {'07-board':Board}), \
patch.object(stagerun.db, 'connect', side_effect=lambda:nullcontext(Mock())), \
patch.object(stagerun.db, 'stage_start'), \
patch.object(scheduler, 'board_slots', return_value={0:{'name':'fixture','march':'nash-p'}}), \
patch.object(scheduler.paths, 'STATE', self.rd/'state'), \
patch.object(scheduler.paths, 'config', return_value={'pools':{'board':{'reserve_for_agent':0}}}), \
self.ex.scope('07-board'):
with self.assertRaisesRegex(RuntimeError, 'transport'):
stagerun._exec_impl(hd, rd=self.rd, protected=True)
self.assertEqual(self.ex.snapshot()['cleanup_state'], 'QUARANTINED')
self.assertTrue((self.rd/'state/locks/board-0.lease.json').exists())
self.assertTrue((self.rd/'state/locks/board-0.quarantine.json').exists())
def test_stale_owner_cannot_mutate_or_cleanup(self):
old = execution.Execution(self.rd).attach()
self.ex.cancel('fixture')
state = self.ex.snapshot(); state['epoch'] += 1; self.ex.epoch = state['epoch']
execution.atomic_json(self.ex.path, state)
before = self.ex.path.read_bytes()
actions = [lambda:old.quarantine('late'), lambda:old.cleanup(), lambda:old.cancel('late'),
lambda:old.activity('late'), lambda:old.finish(0, protocol_complete=True)]
for action in actions:
with self.assertRaisesRegex(execution.ExecutionStopped, 'OWNER_FENCED'):
action()
self.assertEqual(self.ex.path.read_bytes(), before)
def test_unknown_owner_blocks_admission_but_deadline_still_expires(self):
with patch.object(execution, 'process_status', return_value='UNKNOWN'):
with self.assertRaisesRegex(execution.ExecutionStopped, 'OWNER_STATUS_UNKNOWN'):
with self.ex.admission():
self.fail('unknown owner admitted')
self.assertIsNone(self.ex.stop_reason())
state = self.ex.snapshot(); state['deadline_monotonic'] = time.monotonic() - 1
execution.atomic_json(self.ex.path, state)
self.assertEqual(self.ex.stop_reason(), 'EXECUTION_DEADLINE_EXCEEDED')
def test_phase_queue_and_execution_clocks_are_separate_and_supervised(self):
owner = execution.Execution(self.rd/'phase')
cfg = policy(); cfg['phases']['05-compile'].update(queue_deadline_s=2, execution_deadline_s=.1)
owner.start(cfg)
owner.begin_phase('h1', '05-compile')
self.assertIsNone(owner.snapshot()['phases']['h1']['execution_deadline_monotonic'])
time.sleep(.12) # More than the execution budget, still legitimately queued.
self.assertIsNone(owner.stop_reason('h1'))
owner.phase_running('h1')
with owner.scope('05-compile', 'h1'):
result = watchdog.run([sys.executable, '-c', 'import time;time.sleep(.4)'], cwd=self.rd,
stdout=self.rd/'phase.out', stderr=self.rd/'phase.err', execution=owner, role='stage',
env=dict(os.environ, BENCH_OPERATION_ID='h1'), poll_s=.02, hard_ceiling_s=5)
self.assertIn('PHASE_EXECUTION_DEADLINE', result.stalled_reason)
self.assertLess(result.duration_s, .35)
self.assertIsNotNone(owner.snapshot()['phases']['h1']['execution_deadline_monotonic'])
def test_typed_heartbeat_is_not_business_progress(self):
path = self.rd/'events'; path.touch()
tracker = watchdog.BusinessLog(path)
path.write_text('{"type":"heartbeat","state":"RUNNING"}\n')
self.assertEqual(tracker.sample(), 0)
with path.open('a') as out:
out.write('{"type":"message_end","message":{"role":"assistant","content":[]}}\n')
self.assertGreater(tracker.sample(), 0)
with path.open('a') as out:
out.write('unclassified operational text\n')
self.assertIsNone(tracker.sample())
def test_progress_health_is_persisted_and_unknown_not_zero(self):
signals = dict(cpu_ticks=1, io_bytes=0, business_log_bytes=0, stage_sequence=0)
self.ex.record_progress('p', signals)
time.sleep(.12)
self.ex.record_progress('p', signals)
self.assertEqual(self.ex.snapshot()['health'], 'SUSPECT')
self.ex.record_progress('p', dict(signals, io_bytes=None))
state = self.ex.snapshot()
self.assertEqual(state['health'], 'UNKNOWN')
self.assertIsNone(state['progress']['signals']['io_bytes'])
def test_submission_replay_retains_operation_identity(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
source = rd/'stages/_queue/source'; source.mkdir(parents=True)
(source/'recipe.json').write_text('{}')
(source/'request.json').write_text(json.dumps({'recipe':str(source/'recipe.json')}))
(rd/'handles/h1/request.json').write_text('{"stage":"noop","submission":"source"}')
worker.scan()
record = json.loads((rd/'stages/_queue/h1/operation.json').read_text())
self.assertIn('payload_sha256', record)
self.assertEqual(record['execution_id'], owner.snapshot()['execution_id'])
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['result'], 'OK')
if __name__ == '__main__':
unittest.main()