"""Lifecycle contracts using only temporary state and owned test processes.""" import os import signal import subprocess import sys import tempfile import threading import json import hashlib import fcntl import time import unittest from pathlib import Path from unittest.mock import patch from harness import execution, process_reaper, scheduler, watchdog def policy(): base = dict(queue_deadline_s=.15, execution_deadline_s=10, suspect_after_s=.1, cancel_grace_s=.1, cleanup_deadline_s=2) return dict(base, version='test-v1', phases={name: dict(base) for name in execution.PHASES}) class ProcessStatBytesTests(unittest.TestCase): @staticmethod def stat(comm=b'worker', *, birth=42, state=b'S', ppid=10): fields = [state, str(ppid).encode(), b'10', b'10'] + [b'0'] * 16 fields[11:13] = [b'13', b'17'] fields[19] = str(birth).encode() return b'123 (' + comm + b') ' + b' '.join(fields) + b'\n' def test_identity_ignores_arbitrary_comm_bytes(self): for comm in (b'worker', b'a (b) c', b'\xff', '中文线程名截断'.encode()[:15]): with self.subTest(comm=comm), patch.object(Path, 'read_bytes', return_value=self.stat(comm)): identity = execution.process_identity(123) self.assertEqual(identity['status'], 'KNOWN') self.assertEqual((identity['proc_start_ticks'], identity['ppid'], identity['state'], identity['cpu_ticks']), (42, 10, 'S', 30)) json.dumps(identity) # Receipts keep their existing text/number schema. def test_corrupt_identity_is_unknown_and_never_authorizes_a_signal(self): original = dict(host=execution.socket.gethostname(), boot_id='fixture', pid=123, proc_start_ticks=42) for raw in (b'', b'123 no delimiter', self.stat(state=b'\xff'), self.stat().replace(b'42\n', b'bad\n')): with self.subTest(raw=raw), patch.object(Path, 'read_bytes', return_value=raw), \ patch.object(Path, 'exists', return_value=True), \ patch.object(execution, 'boot_id', return_value='fixture'), \ patch.object(os, 'pidfd_open', create=True) as opened: self.assertEqual(execution.process_identity(123)['status'], 'UNKNOWN') self.assertEqual(execution.process_status(original), 'UNKNOWN') self.assertFalse(execution.signal_process(original, signal.SIGTERM)) opened.assert_not_called() def test_reaper_preserves_second_birth_and_parent_check(self): reaper = process_reaper.Reaper.__new__(process_reaper.Reaper) reaper.pid, reaper.host, reaper.boot_id, reaper.members = 10, 'fixture', 'fixture', {} first = self.stat(b'\xff name (nested)') for second, permitted in ((first, True), (self.stat(birth=43), False), (self.stat(ppid=11), False)): with self.subTest(permitted=permitted), \ patch.object(process_reaper, 'children', return_value={123}), \ patch.object(Path, 'read_bytes', side_effect=[first, second]), \ patch.object(os, 'pidfd_open', return_value=99, create=True), \ patch.object(os, 'close') as closed, \ patch.object(signal, 'pidfd_send_signal', create=True) as sent: reaper.signal(signal.SIGTERM) self.assertEqual(sent.call_count, int(permitted)) closed.assert_called_once_with(99) self.assertEqual(reaper.members[(123, 42)]['state'], 'S') @unittest.skipUnless(sys.platform == 'linux', 'Linux comm and subreaper') def test_non_utf8_child_name_is_still_identified_and_reaped(self): with tempfile.TemporaryDirectory() as name: root = Path(name) owner = execution.Execution(root); owner.start(policy()) code = ("import ctypes,time;from pathlib import Path;" "assert ctypes.CDLL(None).prctl(15,b'worker-\\xff',0,0,0)==0;" "Path('armed').touch();time.sleep(30)") result = watchdog.run([sys.executable, '-c', code], cwd=root, stdout=root/'out', stderr=root/'err', execution=owner, poll_s=.01, hard_ceiling_s=3, stop_check=lambda: 'fixture complete' if (root/'armed').exists() else None) self.assertEqual(result.stalled_reason, 'fixture complete') self.assertEqual(result.cleanup_state, 'VERIFIED') for identity in owner.snapshot()['processes']: self.assertNotEqual(execution.process_status(identity), 'ALIVE') class ExecutionTests(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.rd = Path(self.tmp.name) self.ex = execution.Execution(self.rd) self.ex.start(policy()) if not Path('/proc').is_dir(): # Production admission requires Linux identity; mock only this known # test controller for portable pure state-machine checks. original = execution.process_status owner = execution.process_key(self.ex.snapshot()['owner']) override = patch.object(execution, 'process_status', side_effect=lambda identity: 'ALIVE' if execution.process_key(identity) == owner else original(identity)) override.start(); self.addCleanup(override.stop) def test_cancel_fences_spawn_and_cannot_be_completed(self): self.ex.cancel('test cancel') with self.assertRaises(execution.ExecutionStopped): with self.ex.admission(): self.fail('cancelled admission opened') result = self.ex.finish(0, protocol_complete=True) self.assertEqual(result['execution_outcome'], 'CANCELLED') self.assertEqual(result['cleanup_state'], 'VERIFIED') @unittest.skipUnless(sys.platform == 'linux', 'real owned process cleanup') def test_worker_drain_uses_stage_deadline_not_cleanup_budget(self): from harness.stageworker import Worker owner = execution.Execution(self.rd / 'drain') cfg = policy(); cfg.update(cleanup_deadline_s=.05, cancel_grace_s=.01) owner.start(cfg) worker = Worker(owner.rd, execution=owner) started, results = threading.Event(), [] def stage(): with owner.scope('05-compile', 'h1'): owner.begin_phase('h1', '05-compile'); owner.phase_running('h1') results.append(watchdog.run([sys.executable, '-c', 'import time;time.sleep(.3)'], cwd=owner.rd, stdout=owner.rd/'out', stderr=owner.rd/'err', execution=owner, role='stage', poll_s=.01, on_start=lambda _: started.set())) worker.thread = threading.Thread(target=stage) with worker: self.assertTrue(started.wait(3)) self.assertEqual(results[0].state, 'OK') self.assertGreater(results[0].duration_s, cfg['cleanup_deadline_s']) self.assertFalse(owner.snapshot()['cancel_requested']) self.assertEqual(results[0].cleanup_state, 'VERIFIED') @unittest.skipUnless(sys.platform == 'linux', 'real owned process cleanup') def test_worker_drain_keeps_explicit_cancel_and_exception_cleanup(self): from harness.stageworker import Worker for explicit in (True, False): with self.subTest(explicit=explicit): owner = execution.Execution(self.rd / str(explicit)); owner.start(policy()) worker = Worker(owner.rd, execution=owner) started, results = threading.Event(), [] def stage(): with owner.scope('05-compile', 'h1'): owner.begin_phase('h1', '05-compile'); owner.phase_running('h1') results.append(watchdog.run([sys.executable, '-c', 'import time;time.sleep(20)'], cwd=owner.rd, stdout=owner.rd/'out', stderr=owner.rd/'err', execution=owner, role='stage', poll_s=.01, on_start=lambda _: started.set())) worker.thread = threading.Thread(target=stage) try: with worker: self.assertTrue(started.wait(3)) if explicit: owner.cancel('USER_STOP') else: raise ValueError('fixture context failed') except ValueError as error: self.assertFalse(explicit) self.assertEqual(str(error), 'fixture context failed') self.assertEqual(owner.snapshot()['cancel_requested']['reason'], 'USER_STOP' if explicit else 'WORKER_CONTEXT_ERROR') self.assertEqual(results[0].cleanup_state, 'VERIFIED') self.assertEqual(execution.owned_processes(owner.snapshot()), []) def test_exit_and_protocol_are_independent(self): self.assertEqual(self.ex.finish(143, protocol_complete=True)['execution_outcome'], 'AGENT_ERROR') other = execution.Execution(self.rd / 'other'); other.start(policy()) self.assertEqual(other.finish(0, protocol_complete=False)['execution_outcome'], 'AGENT_ERROR') def test_late_cancel_does_not_overwrite_completion(self): self.ex.finish(0, protocol_complete=True) self.assertEqual(self.ex.cancel('late')['reason'], 'ALREADY_TERMINAL') self.assertEqual(self.ex.snapshot()['execution_outcome'], 'COMPLETED') def test_old_epoch_is_fenced(self): stale = execution.Execution(self.rd) stale.epoch = self.ex.epoch - 1 with self.assertRaises(execution.ExecutionStopped): with stale.admission(): self.fail('old owner admitted') def test_queue_wait_is_bounded(self): with patch.object(scheduler, '_try_slots', return_value=(None, None)), \ patch.object(scheduler, 'pool_size', return_value=1), \ patch.object(scheduler, 'pool_reserve', return_value=0), self.ex.scope('05-compile'): before = time.monotonic() with self.assertRaises(execution.ExecutionStopped): with scheduler.acquire('cpu', poll_s=.01): self.fail('unavailable slot acquired') self.assertLess(time.monotonic() - before, 1) def test_pid_reuse_is_not_alive(self): ident = execution.process_identity(os.getpid()) if ident.get('status') == 'UNKNOWN': self.skipTest('Linux proc identity required') ident['proc_start_ticks'] += 1 self.assertEqual(execution.process_status(ident), 'DEAD') def test_restart_does_not_reset_deadline_or_replay(self): state = self.ex.snapshot() duplicate = execution.Execution(self.rd) with self.assertRaises(execution.ExecutionStopped): duplicate.start(policy()) self.assertEqual(duplicate.snapshot()['deadline_monotonic'], state['deadline_monotonic']) def test_completed_requires_verified_cleanup(self): result = self.ex.finish(0, protocol_complete=True, cleanup_state='UNKNOWN') self.assertEqual(result['cleanup_state'], 'QUARANTINED') self.assertNotEqual(result['execution_outcome'], 'COMPLETED') def test_cancel_serializes_with_spawn_admission(self): started, committed = threading.Event(), threading.Event() def cancel(): started.set() self.ex.cancel('race fixture') committed.set() with self.ex.admission(): thread = threading.Thread(target=cancel) thread.start(); started.wait(1) self.assertFalse(committed.wait(.03)) thread.join(1) self.assertTrue(committed.is_set()) with self.assertRaises(execution.ExecutionStopped): with self.ex.admission(): self.fail('post-cancel spawn admitted') def test_crashed_holder_lease_is_not_reallocated(self): with patch.object(scheduler.paths, 'STATE', self.rd / 'state'), self.ex.scope('05-compile'): lease, _ = scheduler._try_slots('cpu', [0]) self.assertIsNotNone(lease) fcntl.flock(lease, fcntl.LOCK_UN); lease.close() # Simulate kernel lock release after a crash. next_lease, _ = scheduler._try_slots('cpu', [0]) self.assertIsNone(next_lease) self.assertTrue((self.rd / 'state/locks/cpu-0.lease.json').exists()) def test_unknown_telemetry_does_not_disable_deadline(self): with patch.object(watchdog, '_proc_tree_stat', return_value=None): before = time.monotonic() result = watchdog.run([sys.executable, '-c', 'import time; time.sleep(60)'], cwd=self.rd, stdout=self.rd/'out', stderr=self.rd/'err', execution=self.ex, poll_s=.01, hard_ceiling_s=.1) self.assertLess(time.monotonic() - before, 3) self.assertIn('DEADLINE', result.stalled_reason) self.assertEqual(result.cleanup_state, 'VERIFIED') def test_expired_start_bookkeeping_never_releases_command_gate(self): marker = self.rd / 'must-not-start' result = watchdog.run([sys.executable, '-c', "from pathlib import Path;Path('must-not-start').touch()"], cwd=self.rd, stdout=self.rd/'gate.out', stderr=self.rd/'gate.err', execution=self.ex, poll_s=.01, hard_ceiling_s=.1, on_start=lambda _: time.sleep(.2)) self.assertIn('DEADLINE', result.stalled_reason) self.assertEqual(result.cleanup_state, 'VERIFIED') self.assertFalse(marker.exists()) self.assertLess(result.duration_s, 3) def test_predeadline_channel_loss_remains_infrastructure_error(self): def stop_supervisor(proc): execution.signal_process(proc.supervisor_identity, signal.SIGTERM) proc.supervisor.wait(timeout=3) result = watchdog.run([sys.executable, '-c', "from pathlib import Path;Path('must-not-start').touch()"], cwd=self.rd, stdout=self.rd/'closed.out', stderr=self.rd/'closed.err', execution=self.ex, poll_s=.01, hard_ceiling_s=5, on_start=stop_supervisor) self.assertIn('runner monitor failed:', result.stalled_reason) self.assertNotIn('DEADLINE', result.stalled_reason) self.assertEqual(result.cleanup_state, 'VERIFIED') self.assertFalse((self.rd/'must-not-start').exists()) def test_suspect_is_advisory_not_an_idle_kill(self): with patch.object(watchdog, '_proc_tree_stat', return_value=(100, 0)), \ patch.object(watchdog, '_io_bytes', return_value=0): result = watchdog.run([sys.executable, '-c', 'import time; time.sleep(.25)'], cwd=self.rd, stdout=self.rd/'out', stderr=self.rd/'err', execution=self.ex, poll_s=.01) self.assertEqual(result.state, 'OK') self.assertTrue(result.suspect) def worker_fixture(self): from harness.stageworker import Worker rd = self.rd / 'guard-smoke' / 'rollout' (rd / 'handles/h1').mkdir(parents=True) (rd / 'workspace').mkdir() owner = execution.Execution(rd); owner.start(policy()) request = {'stage': 'noop', 'submitted_at': 1} (rd / 'handles/h1/request.json').write_text(json.dumps(request)) return rd, owner, Worker(rd, test_noop=True, execution=owner) def test_worker_restart_reuses_receipt_without_dispatch(self): from harness.stageworker import Worker rd, owner, worker = self.worker_fixture() worker.scan() operation = rd / 'stages/_queue/h1/operation.json' before = operation.read_bytes() Worker(rd, test_noop=True, execution=owner).scan() self.assertEqual(operation.read_bytes(), before) self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['result'], 'OK') def test_worker_payload_conflict_preserves_original_receipt(self): from harness.stageworker import Worker rd, owner, worker = self.worker_fixture() worker.scan() receipt = rd / 'stages/_queue/h1/status.json' before = receipt.read_bytes() (rd / 'handles/h1/request.json').write_text('{"stage":"noop","submitted_at":2}') Worker(rd, test_noop=True, execution=owner).scan() self.assertEqual(receipt.read_bytes(), before) self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['error_class'], 'OPERATION_PAYLOAD_CONFLICT') def test_unknown_operation_is_quarantined_not_replayed(self): from harness.stageworker import Worker rd, owner, worker = self.worker_fixture() worker.scan() (rd/'stages/_queue/h1/status.json').unlink() Worker(rd, test_noop=True, execution=owner).scan() self.assertEqual(owner.snapshot()['cleanup_state'], 'QUARANTINED') self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['error_class'], 'SIDE_EFFECT_UNKNOWN') def checkpoint(self): path = self.rd / 'session.jsonl' path.write_text('{"message_id":"m1"}\n') return {'complete': True, 'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()} def test_pause_resume_preserves_identity_returns_paused_time_and_fences_previous_epoch(self): before = self.ex.snapshot() stale = execution.Execution(self.rd).attach() self.ex.pause('QUOTA_WAIT_BUDGET_EXHAUSTED', self.checkpoint()) with self.assertRaises(execution.ExecutionStopped): with self.ex.admission(): self.fail('paused execution dispatched') paused_at = self.ex.snapshot()['paused_at'] with patch.object(execution, 'marker_processes', return_value=[]), \ patch.object(execution.time, 'time', return_value=paused_at + 4): state = self.ex.resume() self.assertEqual(state['execution_id'], before['execution_id']) self.assertEqual(state['deadline_monotonic'], before['deadline_monotonic'] + 4) # pause handed back self.assertEqual((state['paused_s'], state.get('paused_at')), (4, None)) self.assertEqual(state['epoch'], before['epoch'] + 1) self.assertEqual(state['auto_resume_count'], 1) with self.assertRaises(execution.ExecutionStopped): with stale.admission(): self.fail('old epoch survived takeover') def test_long_pause_outlives_original_deadline_but_credit_is_capped(self): self.ex.pause('CONNECT_FAILED', self.checkpoint()) paused_at = self.ex.snapshot()['paused_at'] with patch.object(execution.time, 'time', return_value=paused_at + 1000), \ patch.object(execution.time, 'monotonic', return_value=self.ex.snapshot()['deadline_monotonic'] + 5), \ patch.object(execution, 'marker_processes', return_value=[]): self.assertNotEqual(self.ex.stop_reason(), 'EXECUTION_DEADLINE_EXCEEDED') state = self.ex.resume(counted=False) self.assertEqual(state['paused_s'], 10) # at most one execution budget (policy: 10 s) def test_incomplete_or_changed_checkpoint_cannot_resume(self): with self.assertRaisesRegex(execution.ExecutionStopped, 'INCOMPLETE'): self.ex.pause('quota', {}) checkpoint = self.checkpoint() self.ex.pause('quota', checkpoint) Path(checkpoint['path']).write_text('changed') with self.assertRaisesRegex(execution.ExecutionStopped, 'HASH_MISMATCH'): self.ex.resume() def test_auto_resume_budget_survives_new_controller_objects(self): checkpoint = self.checkpoint() with patch.object(execution, 'marker_processes', return_value=[]): for _ in range(3): self.ex.pause('quota', checkpoint) self.ex = execution.Execution(self.rd).attach() self.ex.resume() self.ex.pause('quota', checkpoint) with self.assertRaisesRegex(execution.ExecutionStopped, 'AUTO_RESUME_BUDGET_EXHAUSTED'): execution.Execution(self.rd).resume() free = execution.Execution(self.rd) free.resume(counted=False) # a pause that sent nothing (connect/DNS failure) is free self.assertEqual(free.snapshot()['auto_resume_count'], 3) @unittest.skipUnless(Path('/proc').is_dir(), 'Actual worker requires Linux owner identity') def test_worker_executes_real_protected_stage_and_reuses_receipt(self): from harness import stageworker, stagerun import yaml root = self.rd / 'fixture-project' root.mkdir() source = Path(__file__).resolve().parents[1] (root/'harness').symlink_to(source/'harness', target_is_directory=True) (root/'registry').symlink_to(source/'registry', target_is_directory=True) cfg = {'paths': {'harness_python': sys.executable}, 'pools': {'cpu': {'workers': 1}}} (root/'config.yaml').write_text(yaml.safe_dump(cfg)) weights = root/'cache/weights/fixture-sha' weights.mkdir(parents=True) (weights.parent/'index.json').write_text('{"fixture":{"sha256":"fixture-sha"}}') rd = root/'runs/r/j/rollouts/00' (rd/'handles/h1').mkdir(parents=True) (rd/'workspace').mkdir() (rd.parent.parent/'job.json').write_text(json.dumps({'run_id':'r', 'job_id':'j', 'model':'fixture', 'task':'cls', 'march':'nash-p', 'backend':'ptq'})) (rd/'handles/h1/request.json').write_text('{"stage":"01-fetch"}') owner = execution.Execution(rd) fixture_policy = policy(); fixture_policy['phases']['01-fetch']['queue_deadline_s'] = 2 owner.start(fixture_policy) connect = stagerun.db.connect with patch.object(stagerun.db, 'connect', lambda **kw: connect(root/'state/jobs.sqlite', **kw)), \ patch.object(stagerun.paths, 'BENCH_ROOT', root), \ patch.object(stagerun.paths, 'config', return_value=cfg), \ patch.dict(os.environ, {'BENCH_CONFIG': str(root/'config.yaml'), 'BENCH_STATE_DIR': str(root/'state')}): worker = stageworker.Worker(rd, execution=owner) worker.scan() status = json.loads((rd/'handles/h1/status.json').read_text()) self.assertEqual(status['result'], 'SKIPPED', status) self.assertEqual(owner.snapshot()['operations']['h1']['state'], 'DONE') # A later attempt may overwrite a settled attempt's outputs; a resumed worker must # neither replay nor re-prove it (10-13: re-proof quarantined 19 rollouts). with patch.object(stageworker.watchdog, 'run') as replay, \ patch.object(stageworker.Worker, '_reconcile_stage', side_effect=ValueError('completed artifact changed')): stageworker.Worker(rd, execution=owner).scan() replay.assert_not_called() self.assertIsNone(owner.snapshot().get('stop_reason')) self.assertFalse(list((root/'state/locks').glob('*.lease.json'))) def test_receipt_cannot_bind_reused_supervisor_pid(self): binding = dict(proof='waitpid:ECHILD', supervisor=dict(pid=123, proc_start_ticks=1)) receipt = self.rd/'receipt.json' execution.atomic_json(receipt, dict(binding=dict(binding, supervisor=dict(pid=123, proc_start_ticks=2)), cleanup_state='VERIFIED', all_children_reaped=True, owner_lost=False, exit_code=0)) self.assertIsNone(execution.scope_receipt(dict(binding=binding, receipt_path=str(receipt)))) def test_board_exception_preserves_lease_and_quarantines_before_release(self): from contextlib import nullcontext from types import SimpleNamespace from unittest.mock import Mock from harness import stagerun hd = self.rd/'stages/_queue/h1'; hd.mkdir(parents=True) (hd/'request.json').write_text('{"stage":"07-board"}') with self.ex.mutation() as state: operation = dict(operation_id='h1', payload_sha256='fixture', execution_id=state['execution_id'], epoch=state['epoch'], state='DISPATCHED') state['operations']['h1'] = operation self.ex._save(state, 'TEST_DISPATCHED') (hd/'operation.json').write_text(json.dumps(operation)) class Board: name, pool, level = '07-board', 'board', 'job' def run(self, ctx): raise RuntimeError('transport failed after dispatch') ctx = SimpleNamespace(job={'model':'m','march':'nash-p','job_id':'j'},extra={}, run_id='r',rollout=0,lane='normal',agent_config=None) with patch.object(stagerun, '_ctx_from_rollout', return_value=ctx), \ patch.dict(stagerun.BY_NAME, {'07-board':Board}), \ patch.object(stagerun.db, 'connect', side_effect=lambda:nullcontext(Mock())), \ patch.object(stagerun.db, 'stage_start'), \ patch.object(scheduler, 'board_slots', return_value={0:{'name':'fixture','march':'nash-p'}}), \ patch.object(scheduler.paths, 'STATE', self.rd/'state'), \ patch.object(scheduler.paths, 'config', return_value={'pools':{'board':{'reserve_for_agent':0}}}), \ self.ex.scope('07-board', 'h1'): with self.assertRaisesRegex(RuntimeError, 'transport'): stagerun._exec_impl(hd, rd=self.rd, protected=True) self.assertEqual(self.ex.snapshot()['cleanup_state'], 'QUARANTINED') self.assertTrue((self.rd/'state/locks/board-0.lease.json').exists()) self.assertTrue((self.rd/'state/locks/board-0.quarantine.json').exists()) def test_stale_owner_cannot_mutate_or_cleanup(self): old = execution.Execution(self.rd).attach() self.ex.cancel('fixture') state = self.ex.snapshot(); state['epoch'] += 1; self.ex.epoch = state['epoch'] execution.atomic_json(self.ex.path, state) before = self.ex.path.read_bytes() actions = [lambda:old.quarantine('late'), lambda:old.cleanup(), lambda:old.cancel('late'), lambda:old.activity('late'), lambda:old.finish(0, protocol_complete=True)] for action in actions: with self.assertRaisesRegex(execution.ExecutionStopped, 'OWNER_FENCED'): action() self.assertEqual(self.ex.path.read_bytes(), before) def test_unknown_owner_blocks_admission_but_deadline_still_expires(self): with patch.object(execution, 'process_status', return_value='UNKNOWN'): with self.assertRaisesRegex(execution.ExecutionStopped, 'OWNER_STATUS_UNKNOWN'): with self.ex.admission(): self.fail('unknown owner admitted') self.assertIsNone(self.ex.stop_reason()) state = self.ex.snapshot(); state['deadline_monotonic'] = time.monotonic() - 1 execution.atomic_json(self.ex.path, state) self.assertEqual(self.ex.stop_reason(), 'EXECUTION_DEADLINE_EXCEEDED') def test_phase_queue_and_execution_clocks_are_separate_and_supervised(self): owner = execution.Execution(self.rd/'phase') cfg = policy(); cfg['phases']['05-compile'].update(queue_deadline_s=2, execution_deadline_s=.1) owner.start(cfg) owner.begin_phase('h1', '05-compile') self.assertIsNone(owner.snapshot()['phases']['h1']['execution_deadline_monotonic']) time.sleep(.12) # More than the execution budget, still legitimately queued. self.assertIsNone(owner.stop_reason('h1')) owner.phase_running('h1') with owner.scope('05-compile', 'h1'): result = watchdog.run([sys.executable, '-c', 'import time;time.sleep(.4)'], cwd=self.rd, stdout=self.rd/'phase.out', stderr=self.rd/'phase.err', execution=owner, role='stage', env=dict(os.environ, BENCH_OPERATION_ID='h1'), poll_s=.02, hard_ceiling_s=5) self.assertIn('PHASE_EXECUTION_DEADLINE', result.stalled_reason) self.assertLess(result.duration_s, .35) self.assertIsNotNone(owner.snapshot()['phases']['h1']['execution_deadline_monotonic']) def test_typed_heartbeat_is_not_business_progress(self): path = self.rd/'events'; path.touch() tracker = watchdog.BusinessLog(path) path.write_text('{"type":"heartbeat","state":"RUNNING"}\n') self.assertEqual(tracker.sample(), 0) with path.open('a') as out: out.write('{"type":"message_end","message":{"role":"assistant","content":[]}}\n') self.assertGreater(tracker.sample(), 0) with path.open('a') as out: out.write('unclassified operational text\n') self.assertIsNone(tracker.sample()) def test_progress_health_is_persisted_and_unknown_not_zero(self): signals = dict(cpu_ticks=1, io_bytes=0, business_log_bytes=0, stage_sequence=0) self.ex.record_progress('p', signals) time.sleep(.12) self.ex.record_progress('p', signals) self.assertEqual(self.ex.snapshot()['health'], 'SUSPECT') self.ex.record_progress('p', dict(signals, io_bytes=None)) state = self.ex.snapshot() self.assertEqual(state['health'], 'UNKNOWN') self.assertIsNone(state['progress']['signals']['io_bytes']) def test_submission_replay_retains_operation_identity(self): from harness.stageworker import Worker rd, owner, worker = self.worker_fixture() source = rd/'stages/_queue/source'; source.mkdir(parents=True) (source/'recipe.json').write_text('{}') (source/'request.json').write_text(json.dumps({'recipe':str(source/'recipe.json')})) (rd/'handles/h1/request.json').write_text('{"stage":"noop","submission":"source"}') worker.scan() record = json.loads((rd/'stages/_queue/h1/operation.json').read_text()) self.assertIn('payload_sha256', record) self.assertEqual(record['execution_id'], owner.snapshot()['execution_id']) Worker(rd, test_noop=True, execution=owner).scan() self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['result'], 'OK') if __name__ == '__main__': unittest.main()