data-gen-bpu/scripts/test_execution.py
bench ec1a58edf8 续跑不再重新核对已结算的步骤;领题后不再重复查额度
续跑时 stageworker 会重新核对旧步骤回执。同一轨迹每次编译写同一个 .hbm,
旧回执核对失败后被当成状态不明,整条轨迹隔离;10-12/13 共 19 条编译过多次的轨迹,
续跑几秒内就判 INFRA_ERROR。现在已结算的操作直接沿用旧结果,判断条件与
reconcile_completed 相同。
collector 领题前已经查过额度,run_rollout 领到后又查一次(只等 0.5 秒);
机器忙时读超时,题被判 UNVERIFIED,一步都没跑(59 条)。领到后不再重复查。
这两处代码都在调度程序进程里运行,重启后才生效。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 11:15:05 +08:00

525 lines
29 KiB
Python

"""Lifecycle contracts using only temporary state and owned test processes."""
import os
import signal
import subprocess
import sys
import tempfile
import threading
import json
import hashlib
import fcntl
import time
import unittest
from pathlib import Path
from unittest.mock import patch
from harness import execution, process_reaper, scheduler, watchdog
def policy():
base = dict(queue_deadline_s=.15, execution_deadline_s=10, suspect_after_s=.1,
cancel_grace_s=.1, cleanup_deadline_s=2)
return dict(base, version='test-v1', phases={name: dict(base) for name in execution.PHASES})
class ProcessStatBytesTests(unittest.TestCase):
@staticmethod
def stat(comm=b'worker', *, birth=42, state=b'S', ppid=10):
fields = [state, str(ppid).encode(), b'10', b'10'] + [b'0'] * 16
fields[11:13] = [b'13', b'17']
fields[19] = str(birth).encode()
return b'123 (' + comm + b') ' + b' '.join(fields) + b'\n'
def test_identity_ignores_arbitrary_comm_bytes(self):
for comm in (b'worker', b'a (b) c', b'\xff', '中文线程名截断'.encode()[:15]):
with self.subTest(comm=comm), patch.object(Path, 'read_bytes', return_value=self.stat(comm)):
identity = execution.process_identity(123)
self.assertEqual(identity['status'], 'KNOWN')
self.assertEqual((identity['proc_start_ticks'], identity['ppid'], identity['state'],
identity['cpu_ticks']), (42, 10, 'S', 30))
json.dumps(identity) # Receipts keep their existing text/number schema.
def test_corrupt_identity_is_unknown_and_never_authorizes_a_signal(self):
original = dict(host=execution.socket.gethostname(), boot_id='fixture', pid=123,
proc_start_ticks=42)
for raw in (b'', b'123 no delimiter', self.stat(state=b'\xff'),
self.stat().replace(b'42\n', b'bad\n')):
with self.subTest(raw=raw), patch.object(Path, 'read_bytes', return_value=raw), \
patch.object(Path, 'exists', return_value=True), \
patch.object(execution, 'boot_id', return_value='fixture'), \
patch.object(os, 'pidfd_open', create=True) as opened:
self.assertEqual(execution.process_identity(123)['status'], 'UNKNOWN')
self.assertEqual(execution.process_status(original), 'UNKNOWN')
self.assertFalse(execution.signal_process(original, signal.SIGTERM))
opened.assert_not_called()
def test_reaper_preserves_second_birth_and_parent_check(self):
reaper = process_reaper.Reaper.__new__(process_reaper.Reaper)
reaper.pid, reaper.host, reaper.boot_id, reaper.members = 10, 'fixture', 'fixture', {}
first = self.stat(b'\xff name (nested)')
for second, permitted in ((first, True), (self.stat(birth=43), False),
(self.stat(ppid=11), False)):
with self.subTest(permitted=permitted), \
patch.object(process_reaper, 'children', return_value={123}), \
patch.object(Path, 'read_bytes', side_effect=[first, second]), \
patch.object(os, 'pidfd_open', return_value=99, create=True), \
patch.object(os, 'close') as closed, \
patch.object(signal, 'pidfd_send_signal', create=True) as sent:
reaper.signal(signal.SIGTERM)
self.assertEqual(sent.call_count, int(permitted))
closed.assert_called_once_with(99)
self.assertEqual(reaper.members[(123, 42)]['state'], 'S')
@unittest.skipUnless(sys.platform == 'linux', 'Linux comm and subreaper')
def test_non_utf8_child_name_is_still_identified_and_reaped(self):
with tempfile.TemporaryDirectory() as name:
root = Path(name)
owner = execution.Execution(root); owner.start(policy())
code = ("import ctypes,time;from pathlib import Path;"
"assert ctypes.CDLL(None).prctl(15,b'worker-\\xff',0,0,0)==0;"
"Path('armed').touch();time.sleep(30)")
result = watchdog.run([sys.executable, '-c', code], cwd=root,
stdout=root/'out', stderr=root/'err', execution=owner, poll_s=.01, hard_ceiling_s=3,
stop_check=lambda: 'fixture complete' if (root/'armed').exists() else None)
self.assertEqual(result.stalled_reason, 'fixture complete')
self.assertEqual(result.cleanup_state, 'VERIFIED')
for identity in owner.snapshot()['processes']:
self.assertNotEqual(execution.process_status(identity), 'ALIVE')
class ExecutionTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.rd = Path(self.tmp.name)
self.ex = execution.Execution(self.rd)
self.ex.start(policy())
if not Path('/proc').is_dir():
# Production admission requires Linux identity; mock only this known
# test controller for portable pure state-machine checks.
original = execution.process_status
owner = execution.process_key(self.ex.snapshot()['owner'])
override = patch.object(execution, 'process_status', side_effect=lambda identity:
'ALIVE' if execution.process_key(identity) == owner else original(identity))
override.start(); self.addCleanup(override.stop)
def test_cancel_fences_spawn_and_cannot_be_completed(self):
self.ex.cancel('test cancel')
with self.assertRaises(execution.ExecutionStopped):
with self.ex.admission():
self.fail('cancelled admission opened')
result = self.ex.finish(0, protocol_complete=True)
self.assertEqual(result['execution_outcome'], 'CANCELLED')
self.assertEqual(result['cleanup_state'], 'VERIFIED')
@unittest.skipUnless(sys.platform == 'linux', 'real owned process cleanup')
def test_worker_drain_uses_stage_deadline_not_cleanup_budget(self):
from harness.stageworker import Worker
owner = execution.Execution(self.rd / 'drain')
cfg = policy(); cfg.update(cleanup_deadline_s=.05, cancel_grace_s=.01)
owner.start(cfg)
worker = Worker(owner.rd, execution=owner)
started, results = threading.Event(), []
def stage():
with owner.scope('05-compile', 'h1'):
owner.begin_phase('h1', '05-compile'); owner.phase_running('h1')
results.append(watchdog.run([sys.executable, '-c', 'import time;time.sleep(.3)'],
cwd=owner.rd, stdout=owner.rd/'out', stderr=owner.rd/'err', execution=owner,
role='stage', poll_s=.01, on_start=lambda _: started.set()))
worker.thread = threading.Thread(target=stage)
with worker:
self.assertTrue(started.wait(3))
self.assertEqual(results[0].state, 'OK')
self.assertGreater(results[0].duration_s, cfg['cleanup_deadline_s'])
self.assertFalse(owner.snapshot()['cancel_requested'])
self.assertEqual(results[0].cleanup_state, 'VERIFIED')
@unittest.skipUnless(sys.platform == 'linux', 'real owned process cleanup')
def test_worker_drain_keeps_explicit_cancel_and_exception_cleanup(self):
from harness.stageworker import Worker
for explicit in (True, False):
with self.subTest(explicit=explicit):
owner = execution.Execution(self.rd / str(explicit)); owner.start(policy())
worker = Worker(owner.rd, execution=owner)
started, results = threading.Event(), []
def stage():
with owner.scope('05-compile', 'h1'):
owner.begin_phase('h1', '05-compile'); owner.phase_running('h1')
results.append(watchdog.run([sys.executable, '-c', 'import time;time.sleep(20)'],
cwd=owner.rd, stdout=owner.rd/'out', stderr=owner.rd/'err', execution=owner,
role='stage', poll_s=.01, on_start=lambda _: started.set()))
worker.thread = threading.Thread(target=stage)
try:
with worker:
self.assertTrue(started.wait(3))
if explicit:
owner.cancel('USER_STOP')
else:
raise ValueError('fixture context failed')
except ValueError as error:
self.assertFalse(explicit)
self.assertEqual(str(error), 'fixture context failed')
self.assertEqual(owner.snapshot()['cancel_requested']['reason'],
'USER_STOP' if explicit else 'WORKER_CONTEXT_ERROR')
self.assertEqual(results[0].cleanup_state, 'VERIFIED')
self.assertEqual(execution.owned_processes(owner.snapshot()), [])
def test_exit_and_protocol_are_independent(self):
self.assertEqual(self.ex.finish(143, protocol_complete=True)['execution_outcome'], 'AGENT_ERROR')
other = execution.Execution(self.rd / 'other'); other.start(policy())
self.assertEqual(other.finish(0, protocol_complete=False)['execution_outcome'], 'AGENT_ERROR')
def test_late_cancel_does_not_overwrite_completion(self):
self.ex.finish(0, protocol_complete=True)
self.assertEqual(self.ex.cancel('late')['reason'], 'ALREADY_TERMINAL')
self.assertEqual(self.ex.snapshot()['execution_outcome'], 'COMPLETED')
def test_old_epoch_is_fenced(self):
stale = execution.Execution(self.rd)
stale.epoch = self.ex.epoch - 1
with self.assertRaises(execution.ExecutionStopped):
with stale.admission():
self.fail('old owner admitted')
def test_queue_wait_is_bounded(self):
with patch.object(scheduler, '_try_slots', return_value=(None, None)), \
patch.object(scheduler, 'pool_size', return_value=1), \
patch.object(scheduler, 'pool_reserve', return_value=0), self.ex.scope('05-compile'):
before = time.monotonic()
with self.assertRaises(execution.ExecutionStopped):
with scheduler.acquire('cpu', poll_s=.01):
self.fail('unavailable slot acquired')
self.assertLess(time.monotonic() - before, 1)
def test_pid_reuse_is_not_alive(self):
ident = execution.process_identity(os.getpid())
if ident.get('status') == 'UNKNOWN':
self.skipTest('Linux proc identity required')
ident['proc_start_ticks'] += 1
self.assertEqual(execution.process_status(ident), 'DEAD')
def test_restart_does_not_reset_deadline_or_replay(self):
state = self.ex.snapshot()
duplicate = execution.Execution(self.rd)
with self.assertRaises(execution.ExecutionStopped):
duplicate.start(policy())
self.assertEqual(duplicate.snapshot()['deadline_monotonic'], state['deadline_monotonic'])
def test_completed_requires_verified_cleanup(self):
result = self.ex.finish(0, protocol_complete=True, cleanup_state='UNKNOWN')
self.assertEqual(result['cleanup_state'], 'QUARANTINED')
self.assertNotEqual(result['execution_outcome'], 'COMPLETED')
def test_cancel_serializes_with_spawn_admission(self):
started, committed = threading.Event(), threading.Event()
def cancel():
started.set()
self.ex.cancel('race fixture')
committed.set()
with self.ex.admission():
thread = threading.Thread(target=cancel)
thread.start(); started.wait(1)
self.assertFalse(committed.wait(.03))
thread.join(1)
self.assertTrue(committed.is_set())
with self.assertRaises(execution.ExecutionStopped):
with self.ex.admission():
self.fail('post-cancel spawn admitted')
def test_crashed_holder_lease_is_not_reallocated(self):
with patch.object(scheduler.paths, 'STATE', self.rd / 'state'), self.ex.scope('05-compile'):
lease, _ = scheduler._try_slots('cpu', [0])
self.assertIsNotNone(lease)
fcntl.flock(lease, fcntl.LOCK_UN); lease.close() # Simulate kernel lock release after a crash.
next_lease, _ = scheduler._try_slots('cpu', [0])
self.assertIsNone(next_lease)
self.assertTrue((self.rd / 'state/locks/cpu-0.lease.json').exists())
def test_unknown_telemetry_does_not_disable_deadline(self):
with patch.object(watchdog, '_proc_tree_stat', return_value=None):
before = time.monotonic()
result = watchdog.run([sys.executable, '-c', 'import time; time.sleep(60)'], cwd=self.rd,
stdout=self.rd/'out', stderr=self.rd/'err', execution=self.ex, poll_s=.01, hard_ceiling_s=.1)
self.assertLess(time.monotonic() - before, 3)
self.assertIn('DEADLINE', result.stalled_reason)
self.assertEqual(result.cleanup_state, 'VERIFIED')
def test_expired_start_bookkeeping_never_releases_command_gate(self):
marker = self.rd / 'must-not-start'
result = watchdog.run([sys.executable, '-c',
"from pathlib import Path;Path('must-not-start').touch()"], cwd=self.rd,
stdout=self.rd/'gate.out', stderr=self.rd/'gate.err', execution=self.ex,
poll_s=.01, hard_ceiling_s=.1, on_start=lambda _: time.sleep(.2))
self.assertIn('DEADLINE', result.stalled_reason)
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertFalse(marker.exists())
self.assertLess(result.duration_s, 3)
def test_predeadline_channel_loss_remains_infrastructure_error(self):
def stop_supervisor(proc):
execution.signal_process(proc.supervisor_identity, signal.SIGTERM)
proc.supervisor.wait(timeout=3)
result = watchdog.run([sys.executable, '-c',
"from pathlib import Path;Path('must-not-start').touch()"], cwd=self.rd,
stdout=self.rd/'closed.out', stderr=self.rd/'closed.err', execution=self.ex,
poll_s=.01, hard_ceiling_s=5, on_start=stop_supervisor)
self.assertIn('runner monitor failed:', result.stalled_reason)
self.assertNotIn('DEADLINE', result.stalled_reason)
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertFalse((self.rd/'must-not-start').exists())
def test_suspect_is_advisory_not_an_idle_kill(self):
with patch.object(watchdog, '_proc_tree_stat', return_value=(100, 0)), \
patch.object(watchdog, '_io_bytes', return_value=0):
result = watchdog.run([sys.executable, '-c', 'import time; time.sleep(.25)'], cwd=self.rd,
stdout=self.rd/'out', stderr=self.rd/'err', execution=self.ex, poll_s=.01)
self.assertEqual(result.state, 'OK')
self.assertTrue(result.suspect)
def worker_fixture(self):
from harness.stageworker import Worker
rd = self.rd / 'guard-smoke' / 'rollout'
(rd / 'handles/h1').mkdir(parents=True)
(rd / 'workspace').mkdir()
owner = execution.Execution(rd); owner.start(policy())
request = {'stage': 'noop', 'submitted_at': 1}
(rd / 'handles/h1/request.json').write_text(json.dumps(request))
return rd, owner, Worker(rd, test_noop=True, execution=owner)
def test_worker_restart_reuses_receipt_without_dispatch(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
worker.scan()
operation = rd / 'stages/_queue/h1/operation.json'
before = operation.read_bytes()
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(operation.read_bytes(), before)
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['result'], 'OK')
def test_worker_payload_conflict_preserves_original_receipt(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
worker.scan()
receipt = rd / 'stages/_queue/h1/status.json'
before = receipt.read_bytes()
(rd / 'handles/h1/request.json').write_text('{"stage":"noop","submitted_at":2}')
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(receipt.read_bytes(), before)
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['error_class'],
'OPERATION_PAYLOAD_CONFLICT')
def test_unknown_operation_is_quarantined_not_replayed(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
worker.scan()
(rd/'stages/_queue/h1/status.json').unlink()
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(owner.snapshot()['cleanup_state'], 'QUARANTINED')
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['error_class'],
'SIDE_EFFECT_UNKNOWN')
def checkpoint(self):
path = self.rd / 'session.jsonl'
path.write_text('{"message_id":"m1"}\n')
return {'complete': True, 'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}
def test_pause_resume_preserves_identity_deadline_and_fences_previous_epoch(self):
before = self.ex.snapshot()
stale = execution.Execution(self.rd).attach()
self.ex.pause('QUOTA_WAIT_BUDGET_EXHAUSTED', self.checkpoint())
with self.assertRaises(execution.ExecutionStopped):
with self.ex.admission():
self.fail('paused execution dispatched')
with patch.object(execution, 'marker_processes', return_value=[]):
state = self.ex.resume()
self.assertEqual(state['execution_id'], before['execution_id'])
self.assertEqual(state['deadline_monotonic'], before['deadline_monotonic'])
self.assertEqual(state['epoch'], before['epoch'] + 1)
self.assertEqual(state['auto_resume_count'], 1)
with self.assertRaises(execution.ExecutionStopped):
with stale.admission():
self.fail('old epoch survived takeover')
def test_incomplete_or_changed_checkpoint_cannot_resume(self):
with self.assertRaisesRegex(execution.ExecutionStopped, 'INCOMPLETE'):
self.ex.pause('quota', {})
checkpoint = self.checkpoint()
self.ex.pause('quota', checkpoint)
Path(checkpoint['path']).write_text('changed')
with self.assertRaisesRegex(execution.ExecutionStopped, 'HASH_MISMATCH'):
self.ex.resume()
def test_auto_resume_budget_survives_new_controller_objects(self):
checkpoint = self.checkpoint()
with patch.object(execution, 'marker_processes', return_value=[]):
for _ in range(3):
self.ex.pause('quota', checkpoint)
self.ex = execution.Execution(self.rd).attach()
self.ex.resume()
self.ex.pause('quota', checkpoint)
with self.assertRaisesRegex(execution.ExecutionStopped, 'AUTO_RESUME_BUDGET_EXHAUSTED'):
execution.Execution(self.rd).resume()
@unittest.skipUnless(Path('/proc').is_dir(), 'Actual worker requires Linux owner identity')
def test_worker_executes_real_protected_stage_and_reuses_receipt(self):
from harness import stageworker, stagerun
import yaml
root = self.rd / 'fixture-project'
root.mkdir()
source = Path(__file__).resolve().parents[1]
(root/'harness').symlink_to(source/'harness', target_is_directory=True)
(root/'registry').symlink_to(source/'registry', target_is_directory=True)
cfg = {'paths': {'harness_python': sys.executable}, 'pools': {'cpu': {'workers': 1}}}
(root/'config.yaml').write_text(yaml.safe_dump(cfg))
weights = root/'cache/weights/fixture-sha'
weights.mkdir(parents=True)
(weights.parent/'index.json').write_text('{"fixture":{"sha256":"fixture-sha"}}')
rd = root/'runs/r/j/rollouts/00'
(rd/'handles/h1').mkdir(parents=True)
(rd/'workspace').mkdir()
(rd.parent.parent/'job.json').write_text(json.dumps({'run_id':'r', 'job_id':'j',
'model':'fixture', 'task':'cls', 'march':'nash-p', 'backend':'ptq'}))
(rd/'handles/h1/request.json').write_text('{"stage":"01-fetch"}')
owner = execution.Execution(rd)
fixture_policy = policy(); fixture_policy['phases']['01-fetch']['queue_deadline_s'] = 2
owner.start(fixture_policy)
connect = stagerun.db.connect
with patch.object(stagerun.db, 'connect', lambda **kw: connect(root/'state/jobs.sqlite', **kw)), \
patch.object(stagerun.paths, 'BENCH_ROOT', root), \
patch.object(stagerun.paths, 'config', return_value=cfg), \
patch.dict(os.environ, {'BENCH_CONFIG': str(root/'config.yaml'), 'BENCH_STATE_DIR': str(root/'state')}):
worker = stageworker.Worker(rd, execution=owner)
worker.scan()
status = json.loads((rd/'handles/h1/status.json').read_text())
self.assertEqual(status['result'], 'SKIPPED', status)
self.assertEqual(owner.snapshot()['operations']['h1']['state'], 'DONE')
# A later attempt may overwrite a settled attempt's outputs; a resumed worker must
# neither replay nor re-prove it (10-13: re-proof quarantined 19 rollouts).
with patch.object(stageworker.watchdog, 'run') as replay, \
patch.object(stageworker.Worker, '_reconcile_stage', side_effect=ValueError('completed artifact changed')):
stageworker.Worker(rd, execution=owner).scan()
replay.assert_not_called()
self.assertIsNone(owner.snapshot().get('stop_reason'))
self.assertFalse(list((root/'state/locks').glob('*.lease.json')))
def test_receipt_cannot_bind_reused_supervisor_pid(self):
binding = dict(proof='waitpid:ECHILD', supervisor=dict(pid=123, proc_start_ticks=1))
receipt = self.rd/'receipt.json'
execution.atomic_json(receipt, dict(binding=dict(binding, supervisor=dict(pid=123, proc_start_ticks=2)),
cleanup_state='VERIFIED', all_children_reaped=True, owner_lost=False, exit_code=0))
self.assertIsNone(execution.scope_receipt(dict(binding=binding, receipt_path=str(receipt))))
def test_board_exception_preserves_lease_and_quarantines_before_release(self):
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock
from harness import stagerun
hd = self.rd/'stages/_queue/h1'; hd.mkdir(parents=True)
(hd/'request.json').write_text('{"stage":"07-board"}')
with self.ex.mutation() as state:
operation = dict(operation_id='h1', payload_sha256='fixture', execution_id=state['execution_id'],
epoch=state['epoch'], state='DISPATCHED')
state['operations']['h1'] = operation
self.ex._save(state, 'TEST_DISPATCHED')
(hd/'operation.json').write_text(json.dumps(operation))
class Board:
name, pool, level = '07-board', 'board', 'job'
def run(self, ctx):
raise RuntimeError('transport failed after dispatch')
ctx = SimpleNamespace(job={'model':'m','march':'nash-p','job_id':'j'},extra={},
run_id='r',rollout=0,lane='normal',agent_config=None)
with patch.object(stagerun, '_ctx_from_rollout', return_value=ctx), \
patch.dict(stagerun.BY_NAME, {'07-board':Board}), \
patch.object(stagerun.db, 'connect', side_effect=lambda:nullcontext(Mock())), \
patch.object(stagerun.db, 'stage_start'), \
patch.object(scheduler, 'board_slots', return_value={0:{'name':'fixture','march':'nash-p'}}), \
patch.object(scheduler.paths, 'STATE', self.rd/'state'), \
patch.object(scheduler.paths, 'config', return_value={'pools':{'board':{'reserve_for_agent':0}}}), \
self.ex.scope('07-board', 'h1'):
with self.assertRaisesRegex(RuntimeError, 'transport'):
stagerun._exec_impl(hd, rd=self.rd, protected=True)
self.assertEqual(self.ex.snapshot()['cleanup_state'], 'QUARANTINED')
self.assertTrue((self.rd/'state/locks/board-0.lease.json').exists())
self.assertTrue((self.rd/'state/locks/board-0.quarantine.json').exists())
def test_stale_owner_cannot_mutate_or_cleanup(self):
old = execution.Execution(self.rd).attach()
self.ex.cancel('fixture')
state = self.ex.snapshot(); state['epoch'] += 1; self.ex.epoch = state['epoch']
execution.atomic_json(self.ex.path, state)
before = self.ex.path.read_bytes()
actions = [lambda:old.quarantine('late'), lambda:old.cleanup(), lambda:old.cancel('late'),
lambda:old.activity('late'), lambda:old.finish(0, protocol_complete=True)]
for action in actions:
with self.assertRaisesRegex(execution.ExecutionStopped, 'OWNER_FENCED'):
action()
self.assertEqual(self.ex.path.read_bytes(), before)
def test_unknown_owner_blocks_admission_but_deadline_still_expires(self):
with patch.object(execution, 'process_status', return_value='UNKNOWN'):
with self.assertRaisesRegex(execution.ExecutionStopped, 'OWNER_STATUS_UNKNOWN'):
with self.ex.admission():
self.fail('unknown owner admitted')
self.assertIsNone(self.ex.stop_reason())
state = self.ex.snapshot(); state['deadline_monotonic'] = time.monotonic() - 1
execution.atomic_json(self.ex.path, state)
self.assertEqual(self.ex.stop_reason(), 'EXECUTION_DEADLINE_EXCEEDED')
def test_phase_queue_and_execution_clocks_are_separate_and_supervised(self):
owner = execution.Execution(self.rd/'phase')
cfg = policy(); cfg['phases']['05-compile'].update(queue_deadline_s=2, execution_deadline_s=.1)
owner.start(cfg)
owner.begin_phase('h1', '05-compile')
self.assertIsNone(owner.snapshot()['phases']['h1']['execution_deadline_monotonic'])
time.sleep(.12) # More than the execution budget, still legitimately queued.
self.assertIsNone(owner.stop_reason('h1'))
owner.phase_running('h1')
with owner.scope('05-compile', 'h1'):
result = watchdog.run([sys.executable, '-c', 'import time;time.sleep(.4)'], cwd=self.rd,
stdout=self.rd/'phase.out', stderr=self.rd/'phase.err', execution=owner, role='stage',
env=dict(os.environ, BENCH_OPERATION_ID='h1'), poll_s=.02, hard_ceiling_s=5)
self.assertIn('PHASE_EXECUTION_DEADLINE', result.stalled_reason)
self.assertLess(result.duration_s, .35)
self.assertIsNotNone(owner.snapshot()['phases']['h1']['execution_deadline_monotonic'])
def test_typed_heartbeat_is_not_business_progress(self):
path = self.rd/'events'; path.touch()
tracker = watchdog.BusinessLog(path)
path.write_text('{"type":"heartbeat","state":"RUNNING"}\n')
self.assertEqual(tracker.sample(), 0)
with path.open('a') as out:
out.write('{"type":"message_end","message":{"role":"assistant","content":[]}}\n')
self.assertGreater(tracker.sample(), 0)
with path.open('a') as out:
out.write('unclassified operational text\n')
self.assertIsNone(tracker.sample())
def test_progress_health_is_persisted_and_unknown_not_zero(self):
signals = dict(cpu_ticks=1, io_bytes=0, business_log_bytes=0, stage_sequence=0)
self.ex.record_progress('p', signals)
time.sleep(.12)
self.ex.record_progress('p', signals)
self.assertEqual(self.ex.snapshot()['health'], 'SUSPECT')
self.ex.record_progress('p', dict(signals, io_bytes=None))
state = self.ex.snapshot()
self.assertEqual(state['health'], 'UNKNOWN')
self.assertIsNone(state['progress']['signals']['io_bytes'])
def test_submission_replay_retains_operation_identity(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
source = rd/'stages/_queue/source'; source.mkdir(parents=True)
(source/'recipe.json').write_text('{}')
(source/'request.json').write_text(json.dumps({'recipe':str(source/'recipe.json')}))
(rd/'handles/h1/request.json').write_text('{"stage":"noop","submission":"source"}')
worker.scan()
record = json.loads((rd/'stages/_queue/h1/operation.json').read_text())
self.assertIn('payload_sha256', record)
self.assertEqual(record['execution_id'], owner.snapshot()['execution_id'])
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['result'], 'OK')
if __name__ == '__main__':
unittest.main()