data-gen-bpu/scripts/test_execution.py
bench f83fa3948c 网络小故障不再拖垮整条轨迹:原地重发、没发出去的暂停不扣续跑次数、结果不明的请求可以续跑
GLM 请求失败率约 0.3%,但一次失败就让整条轨迹暂停;3 次续跑次数被网络故障用完后,
轨迹只能挂到 24 小时上限(10-14 有 4 条);回复中途断开的请求过了等待期会被自动放弃,
同时整条执行被取消(10-13/14 共 16 条)。

A. 连接失败、域名解析失败(RETRY_UNSENT,确定什么都没发出去)在 broker 里等一会儿原地重发。
   每次重试把等待时间加上一次连接时限计入排队预算 wait_spent,保持
   客户端超时 = 排队预算 + 次数×单次时限 的上界;预算用完才照旧暂停。
B. 续跑次数只在最近一次请求可能已经送达时才扣(看账本最近一次 attempt 是否 sent)。
   连接/解析失败、因为别的请求结果不明而在准入时被挡下的暂停,续跑不扣次数。
   执行记录和账本两处计数同步,调度程序不再按原始次数预先过滤。
C. reconcile 自动放弃时(截止时间再过一个 token 窗口)不再取消执行,而是把这一轮
   标成 REQUEST_OUTCOME_UNKNOWN、可以续跑:重发一轮只花一次请求的 token,并计入续跑次数。
   人工放弃(operator)仍然永久隔离。续跑资格检查不再被已放弃的 attempt 挡住。
都在调度程序进程里运行,下次重启生效。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 15:41:49 +08:00

528 lines
29 KiB
Python

"""Lifecycle contracts using only temporary state and owned test processes."""
import os
import signal
import subprocess
import sys
import tempfile
import threading
import json
import hashlib
import fcntl
import time
import unittest
from pathlib import Path
from unittest.mock import patch
from harness import execution, process_reaper, scheduler, watchdog
def policy():
base = dict(queue_deadline_s=.15, execution_deadline_s=10, suspect_after_s=.1,
cancel_grace_s=.1, cleanup_deadline_s=2)
return dict(base, version='test-v1', phases={name: dict(base) for name in execution.PHASES})
class ProcessStatBytesTests(unittest.TestCase):
@staticmethod
def stat(comm=b'worker', *, birth=42, state=b'S', ppid=10):
fields = [state, str(ppid).encode(), b'10', b'10'] + [b'0'] * 16
fields[11:13] = [b'13', b'17']
fields[19] = str(birth).encode()
return b'123 (' + comm + b') ' + b' '.join(fields) + b'\n'
def test_identity_ignores_arbitrary_comm_bytes(self):
for comm in (b'worker', b'a (b) c', b'\xff', '中文线程名截断'.encode()[:15]):
with self.subTest(comm=comm), patch.object(Path, 'read_bytes', return_value=self.stat(comm)):
identity = execution.process_identity(123)
self.assertEqual(identity['status'], 'KNOWN')
self.assertEqual((identity['proc_start_ticks'], identity['ppid'], identity['state'],
identity['cpu_ticks']), (42, 10, 'S', 30))
json.dumps(identity) # Receipts keep their existing text/number schema.
def test_corrupt_identity_is_unknown_and_never_authorizes_a_signal(self):
original = dict(host=execution.socket.gethostname(), boot_id='fixture', pid=123,
proc_start_ticks=42)
for raw in (b'', b'123 no delimiter', self.stat(state=b'\xff'),
self.stat().replace(b'42\n', b'bad\n')):
with self.subTest(raw=raw), patch.object(Path, 'read_bytes', return_value=raw), \
patch.object(Path, 'exists', return_value=True), \
patch.object(execution, 'boot_id', return_value='fixture'), \
patch.object(os, 'pidfd_open', create=True) as opened:
self.assertEqual(execution.process_identity(123)['status'], 'UNKNOWN')
self.assertEqual(execution.process_status(original), 'UNKNOWN')
self.assertFalse(execution.signal_process(original, signal.SIGTERM))
opened.assert_not_called()
def test_reaper_preserves_second_birth_and_parent_check(self):
reaper = process_reaper.Reaper.__new__(process_reaper.Reaper)
reaper.pid, reaper.host, reaper.boot_id, reaper.members = 10, 'fixture', 'fixture', {}
first = self.stat(b'\xff name (nested)')
for second, permitted in ((first, True), (self.stat(birth=43), False),
(self.stat(ppid=11), False)):
with self.subTest(permitted=permitted), \
patch.object(process_reaper, 'children', return_value={123}), \
patch.object(Path, 'read_bytes', side_effect=[first, second]), \
patch.object(os, 'pidfd_open', return_value=99, create=True), \
patch.object(os, 'close') as closed, \
patch.object(signal, 'pidfd_send_signal', create=True) as sent:
reaper.signal(signal.SIGTERM)
self.assertEqual(sent.call_count, int(permitted))
closed.assert_called_once_with(99)
self.assertEqual(reaper.members[(123, 42)]['state'], 'S')
@unittest.skipUnless(sys.platform == 'linux', 'Linux comm and subreaper')
def test_non_utf8_child_name_is_still_identified_and_reaped(self):
with tempfile.TemporaryDirectory() as name:
root = Path(name)
owner = execution.Execution(root); owner.start(policy())
code = ("import ctypes,time;from pathlib import Path;"
"assert ctypes.CDLL(None).prctl(15,b'worker-\\xff',0,0,0)==0;"
"Path('armed').touch();time.sleep(30)")
result = watchdog.run([sys.executable, '-c', code], cwd=root,
stdout=root/'out', stderr=root/'err', execution=owner, poll_s=.01, hard_ceiling_s=3,
stop_check=lambda: 'fixture complete' if (root/'armed').exists() else None)
self.assertEqual(result.stalled_reason, 'fixture complete')
self.assertEqual(result.cleanup_state, 'VERIFIED')
for identity in owner.snapshot()['processes']:
self.assertNotEqual(execution.process_status(identity), 'ALIVE')
class ExecutionTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.rd = Path(self.tmp.name)
self.ex = execution.Execution(self.rd)
self.ex.start(policy())
if not Path('/proc').is_dir():
# Production admission requires Linux identity; mock only this known
# test controller for portable pure state-machine checks.
original = execution.process_status
owner = execution.process_key(self.ex.snapshot()['owner'])
override = patch.object(execution, 'process_status', side_effect=lambda identity:
'ALIVE' if execution.process_key(identity) == owner else original(identity))
override.start(); self.addCleanup(override.stop)
def test_cancel_fences_spawn_and_cannot_be_completed(self):
self.ex.cancel('test cancel')
with self.assertRaises(execution.ExecutionStopped):
with self.ex.admission():
self.fail('cancelled admission opened')
result = self.ex.finish(0, protocol_complete=True)
self.assertEqual(result['execution_outcome'], 'CANCELLED')
self.assertEqual(result['cleanup_state'], 'VERIFIED')
@unittest.skipUnless(sys.platform == 'linux', 'real owned process cleanup')
def test_worker_drain_uses_stage_deadline_not_cleanup_budget(self):
from harness.stageworker import Worker
owner = execution.Execution(self.rd / 'drain')
cfg = policy(); cfg.update(cleanup_deadline_s=.05, cancel_grace_s=.01)
owner.start(cfg)
worker = Worker(owner.rd, execution=owner)
started, results = threading.Event(), []
def stage():
with owner.scope('05-compile', 'h1'):
owner.begin_phase('h1', '05-compile'); owner.phase_running('h1')
results.append(watchdog.run([sys.executable, '-c', 'import time;time.sleep(.3)'],
cwd=owner.rd, stdout=owner.rd/'out', stderr=owner.rd/'err', execution=owner,
role='stage', poll_s=.01, on_start=lambda _: started.set()))
worker.thread = threading.Thread(target=stage)
with worker:
self.assertTrue(started.wait(3))
self.assertEqual(results[0].state, 'OK')
self.assertGreater(results[0].duration_s, cfg['cleanup_deadline_s'])
self.assertFalse(owner.snapshot()['cancel_requested'])
self.assertEqual(results[0].cleanup_state, 'VERIFIED')
@unittest.skipUnless(sys.platform == 'linux', 'real owned process cleanup')
def test_worker_drain_keeps_explicit_cancel_and_exception_cleanup(self):
from harness.stageworker import Worker
for explicit in (True, False):
with self.subTest(explicit=explicit):
owner = execution.Execution(self.rd / str(explicit)); owner.start(policy())
worker = Worker(owner.rd, execution=owner)
started, results = threading.Event(), []
def stage():
with owner.scope('05-compile', 'h1'):
owner.begin_phase('h1', '05-compile'); owner.phase_running('h1')
results.append(watchdog.run([sys.executable, '-c', 'import time;time.sleep(20)'],
cwd=owner.rd, stdout=owner.rd/'out', stderr=owner.rd/'err', execution=owner,
role='stage', poll_s=.01, on_start=lambda _: started.set()))
worker.thread = threading.Thread(target=stage)
try:
with worker:
self.assertTrue(started.wait(3))
if explicit:
owner.cancel('USER_STOP')
else:
raise ValueError('fixture context failed')
except ValueError as error:
self.assertFalse(explicit)
self.assertEqual(str(error), 'fixture context failed')
self.assertEqual(owner.snapshot()['cancel_requested']['reason'],
'USER_STOP' if explicit else 'WORKER_CONTEXT_ERROR')
self.assertEqual(results[0].cleanup_state, 'VERIFIED')
self.assertEqual(execution.owned_processes(owner.snapshot()), [])
def test_exit_and_protocol_are_independent(self):
self.assertEqual(self.ex.finish(143, protocol_complete=True)['execution_outcome'], 'AGENT_ERROR')
other = execution.Execution(self.rd / 'other'); other.start(policy())
self.assertEqual(other.finish(0, protocol_complete=False)['execution_outcome'], 'AGENT_ERROR')
def test_late_cancel_does_not_overwrite_completion(self):
self.ex.finish(0, protocol_complete=True)
self.assertEqual(self.ex.cancel('late')['reason'], 'ALREADY_TERMINAL')
self.assertEqual(self.ex.snapshot()['execution_outcome'], 'COMPLETED')
def test_old_epoch_is_fenced(self):
stale = execution.Execution(self.rd)
stale.epoch = self.ex.epoch - 1
with self.assertRaises(execution.ExecutionStopped):
with stale.admission():
self.fail('old owner admitted')
def test_queue_wait_is_bounded(self):
with patch.object(scheduler, '_try_slots', return_value=(None, None)), \
patch.object(scheduler, 'pool_size', return_value=1), \
patch.object(scheduler, 'pool_reserve', return_value=0), self.ex.scope('05-compile'):
before = time.monotonic()
with self.assertRaises(execution.ExecutionStopped):
with scheduler.acquire('cpu', poll_s=.01):
self.fail('unavailable slot acquired')
self.assertLess(time.monotonic() - before, 1)
def test_pid_reuse_is_not_alive(self):
ident = execution.process_identity(os.getpid())
if ident.get('status') == 'UNKNOWN':
self.skipTest('Linux proc identity required')
ident['proc_start_ticks'] += 1
self.assertEqual(execution.process_status(ident), 'DEAD')
def test_restart_does_not_reset_deadline_or_replay(self):
state = self.ex.snapshot()
duplicate = execution.Execution(self.rd)
with self.assertRaises(execution.ExecutionStopped):
duplicate.start(policy())
self.assertEqual(duplicate.snapshot()['deadline_monotonic'], state['deadline_monotonic'])
def test_completed_requires_verified_cleanup(self):
result = self.ex.finish(0, protocol_complete=True, cleanup_state='UNKNOWN')
self.assertEqual(result['cleanup_state'], 'QUARANTINED')
self.assertNotEqual(result['execution_outcome'], 'COMPLETED')
def test_cancel_serializes_with_spawn_admission(self):
started, committed = threading.Event(), threading.Event()
def cancel():
started.set()
self.ex.cancel('race fixture')
committed.set()
with self.ex.admission():
thread = threading.Thread(target=cancel)
thread.start(); started.wait(1)
self.assertFalse(committed.wait(.03))
thread.join(1)
self.assertTrue(committed.is_set())
with self.assertRaises(execution.ExecutionStopped):
with self.ex.admission():
self.fail('post-cancel spawn admitted')
def test_crashed_holder_lease_is_not_reallocated(self):
with patch.object(scheduler.paths, 'STATE', self.rd / 'state'), self.ex.scope('05-compile'):
lease, _ = scheduler._try_slots('cpu', [0])
self.assertIsNotNone(lease)
fcntl.flock(lease, fcntl.LOCK_UN); lease.close() # Simulate kernel lock release after a crash.
next_lease, _ = scheduler._try_slots('cpu', [0])
self.assertIsNone(next_lease)
self.assertTrue((self.rd / 'state/locks/cpu-0.lease.json').exists())
def test_unknown_telemetry_does_not_disable_deadline(self):
with patch.object(watchdog, '_proc_tree_stat', return_value=None):
before = time.monotonic()
result = watchdog.run([sys.executable, '-c', 'import time; time.sleep(60)'], cwd=self.rd,
stdout=self.rd/'out', stderr=self.rd/'err', execution=self.ex, poll_s=.01, hard_ceiling_s=.1)
self.assertLess(time.monotonic() - before, 3)
self.assertIn('DEADLINE', result.stalled_reason)
self.assertEqual(result.cleanup_state, 'VERIFIED')
def test_expired_start_bookkeeping_never_releases_command_gate(self):
marker = self.rd / 'must-not-start'
result = watchdog.run([sys.executable, '-c',
"from pathlib import Path;Path('must-not-start').touch()"], cwd=self.rd,
stdout=self.rd/'gate.out', stderr=self.rd/'gate.err', execution=self.ex,
poll_s=.01, hard_ceiling_s=.1, on_start=lambda _: time.sleep(.2))
self.assertIn('DEADLINE', result.stalled_reason)
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertFalse(marker.exists())
self.assertLess(result.duration_s, 3)
def test_predeadline_channel_loss_remains_infrastructure_error(self):
def stop_supervisor(proc):
execution.signal_process(proc.supervisor_identity, signal.SIGTERM)
proc.supervisor.wait(timeout=3)
result = watchdog.run([sys.executable, '-c',
"from pathlib import Path;Path('must-not-start').touch()"], cwd=self.rd,
stdout=self.rd/'closed.out', stderr=self.rd/'closed.err', execution=self.ex,
poll_s=.01, hard_ceiling_s=5, on_start=stop_supervisor)
self.assertIn('runner monitor failed:', result.stalled_reason)
self.assertNotIn('DEADLINE', result.stalled_reason)
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertFalse((self.rd/'must-not-start').exists())
def test_suspect_is_advisory_not_an_idle_kill(self):
with patch.object(watchdog, '_proc_tree_stat', return_value=(100, 0)), \
patch.object(watchdog, '_io_bytes', return_value=0):
result = watchdog.run([sys.executable, '-c', 'import time; time.sleep(.25)'], cwd=self.rd,
stdout=self.rd/'out', stderr=self.rd/'err', execution=self.ex, poll_s=.01)
self.assertEqual(result.state, 'OK')
self.assertTrue(result.suspect)
def worker_fixture(self):
from harness.stageworker import Worker
rd = self.rd / 'guard-smoke' / 'rollout'
(rd / 'handles/h1').mkdir(parents=True)
(rd / 'workspace').mkdir()
owner = execution.Execution(rd); owner.start(policy())
request = {'stage': 'noop', 'submitted_at': 1}
(rd / 'handles/h1/request.json').write_text(json.dumps(request))
return rd, owner, Worker(rd, test_noop=True, execution=owner)
def test_worker_restart_reuses_receipt_without_dispatch(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
worker.scan()
operation = rd / 'stages/_queue/h1/operation.json'
before = operation.read_bytes()
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(operation.read_bytes(), before)
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['result'], 'OK')
def test_worker_payload_conflict_preserves_original_receipt(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
worker.scan()
receipt = rd / 'stages/_queue/h1/status.json'
before = receipt.read_bytes()
(rd / 'handles/h1/request.json').write_text('{"stage":"noop","submitted_at":2}')
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(receipt.read_bytes(), before)
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['error_class'],
'OPERATION_PAYLOAD_CONFLICT')
def test_unknown_operation_is_quarantined_not_replayed(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
worker.scan()
(rd/'stages/_queue/h1/status.json').unlink()
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(owner.snapshot()['cleanup_state'], 'QUARANTINED')
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['error_class'],
'SIDE_EFFECT_UNKNOWN')
def checkpoint(self):
path = self.rd / 'session.jsonl'
path.write_text('{"message_id":"m1"}\n')
return {'complete': True, 'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}
def test_pause_resume_preserves_identity_deadline_and_fences_previous_epoch(self):
before = self.ex.snapshot()
stale = execution.Execution(self.rd).attach()
self.ex.pause('QUOTA_WAIT_BUDGET_EXHAUSTED', self.checkpoint())
with self.assertRaises(execution.ExecutionStopped):
with self.ex.admission():
self.fail('paused execution dispatched')
with patch.object(execution, 'marker_processes', return_value=[]):
state = self.ex.resume()
self.assertEqual(state['execution_id'], before['execution_id'])
self.assertEqual(state['deadline_monotonic'], before['deadline_monotonic'])
self.assertEqual(state['epoch'], before['epoch'] + 1)
self.assertEqual(state['auto_resume_count'], 1)
with self.assertRaises(execution.ExecutionStopped):
with stale.admission():
self.fail('old epoch survived takeover')
def test_incomplete_or_changed_checkpoint_cannot_resume(self):
with self.assertRaisesRegex(execution.ExecutionStopped, 'INCOMPLETE'):
self.ex.pause('quota', {})
checkpoint = self.checkpoint()
self.ex.pause('quota', checkpoint)
Path(checkpoint['path']).write_text('changed')
with self.assertRaisesRegex(execution.ExecutionStopped, 'HASH_MISMATCH'):
self.ex.resume()
def test_auto_resume_budget_survives_new_controller_objects(self):
checkpoint = self.checkpoint()
with patch.object(execution, 'marker_processes', return_value=[]):
for _ in range(3):
self.ex.pause('quota', checkpoint)
self.ex = execution.Execution(self.rd).attach()
self.ex.resume()
self.ex.pause('quota', checkpoint)
with self.assertRaisesRegex(execution.ExecutionStopped, 'AUTO_RESUME_BUDGET_EXHAUSTED'):
execution.Execution(self.rd).resume()
free = execution.Execution(self.rd)
free.resume(counted=False) # a pause that sent nothing (connect/DNS failure) is free
self.assertEqual(free.snapshot()['auto_resume_count'], 3)
@unittest.skipUnless(Path('/proc').is_dir(), 'Actual worker requires Linux owner identity')
def test_worker_executes_real_protected_stage_and_reuses_receipt(self):
from harness import stageworker, stagerun
import yaml
root = self.rd / 'fixture-project'
root.mkdir()
source = Path(__file__).resolve().parents[1]
(root/'harness').symlink_to(source/'harness', target_is_directory=True)
(root/'registry').symlink_to(source/'registry', target_is_directory=True)
cfg = {'paths': {'harness_python': sys.executable}, 'pools': {'cpu': {'workers': 1}}}
(root/'config.yaml').write_text(yaml.safe_dump(cfg))
weights = root/'cache/weights/fixture-sha'
weights.mkdir(parents=True)
(weights.parent/'index.json').write_text('{"fixture":{"sha256":"fixture-sha"}}')
rd = root/'runs/r/j/rollouts/00'
(rd/'handles/h1').mkdir(parents=True)
(rd/'workspace').mkdir()
(rd.parent.parent/'job.json').write_text(json.dumps({'run_id':'r', 'job_id':'j',
'model':'fixture', 'task':'cls', 'march':'nash-p', 'backend':'ptq'}))
(rd/'handles/h1/request.json').write_text('{"stage":"01-fetch"}')
owner = execution.Execution(rd)
fixture_policy = policy(); fixture_policy['phases']['01-fetch']['queue_deadline_s'] = 2
owner.start(fixture_policy)
connect = stagerun.db.connect
with patch.object(stagerun.db, 'connect', lambda **kw: connect(root/'state/jobs.sqlite', **kw)), \
patch.object(stagerun.paths, 'BENCH_ROOT', root), \
patch.object(stagerun.paths, 'config', return_value=cfg), \
patch.dict(os.environ, {'BENCH_CONFIG': str(root/'config.yaml'), 'BENCH_STATE_DIR': str(root/'state')}):
worker = stageworker.Worker(rd, execution=owner)
worker.scan()
status = json.loads((rd/'handles/h1/status.json').read_text())
self.assertEqual(status['result'], 'SKIPPED', status)
self.assertEqual(owner.snapshot()['operations']['h1']['state'], 'DONE')
# A later attempt may overwrite a settled attempt's outputs; a resumed worker must
# neither replay nor re-prove it (10-13: re-proof quarantined 19 rollouts).
with patch.object(stageworker.watchdog, 'run') as replay, \
patch.object(stageworker.Worker, '_reconcile_stage', side_effect=ValueError('completed artifact changed')):
stageworker.Worker(rd, execution=owner).scan()
replay.assert_not_called()
self.assertIsNone(owner.snapshot().get('stop_reason'))
self.assertFalse(list((root/'state/locks').glob('*.lease.json')))
def test_receipt_cannot_bind_reused_supervisor_pid(self):
binding = dict(proof='waitpid:ECHILD', supervisor=dict(pid=123, proc_start_ticks=1))
receipt = self.rd/'receipt.json'
execution.atomic_json(receipt, dict(binding=dict(binding, supervisor=dict(pid=123, proc_start_ticks=2)),
cleanup_state='VERIFIED', all_children_reaped=True, owner_lost=False, exit_code=0))
self.assertIsNone(execution.scope_receipt(dict(binding=binding, receipt_path=str(receipt))))
def test_board_exception_preserves_lease_and_quarantines_before_release(self):
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock
from harness import stagerun
hd = self.rd/'stages/_queue/h1'; hd.mkdir(parents=True)
(hd/'request.json').write_text('{"stage":"07-board"}')
with self.ex.mutation() as state:
operation = dict(operation_id='h1', payload_sha256='fixture', execution_id=state['execution_id'],
epoch=state['epoch'], state='DISPATCHED')
state['operations']['h1'] = operation
self.ex._save(state, 'TEST_DISPATCHED')
(hd/'operation.json').write_text(json.dumps(operation))
class Board:
name, pool, level = '07-board', 'board', 'job'
def run(self, ctx):
raise RuntimeError('transport failed after dispatch')
ctx = SimpleNamespace(job={'model':'m','march':'nash-p','job_id':'j'},extra={},
run_id='r',rollout=0,lane='normal',agent_config=None)
with patch.object(stagerun, '_ctx_from_rollout', return_value=ctx), \
patch.dict(stagerun.BY_NAME, {'07-board':Board}), \
patch.object(stagerun.db, 'connect', side_effect=lambda:nullcontext(Mock())), \
patch.object(stagerun.db, 'stage_start'), \
patch.object(scheduler, 'board_slots', return_value={0:{'name':'fixture','march':'nash-p'}}), \
patch.object(scheduler.paths, 'STATE', self.rd/'state'), \
patch.object(scheduler.paths, 'config', return_value={'pools':{'board':{'reserve_for_agent':0}}}), \
self.ex.scope('07-board', 'h1'):
with self.assertRaisesRegex(RuntimeError, 'transport'):
stagerun._exec_impl(hd, rd=self.rd, protected=True)
self.assertEqual(self.ex.snapshot()['cleanup_state'], 'QUARANTINED')
self.assertTrue((self.rd/'state/locks/board-0.lease.json').exists())
self.assertTrue((self.rd/'state/locks/board-0.quarantine.json').exists())
def test_stale_owner_cannot_mutate_or_cleanup(self):
old = execution.Execution(self.rd).attach()
self.ex.cancel('fixture')
state = self.ex.snapshot(); state['epoch'] += 1; self.ex.epoch = state['epoch']
execution.atomic_json(self.ex.path, state)
before = self.ex.path.read_bytes()
actions = [lambda:old.quarantine('late'), lambda:old.cleanup(), lambda:old.cancel('late'),
lambda:old.activity('late'), lambda:old.finish(0, protocol_complete=True)]
for action in actions:
with self.assertRaisesRegex(execution.ExecutionStopped, 'OWNER_FENCED'):
action()
self.assertEqual(self.ex.path.read_bytes(), before)
def test_unknown_owner_blocks_admission_but_deadline_still_expires(self):
with patch.object(execution, 'process_status', return_value='UNKNOWN'):
with self.assertRaisesRegex(execution.ExecutionStopped, 'OWNER_STATUS_UNKNOWN'):
with self.ex.admission():
self.fail('unknown owner admitted')
self.assertIsNone(self.ex.stop_reason())
state = self.ex.snapshot(); state['deadline_monotonic'] = time.monotonic() - 1
execution.atomic_json(self.ex.path, state)
self.assertEqual(self.ex.stop_reason(), 'EXECUTION_DEADLINE_EXCEEDED')
def test_phase_queue_and_execution_clocks_are_separate_and_supervised(self):
owner = execution.Execution(self.rd/'phase')
cfg = policy(); cfg['phases']['05-compile'].update(queue_deadline_s=2, execution_deadline_s=.1)
owner.start(cfg)
owner.begin_phase('h1', '05-compile')
self.assertIsNone(owner.snapshot()['phases']['h1']['execution_deadline_monotonic'])
time.sleep(.12) # More than the execution budget, still legitimately queued.
self.assertIsNone(owner.stop_reason('h1'))
owner.phase_running('h1')
with owner.scope('05-compile', 'h1'):
result = watchdog.run([sys.executable, '-c', 'import time;time.sleep(.4)'], cwd=self.rd,
stdout=self.rd/'phase.out', stderr=self.rd/'phase.err', execution=owner, role='stage',
env=dict(os.environ, BENCH_OPERATION_ID='h1'), poll_s=.02, hard_ceiling_s=5)
self.assertIn('PHASE_EXECUTION_DEADLINE', result.stalled_reason)
self.assertLess(result.duration_s, .35)
self.assertIsNotNone(owner.snapshot()['phases']['h1']['execution_deadline_monotonic'])
def test_typed_heartbeat_is_not_business_progress(self):
path = self.rd/'events'; path.touch()
tracker = watchdog.BusinessLog(path)
path.write_text('{"type":"heartbeat","state":"RUNNING"}\n')
self.assertEqual(tracker.sample(), 0)
with path.open('a') as out:
out.write('{"type":"message_end","message":{"role":"assistant","content":[]}}\n')
self.assertGreater(tracker.sample(), 0)
with path.open('a') as out:
out.write('unclassified operational text\n')
self.assertIsNone(tracker.sample())
def test_progress_health_is_persisted_and_unknown_not_zero(self):
signals = dict(cpu_ticks=1, io_bytes=0, business_log_bytes=0, stage_sequence=0)
self.ex.record_progress('p', signals)
time.sleep(.12)
self.ex.record_progress('p', signals)
self.assertEqual(self.ex.snapshot()['health'], 'SUSPECT')
self.ex.record_progress('p', dict(signals, io_bytes=None))
state = self.ex.snapshot()
self.assertEqual(state['health'], 'UNKNOWN')
self.assertIsNone(state['progress']['signals']['io_bytes'])
def test_submission_replay_retains_operation_identity(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
source = rd/'stages/_queue/source'; source.mkdir(parents=True)
(source/'recipe.json').write_text('{}')
(source/'request.json').write_text(json.dumps({'recipe':str(source/'recipe.json')}))
(rd/'handles/h1/request.json').write_text('{"stage":"noop","submission":"source"}')
worker.scan()
record = json.loads((rd/'stages/_queue/h1/operation.json').read_text())
self.assertIn('payload_sha256', record)
self.assertEqual(record['execution_id'], owner.snapshot()['execution_id'])
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['result'], 'OK')
if __name__ == '__main__':
unittest.main()