521 lines
28 KiB
Python
521 lines
28 KiB
Python
"""Lifecycle contracts using only temporary state and owned test processes."""
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import json
|
|
import hashlib
|
|
import fcntl
|
|
import time
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from harness import execution, process_reaper, scheduler, watchdog
|
|
|
|
|
|
def policy():
|
|
base = dict(queue_deadline_s=.15, execution_deadline_s=10, suspect_after_s=.1,
|
|
cancel_grace_s=.1, cleanup_deadline_s=2)
|
|
return dict(base, version='test-v1', phases={name: dict(base) for name in execution.PHASES})
|
|
|
|
|
|
class ProcessStatBytesTests(unittest.TestCase):
|
|
@staticmethod
|
|
def stat(comm=b'worker', *, birth=42, state=b'S', ppid=10):
|
|
fields = [state, str(ppid).encode(), b'10', b'10'] + [b'0'] * 16
|
|
fields[11:13] = [b'13', b'17']
|
|
fields[19] = str(birth).encode()
|
|
return b'123 (' + comm + b') ' + b' '.join(fields) + b'\n'
|
|
|
|
def test_identity_ignores_arbitrary_comm_bytes(self):
|
|
for comm in (b'worker', b'a (b) c', b'\xff', '中文线程名截断'.encode()[:15]):
|
|
with self.subTest(comm=comm), patch.object(Path, 'read_bytes', return_value=self.stat(comm)):
|
|
identity = execution.process_identity(123)
|
|
self.assertEqual(identity['status'], 'KNOWN')
|
|
self.assertEqual((identity['proc_start_ticks'], identity['ppid'], identity['state'],
|
|
identity['cpu_ticks']), (42, 10, 'S', 30))
|
|
json.dumps(identity) # Receipts keep their existing text/number schema.
|
|
|
|
def test_corrupt_identity_is_unknown_and_never_authorizes_a_signal(self):
|
|
original = dict(host=execution.socket.gethostname(), boot_id='fixture', pid=123,
|
|
proc_start_ticks=42)
|
|
for raw in (b'', b'123 no delimiter', self.stat(state=b'\xff'),
|
|
self.stat().replace(b'42\n', b'bad\n')):
|
|
with self.subTest(raw=raw), patch.object(Path, 'read_bytes', return_value=raw), \
|
|
patch.object(Path, 'exists', return_value=True), \
|
|
patch.object(execution, 'boot_id', return_value='fixture'), \
|
|
patch.object(os, 'pidfd_open', create=True) as opened:
|
|
self.assertEqual(execution.process_identity(123)['status'], 'UNKNOWN')
|
|
self.assertEqual(execution.process_status(original), 'UNKNOWN')
|
|
self.assertFalse(execution.signal_process(original, signal.SIGTERM))
|
|
opened.assert_not_called()
|
|
|
|
def test_reaper_preserves_second_birth_and_parent_check(self):
|
|
reaper = process_reaper.Reaper.__new__(process_reaper.Reaper)
|
|
reaper.pid, reaper.host, reaper.boot_id, reaper.members = 10, 'fixture', 'fixture', {}
|
|
first = self.stat(b'\xff name (nested)')
|
|
for second, permitted in ((first, True), (self.stat(birth=43), False),
|
|
(self.stat(ppid=11), False)):
|
|
with self.subTest(permitted=permitted), \
|
|
patch.object(process_reaper, 'children', return_value={123}), \
|
|
patch.object(Path, 'read_bytes', side_effect=[first, second]), \
|
|
patch.object(os, 'pidfd_open', return_value=99, create=True), \
|
|
patch.object(os, 'close') as closed, \
|
|
patch.object(signal, 'pidfd_send_signal', create=True) as sent:
|
|
reaper.signal(signal.SIGTERM)
|
|
self.assertEqual(sent.call_count, int(permitted))
|
|
closed.assert_called_once_with(99)
|
|
self.assertEqual(reaper.members[(123, 42)]['state'], 'S')
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'Linux comm and subreaper')
|
|
def test_non_utf8_child_name_is_still_identified_and_reaped(self):
|
|
with tempfile.TemporaryDirectory() as name:
|
|
root = Path(name)
|
|
owner = execution.Execution(root); owner.start(policy())
|
|
code = ("import ctypes,time;from pathlib import Path;"
|
|
"assert ctypes.CDLL(None).prctl(15,b'worker-\\xff',0,0,0)==0;"
|
|
"Path('armed').touch();time.sleep(30)")
|
|
result = watchdog.run([sys.executable, '-c', code], cwd=root,
|
|
stdout=root/'out', stderr=root/'err', execution=owner, poll_s=.01, hard_ceiling_s=3,
|
|
stop_check=lambda: 'fixture complete' if (root/'armed').exists() else None)
|
|
self.assertEqual(result.stalled_reason, 'fixture complete')
|
|
self.assertEqual(result.cleanup_state, 'VERIFIED')
|
|
for identity in owner.snapshot()['processes']:
|
|
self.assertNotEqual(execution.process_status(identity), 'ALIVE')
|
|
|
|
|
|
class ExecutionTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.tmp.cleanup)
|
|
self.rd = Path(self.tmp.name)
|
|
self.ex = execution.Execution(self.rd)
|
|
self.ex.start(policy())
|
|
if not Path('/proc').is_dir():
|
|
# Production admission requires Linux identity; mock only this known
|
|
# test controller for portable pure state-machine checks.
|
|
original = execution.process_status
|
|
owner = execution.process_key(self.ex.snapshot()['owner'])
|
|
override = patch.object(execution, 'process_status', side_effect=lambda identity:
|
|
'ALIVE' if execution.process_key(identity) == owner else original(identity))
|
|
override.start(); self.addCleanup(override.stop)
|
|
|
|
def test_cancel_fences_spawn_and_cannot_be_completed(self):
|
|
self.ex.cancel('test cancel')
|
|
with self.assertRaises(execution.ExecutionStopped):
|
|
with self.ex.admission():
|
|
self.fail('cancelled admission opened')
|
|
result = self.ex.finish(0, protocol_complete=True)
|
|
self.assertEqual(result['execution_outcome'], 'CANCELLED')
|
|
self.assertEqual(result['cleanup_state'], 'VERIFIED')
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'real owned process cleanup')
|
|
def test_worker_drain_uses_stage_deadline_not_cleanup_budget(self):
|
|
from harness.stageworker import Worker
|
|
owner = execution.Execution(self.rd / 'drain')
|
|
cfg = policy(); cfg.update(cleanup_deadline_s=.05, cancel_grace_s=.01)
|
|
owner.start(cfg)
|
|
worker = Worker(owner.rd, execution=owner)
|
|
started, results = threading.Event(), []
|
|
def stage():
|
|
with owner.scope('05-compile', 'h1'):
|
|
owner.begin_phase('h1', '05-compile'); owner.phase_running('h1')
|
|
results.append(watchdog.run([sys.executable, '-c', 'import time;time.sleep(.3)'],
|
|
cwd=owner.rd, stdout=owner.rd/'out', stderr=owner.rd/'err', execution=owner,
|
|
role='stage', poll_s=.01, on_start=lambda _: started.set()))
|
|
worker.thread = threading.Thread(target=stage)
|
|
with worker:
|
|
self.assertTrue(started.wait(3))
|
|
self.assertEqual(results[0].state, 'OK')
|
|
self.assertGreater(results[0].duration_s, cfg['cleanup_deadline_s'])
|
|
self.assertFalse(owner.snapshot()['cancel_requested'])
|
|
self.assertEqual(results[0].cleanup_state, 'VERIFIED')
|
|
|
|
@unittest.skipUnless(sys.platform == 'linux', 'real owned process cleanup')
|
|
def test_worker_drain_keeps_explicit_cancel_and_exception_cleanup(self):
|
|
from harness.stageworker import Worker
|
|
for explicit in (True, False):
|
|
with self.subTest(explicit=explicit):
|
|
owner = execution.Execution(self.rd / str(explicit)); owner.start(policy())
|
|
worker = Worker(owner.rd, execution=owner)
|
|
started, results = threading.Event(), []
|
|
def stage():
|
|
with owner.scope('05-compile', 'h1'):
|
|
owner.begin_phase('h1', '05-compile'); owner.phase_running('h1')
|
|
results.append(watchdog.run([sys.executable, '-c', 'import time;time.sleep(20)'],
|
|
cwd=owner.rd, stdout=owner.rd/'out', stderr=owner.rd/'err', execution=owner,
|
|
role='stage', poll_s=.01, on_start=lambda _: started.set()))
|
|
worker.thread = threading.Thread(target=stage)
|
|
try:
|
|
with worker:
|
|
self.assertTrue(started.wait(3))
|
|
if explicit:
|
|
owner.cancel('USER_STOP')
|
|
else:
|
|
raise ValueError('fixture context failed')
|
|
except ValueError as error:
|
|
self.assertFalse(explicit)
|
|
self.assertEqual(str(error), 'fixture context failed')
|
|
self.assertEqual(owner.snapshot()['cancel_requested']['reason'],
|
|
'USER_STOP' if explicit else 'WORKER_CONTEXT_ERROR')
|
|
self.assertEqual(results[0].cleanup_state, 'VERIFIED')
|
|
self.assertEqual(execution.owned_processes(owner.snapshot()), [])
|
|
|
|
def test_exit_and_protocol_are_independent(self):
|
|
self.assertEqual(self.ex.finish(143, protocol_complete=True)['execution_outcome'], 'AGENT_ERROR')
|
|
other = execution.Execution(self.rd / 'other'); other.start(policy())
|
|
self.assertEqual(other.finish(0, protocol_complete=False)['execution_outcome'], 'AGENT_ERROR')
|
|
|
|
def test_late_cancel_does_not_overwrite_completion(self):
|
|
self.ex.finish(0, protocol_complete=True)
|
|
self.assertEqual(self.ex.cancel('late')['reason'], 'ALREADY_TERMINAL')
|
|
self.assertEqual(self.ex.snapshot()['execution_outcome'], 'COMPLETED')
|
|
|
|
def test_old_epoch_is_fenced(self):
|
|
stale = execution.Execution(self.rd)
|
|
stale.epoch = self.ex.epoch - 1
|
|
with self.assertRaises(execution.ExecutionStopped):
|
|
with stale.admission():
|
|
self.fail('old owner admitted')
|
|
|
|
def test_queue_wait_is_bounded(self):
|
|
with patch.object(scheduler, '_try_slots', return_value=(None, None)), \
|
|
patch.object(scheduler, 'pool_size', return_value=1), \
|
|
patch.object(scheduler, 'pool_reserve', return_value=0), self.ex.scope('05-compile'):
|
|
before = time.monotonic()
|
|
with self.assertRaises(execution.ExecutionStopped):
|
|
with scheduler.acquire('cpu', poll_s=.01):
|
|
self.fail('unavailable slot acquired')
|
|
self.assertLess(time.monotonic() - before, 1)
|
|
|
|
def test_pid_reuse_is_not_alive(self):
|
|
ident = execution.process_identity(os.getpid())
|
|
if ident.get('status') == 'UNKNOWN':
|
|
self.skipTest('Linux proc identity required')
|
|
ident['proc_start_ticks'] += 1
|
|
self.assertEqual(execution.process_status(ident), 'DEAD')
|
|
|
|
def test_restart_does_not_reset_deadline_or_replay(self):
|
|
state = self.ex.snapshot()
|
|
duplicate = execution.Execution(self.rd)
|
|
with self.assertRaises(execution.ExecutionStopped):
|
|
duplicate.start(policy())
|
|
self.assertEqual(duplicate.snapshot()['deadline_monotonic'], state['deadline_monotonic'])
|
|
|
|
def test_completed_requires_verified_cleanup(self):
|
|
result = self.ex.finish(0, protocol_complete=True, cleanup_state='UNKNOWN')
|
|
self.assertEqual(result['cleanup_state'], 'QUARANTINED')
|
|
self.assertNotEqual(result['execution_outcome'], 'COMPLETED')
|
|
|
|
def test_cancel_serializes_with_spawn_admission(self):
|
|
started, committed = threading.Event(), threading.Event()
|
|
def cancel():
|
|
started.set()
|
|
self.ex.cancel('race fixture')
|
|
committed.set()
|
|
with self.ex.admission():
|
|
thread = threading.Thread(target=cancel)
|
|
thread.start(); started.wait(1)
|
|
self.assertFalse(committed.wait(.03))
|
|
thread.join(1)
|
|
self.assertTrue(committed.is_set())
|
|
with self.assertRaises(execution.ExecutionStopped):
|
|
with self.ex.admission():
|
|
self.fail('post-cancel spawn admitted')
|
|
|
|
def test_crashed_holder_lease_is_not_reallocated(self):
|
|
with patch.object(scheduler.paths, 'STATE', self.rd / 'state'), self.ex.scope('05-compile'):
|
|
lease, _ = scheduler._try_slots('cpu', [0])
|
|
self.assertIsNotNone(lease)
|
|
fcntl.flock(lease, fcntl.LOCK_UN); lease.close() # Simulate kernel lock release after a crash.
|
|
next_lease, _ = scheduler._try_slots('cpu', [0])
|
|
self.assertIsNone(next_lease)
|
|
self.assertTrue((self.rd / 'state/locks/cpu-0.lease.json').exists())
|
|
|
|
def test_unknown_telemetry_does_not_disable_deadline(self):
|
|
with patch.object(watchdog, '_proc_tree_stat', return_value=None):
|
|
before = time.monotonic()
|
|
result = watchdog.run([sys.executable, '-c', 'import time; time.sleep(60)'], cwd=self.rd,
|
|
stdout=self.rd/'out', stderr=self.rd/'err', execution=self.ex, poll_s=.01, hard_ceiling_s=.1)
|
|
self.assertLess(time.monotonic() - before, 3)
|
|
self.assertIn('DEADLINE', result.stalled_reason)
|
|
self.assertEqual(result.cleanup_state, 'VERIFIED')
|
|
|
|
def test_expired_start_bookkeeping_never_releases_command_gate(self):
|
|
marker = self.rd / 'must-not-start'
|
|
result = watchdog.run([sys.executable, '-c',
|
|
"from pathlib import Path;Path('must-not-start').touch()"], cwd=self.rd,
|
|
stdout=self.rd/'gate.out', stderr=self.rd/'gate.err', execution=self.ex,
|
|
poll_s=.01, hard_ceiling_s=.1, on_start=lambda _: time.sleep(.2))
|
|
self.assertIn('DEADLINE', result.stalled_reason)
|
|
self.assertEqual(result.cleanup_state, 'VERIFIED')
|
|
self.assertFalse(marker.exists())
|
|
self.assertLess(result.duration_s, 3)
|
|
|
|
def test_predeadline_channel_loss_remains_infrastructure_error(self):
|
|
def stop_supervisor(proc):
|
|
execution.signal_process(proc.supervisor_identity, signal.SIGTERM)
|
|
proc.supervisor.wait(timeout=3)
|
|
result = watchdog.run([sys.executable, '-c',
|
|
"from pathlib import Path;Path('must-not-start').touch()"], cwd=self.rd,
|
|
stdout=self.rd/'closed.out', stderr=self.rd/'closed.err', execution=self.ex,
|
|
poll_s=.01, hard_ceiling_s=5, on_start=stop_supervisor)
|
|
self.assertIn('runner monitor failed:', result.stalled_reason)
|
|
self.assertNotIn('DEADLINE', result.stalled_reason)
|
|
self.assertEqual(result.cleanup_state, 'VERIFIED')
|
|
self.assertFalse((self.rd/'must-not-start').exists())
|
|
|
|
def test_suspect_is_advisory_not_an_idle_kill(self):
|
|
with patch.object(watchdog, '_proc_tree_stat', return_value=(100, 0)), \
|
|
patch.object(watchdog, '_io_bytes', return_value=0):
|
|
result = watchdog.run([sys.executable, '-c', 'import time; time.sleep(.25)'], cwd=self.rd,
|
|
stdout=self.rd/'out', stderr=self.rd/'err', execution=self.ex, poll_s=.01)
|
|
self.assertEqual(result.state, 'OK')
|
|
self.assertTrue(result.suspect)
|
|
|
|
def worker_fixture(self):
|
|
from harness.stageworker import Worker
|
|
rd = self.rd / 'guard-smoke' / 'rollout'
|
|
(rd / 'handles/h1').mkdir(parents=True)
|
|
(rd / 'workspace').mkdir()
|
|
owner = execution.Execution(rd); owner.start(policy())
|
|
request = {'stage': 'noop', 'submitted_at': 1}
|
|
(rd / 'handles/h1/request.json').write_text(json.dumps(request))
|
|
return rd, owner, Worker(rd, test_noop=True, execution=owner)
|
|
|
|
def test_worker_restart_reuses_receipt_without_dispatch(self):
|
|
from harness.stageworker import Worker
|
|
rd, owner, worker = self.worker_fixture()
|
|
worker.scan()
|
|
operation = rd / 'stages/_queue/h1/operation.json'
|
|
before = operation.read_bytes()
|
|
Worker(rd, test_noop=True, execution=owner).scan()
|
|
self.assertEqual(operation.read_bytes(), before)
|
|
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['result'], 'OK')
|
|
|
|
def test_worker_payload_conflict_preserves_original_receipt(self):
|
|
from harness.stageworker import Worker
|
|
rd, owner, worker = self.worker_fixture()
|
|
worker.scan()
|
|
receipt = rd / 'stages/_queue/h1/status.json'
|
|
before = receipt.read_bytes()
|
|
(rd / 'handles/h1/request.json').write_text('{"stage":"noop","submitted_at":2}')
|
|
Worker(rd, test_noop=True, execution=owner).scan()
|
|
self.assertEqual(receipt.read_bytes(), before)
|
|
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['error_class'],
|
|
'OPERATION_PAYLOAD_CONFLICT')
|
|
|
|
def test_unknown_operation_is_quarantined_not_replayed(self):
|
|
from harness.stageworker import Worker
|
|
rd, owner, worker = self.worker_fixture()
|
|
worker.scan()
|
|
(rd/'stages/_queue/h1/status.json').unlink()
|
|
Worker(rd, test_noop=True, execution=owner).scan()
|
|
self.assertEqual(owner.snapshot()['cleanup_state'], 'QUARANTINED')
|
|
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['error_class'],
|
|
'SIDE_EFFECT_UNKNOWN')
|
|
|
|
def checkpoint(self):
|
|
path = self.rd / 'session.jsonl'
|
|
path.write_text('{"message_id":"m1"}\n')
|
|
return {'complete': True, 'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}
|
|
|
|
def test_pause_resume_preserves_identity_deadline_and_fences_previous_epoch(self):
|
|
before = self.ex.snapshot()
|
|
stale = execution.Execution(self.rd).attach()
|
|
self.ex.pause('QUOTA_WAIT_BUDGET_EXHAUSTED', self.checkpoint())
|
|
with self.assertRaises(execution.ExecutionStopped):
|
|
with self.ex.admission():
|
|
self.fail('paused execution dispatched')
|
|
with patch.object(execution, 'marker_processes', return_value=[]):
|
|
state = self.ex.resume()
|
|
self.assertEqual(state['execution_id'], before['execution_id'])
|
|
self.assertEqual(state['deadline_monotonic'], before['deadline_monotonic'])
|
|
self.assertEqual(state['epoch'], before['epoch'] + 1)
|
|
self.assertEqual(state['auto_resume_count'], 1)
|
|
with self.assertRaises(execution.ExecutionStopped):
|
|
with stale.admission():
|
|
self.fail('old epoch survived takeover')
|
|
|
|
def test_incomplete_or_changed_checkpoint_cannot_resume(self):
|
|
with self.assertRaisesRegex(execution.ExecutionStopped, 'INCOMPLETE'):
|
|
self.ex.pause('quota', {})
|
|
checkpoint = self.checkpoint()
|
|
self.ex.pause('quota', checkpoint)
|
|
Path(checkpoint['path']).write_text('changed')
|
|
with self.assertRaisesRegex(execution.ExecutionStopped, 'HASH_MISMATCH'):
|
|
self.ex.resume()
|
|
|
|
def test_auto_resume_budget_survives_new_controller_objects(self):
|
|
checkpoint = self.checkpoint()
|
|
with patch.object(execution, 'marker_processes', return_value=[]):
|
|
for _ in range(3):
|
|
self.ex.pause('quota', checkpoint)
|
|
self.ex = execution.Execution(self.rd).attach()
|
|
self.ex.resume()
|
|
self.ex.pause('quota', checkpoint)
|
|
with self.assertRaisesRegex(execution.ExecutionStopped, 'AUTO_RESUME_BUDGET_EXHAUSTED'):
|
|
execution.Execution(self.rd).resume()
|
|
|
|
@unittest.skipUnless(Path('/proc').is_dir(), 'Actual worker requires Linux owner identity')
|
|
def test_worker_executes_real_protected_stage_and_reuses_receipt(self):
|
|
from harness import stageworker, stagerun
|
|
import yaml
|
|
root = self.rd / 'fixture-project'
|
|
root.mkdir()
|
|
source = Path(__file__).resolve().parents[1]
|
|
(root/'harness').symlink_to(source/'harness', target_is_directory=True)
|
|
(root/'registry').symlink_to(source/'registry', target_is_directory=True)
|
|
cfg = {'paths': {'harness_python': sys.executable}, 'pools': {'cpu': {'workers': 1}}}
|
|
(root/'config.yaml').write_text(yaml.safe_dump(cfg))
|
|
weights = root/'cache/weights/fixture-sha'
|
|
weights.mkdir(parents=True)
|
|
(weights.parent/'index.json').write_text('{"fixture":{"sha256":"fixture-sha"}}')
|
|
rd = root/'runs/r/j/rollouts/00'
|
|
(rd/'handles/h1').mkdir(parents=True)
|
|
(rd/'workspace').mkdir()
|
|
(rd.parent.parent/'job.json').write_text(json.dumps({'run_id':'r', 'job_id':'j',
|
|
'model':'fixture', 'task':'cls', 'march':'nash-p', 'backend':'ptq'}))
|
|
(rd/'handles/h1/request.json').write_text('{"stage":"01-fetch"}')
|
|
owner = execution.Execution(rd)
|
|
fixture_policy = policy(); fixture_policy['phases']['01-fetch']['queue_deadline_s'] = 2
|
|
owner.start(fixture_policy)
|
|
connect = stagerun.db.connect
|
|
with patch.object(stagerun.db, 'connect', lambda **kw: connect(root/'state/jobs.sqlite', **kw)), \
|
|
patch.object(stagerun.paths, 'BENCH_ROOT', root), \
|
|
patch.object(stagerun.paths, 'config', return_value=cfg), \
|
|
patch.dict(os.environ, {'BENCH_CONFIG': str(root/'config.yaml'), 'BENCH_STATE_DIR': str(root/'state')}):
|
|
worker = stageworker.Worker(rd, execution=owner)
|
|
worker.scan()
|
|
status = json.loads((rd/'handles/h1/status.json').read_text())
|
|
self.assertEqual(status['result'], 'SKIPPED', status)
|
|
self.assertEqual(owner.snapshot()['operations']['h1']['state'], 'DONE')
|
|
with patch.object(stageworker.watchdog, 'run') as replay:
|
|
stageworker.Worker(rd, execution=owner).scan()
|
|
replay.assert_not_called()
|
|
self.assertFalse(list((root/'state/locks').glob('*.lease.json')))
|
|
|
|
def test_receipt_cannot_bind_reused_supervisor_pid(self):
|
|
binding = dict(proof='waitpid:ECHILD', supervisor=dict(pid=123, proc_start_ticks=1))
|
|
receipt = self.rd/'receipt.json'
|
|
execution.atomic_json(receipt, dict(binding=dict(binding, supervisor=dict(pid=123, proc_start_ticks=2)),
|
|
cleanup_state='VERIFIED', all_children_reaped=True, owner_lost=False, exit_code=0))
|
|
self.assertIsNone(execution.scope_receipt(dict(binding=binding, receipt_path=str(receipt))))
|
|
|
|
def test_board_exception_preserves_lease_and_quarantines_before_release(self):
|
|
from contextlib import nullcontext
|
|
from types import SimpleNamespace
|
|
from unittest.mock import Mock
|
|
from harness import stagerun
|
|
hd = self.rd/'stages/_queue/h1'; hd.mkdir(parents=True)
|
|
(hd/'request.json').write_text('{"stage":"07-board"}')
|
|
with self.ex.mutation() as state:
|
|
operation = dict(operation_id='h1', payload_sha256='fixture', execution_id=state['execution_id'],
|
|
epoch=state['epoch'], state='DISPATCHED')
|
|
state['operations']['h1'] = operation
|
|
self.ex._save(state, 'TEST_DISPATCHED')
|
|
(hd/'operation.json').write_text(json.dumps(operation))
|
|
class Board:
|
|
name, pool, level = '07-board', 'board', 'job'
|
|
def run(self, ctx):
|
|
raise RuntimeError('transport failed after dispatch')
|
|
ctx = SimpleNamespace(job={'model':'m','march':'nash-p','job_id':'j'},extra={},
|
|
run_id='r',rollout=0,lane='normal',agent_config=None)
|
|
with patch.object(stagerun, '_ctx_from_rollout', return_value=ctx), \
|
|
patch.dict(stagerun.BY_NAME, {'07-board':Board}), \
|
|
patch.object(stagerun.db, 'connect', side_effect=lambda:nullcontext(Mock())), \
|
|
patch.object(stagerun.db, 'stage_start'), \
|
|
patch.object(scheduler, 'board_slots', return_value={0:{'name':'fixture','march':'nash-p'}}), \
|
|
patch.object(scheduler.paths, 'STATE', self.rd/'state'), \
|
|
patch.object(scheduler.paths, 'config', return_value={'pools':{'board':{'reserve_for_agent':0}}}), \
|
|
self.ex.scope('07-board', 'h1'):
|
|
with self.assertRaisesRegex(RuntimeError, 'transport'):
|
|
stagerun._exec_impl(hd, rd=self.rd, protected=True)
|
|
self.assertEqual(self.ex.snapshot()['cleanup_state'], 'QUARANTINED')
|
|
self.assertTrue((self.rd/'state/locks/board-0.lease.json').exists())
|
|
self.assertTrue((self.rd/'state/locks/board-0.quarantine.json').exists())
|
|
|
|
def test_stale_owner_cannot_mutate_or_cleanup(self):
|
|
old = execution.Execution(self.rd).attach()
|
|
self.ex.cancel('fixture')
|
|
state = self.ex.snapshot(); state['epoch'] += 1; self.ex.epoch = state['epoch']
|
|
execution.atomic_json(self.ex.path, state)
|
|
before = self.ex.path.read_bytes()
|
|
actions = [lambda:old.quarantine('late'), lambda:old.cleanup(), lambda:old.cancel('late'),
|
|
lambda:old.activity('late'), lambda:old.finish(0, protocol_complete=True)]
|
|
for action in actions:
|
|
with self.assertRaisesRegex(execution.ExecutionStopped, 'OWNER_FENCED'):
|
|
action()
|
|
self.assertEqual(self.ex.path.read_bytes(), before)
|
|
|
|
def test_unknown_owner_blocks_admission_but_deadline_still_expires(self):
|
|
with patch.object(execution, 'process_status', return_value='UNKNOWN'):
|
|
with self.assertRaisesRegex(execution.ExecutionStopped, 'OWNER_STATUS_UNKNOWN'):
|
|
with self.ex.admission():
|
|
self.fail('unknown owner admitted')
|
|
self.assertIsNone(self.ex.stop_reason())
|
|
state = self.ex.snapshot(); state['deadline_monotonic'] = time.monotonic() - 1
|
|
execution.atomic_json(self.ex.path, state)
|
|
self.assertEqual(self.ex.stop_reason(), 'EXECUTION_DEADLINE_EXCEEDED')
|
|
|
|
def test_phase_queue_and_execution_clocks_are_separate_and_supervised(self):
|
|
owner = execution.Execution(self.rd/'phase')
|
|
cfg = policy(); cfg['phases']['05-compile'].update(queue_deadline_s=2, execution_deadline_s=.1)
|
|
owner.start(cfg)
|
|
owner.begin_phase('h1', '05-compile')
|
|
self.assertIsNone(owner.snapshot()['phases']['h1']['execution_deadline_monotonic'])
|
|
time.sleep(.12) # More than the execution budget, still legitimately queued.
|
|
self.assertIsNone(owner.stop_reason('h1'))
|
|
owner.phase_running('h1')
|
|
with owner.scope('05-compile', 'h1'):
|
|
result = watchdog.run([sys.executable, '-c', 'import time;time.sleep(.4)'], cwd=self.rd,
|
|
stdout=self.rd/'phase.out', stderr=self.rd/'phase.err', execution=owner, role='stage',
|
|
env=dict(os.environ, BENCH_OPERATION_ID='h1'), poll_s=.02, hard_ceiling_s=5)
|
|
self.assertIn('PHASE_EXECUTION_DEADLINE', result.stalled_reason)
|
|
self.assertLess(result.duration_s, .35)
|
|
self.assertIsNotNone(owner.snapshot()['phases']['h1']['execution_deadline_monotonic'])
|
|
|
|
def test_typed_heartbeat_is_not_business_progress(self):
|
|
path = self.rd/'events'; path.touch()
|
|
tracker = watchdog.BusinessLog(path)
|
|
path.write_text('{"type":"heartbeat","state":"RUNNING"}\n')
|
|
self.assertEqual(tracker.sample(), 0)
|
|
with path.open('a') as out:
|
|
out.write('{"type":"message_end","message":{"role":"assistant","content":[]}}\n')
|
|
self.assertGreater(tracker.sample(), 0)
|
|
with path.open('a') as out:
|
|
out.write('unclassified operational text\n')
|
|
self.assertIsNone(tracker.sample())
|
|
|
|
def test_progress_health_is_persisted_and_unknown_not_zero(self):
|
|
signals = dict(cpu_ticks=1, io_bytes=0, business_log_bytes=0, stage_sequence=0)
|
|
self.ex.record_progress('p', signals)
|
|
time.sleep(.12)
|
|
self.ex.record_progress('p', signals)
|
|
self.assertEqual(self.ex.snapshot()['health'], 'SUSPECT')
|
|
self.ex.record_progress('p', dict(signals, io_bytes=None))
|
|
state = self.ex.snapshot()
|
|
self.assertEqual(state['health'], 'UNKNOWN')
|
|
self.assertIsNone(state['progress']['signals']['io_bytes'])
|
|
|
|
def test_submission_replay_retains_operation_identity(self):
|
|
from harness.stageworker import Worker
|
|
rd, owner, worker = self.worker_fixture()
|
|
source = rd/'stages/_queue/source'; source.mkdir(parents=True)
|
|
(source/'recipe.json').write_text('{}')
|
|
(source/'request.json').write_text(json.dumps({'recipe':str(source/'recipe.json')}))
|
|
(rd/'handles/h1/request.json').write_text('{"stage":"noop","submission":"source"}')
|
|
worker.scan()
|
|
record = json.loads((rd/'stages/_queue/h1/operation.json').read_text())
|
|
self.assertIn('payload_sha256', record)
|
|
self.assertEqual(record['execution_id'], owner.snapshot()['execution_id'])
|
|
Worker(rd, test_noop=True, execution=owner).scan()
|
|
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['result'], 'OK')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|