data-gen-bpu/scripts/test_execution.py
bench 4709c7e44c boards: borrow from the RDK Sandbox platform per use, give back after
The platform now hands out boards per request and resets a board on release
(11-07 measured: free again after 42 s, new password, new host key, our key
gone). Boards are no longer ours between uses, so:

- harness/board_lease.py: request by board id (looked up by SSH port), install
  our public key with that lease's password (SSH_ASKPASS, never on a command
  line), release. A board-<slot>.rdk.json marker records each borrow, so a
  crashed holder's board is found and released; boards a person holds on the
  same account are never touched. Token in .env.local (RDK_SANDBOX_TOKEN) stays
  in controller processes; without it boards are used as statically configured.
- scheduler: borrow after taking the slot lock, give back right after use; an
  unavailable board rests 30 s per waiter. The reset replaces the board
  quarantine: receipts/idle-probe release and the orphan-marker loop in
  reconcile_leases become one sweep that gives back a dead holder's board.
- Host keys are no longer pinned (every borrow changes them): board_perf,
  the idle probe, Salty containers stop copying known_hosts.
- health_check: platform status replaces TCP and pinned-key ssh probes; warns
  when a march has no borrowable board or a borrow is older than 2 hours.

The agent's board access is unchanged (bench dev --board, stage run
board/eval); it never sees the token, passwords or which board it gets.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 22:55:19 +08:00

551 lines
30 KiB
Python

"""Lifecycle contracts using only temporary state and owned test processes."""
import os
import signal
import subprocess
import sys
import tempfile
import threading
import json
import hashlib
import fcntl
import time
import unittest
from pathlib import Path
from unittest.mock import patch
from harness import execution, process_reaper, scheduler, watchdog
def policy():
base = dict(queue_deadline_s=.15, execution_deadline_s=10, suspect_after_s=.1,
cancel_grace_s=.1, cleanup_deadline_s=2)
return dict(base, version='test-v1', phases={name: dict(base) for name in execution.PHASES})
class ProcessStatBytesTests(unittest.TestCase):
@staticmethod
def stat(comm=b'worker', *, birth=42, state=b'S', ppid=10):
fields = [state, str(ppid).encode(), b'10', b'10'] + [b'0'] * 16
fields[11:13] = [b'13', b'17']
fields[19] = str(birth).encode()
return b'123 (' + comm + b') ' + b' '.join(fields) + b'\n'
def test_identity_ignores_arbitrary_comm_bytes(self):
for comm in (b'worker', b'a (b) c', b'\xff', '中文线程名截断'.encode()[:15]):
with self.subTest(comm=comm), patch.object(Path, 'read_bytes', return_value=self.stat(comm)):
identity = execution.process_identity(123)
self.assertEqual(identity['status'], 'KNOWN')
self.assertEqual((identity['proc_start_ticks'], identity['ppid'], identity['state'],
identity['cpu_ticks']), (42, 10, 'S', 30))
json.dumps(identity) # Receipts keep their existing text/number schema.
def test_corrupt_identity_is_unknown_and_never_authorizes_a_signal(self):
original = dict(host=execution.socket.gethostname(), boot_id='fixture', pid=123,
proc_start_ticks=42)
for raw in (b'', b'123 no delimiter', self.stat(state=b'\xff'),
self.stat().replace(b'42\n', b'bad\n')):
with self.subTest(raw=raw), patch.object(Path, 'read_bytes', return_value=raw), \
patch.object(Path, 'exists', return_value=True), \
patch.object(execution, 'boot_id', return_value='fixture'), \
patch.object(os, 'pidfd_open', create=True) as opened:
self.assertEqual(execution.process_identity(123)['status'], 'UNKNOWN')
self.assertEqual(execution.process_status(original), 'UNKNOWN')
self.assertFalse(execution.signal_process(original, signal.SIGTERM))
opened.assert_not_called()
def test_reaper_preserves_second_birth_and_parent_check(self):
reaper = process_reaper.Reaper.__new__(process_reaper.Reaper)
reaper.pid, reaper.host, reaper.boot_id, reaper.members = 10, 'fixture', 'fixture', {}
first = self.stat(b'\xff name (nested)')
for second, permitted in ((first, True), (self.stat(birth=43), False),
(self.stat(ppid=11), False)):
with self.subTest(permitted=permitted), \
patch.object(process_reaper, 'children', return_value={123}), \
patch.object(Path, 'read_bytes', side_effect=[first, second]), \
patch.object(os, 'pidfd_open', return_value=99, create=True), \
patch.object(os, 'close') as closed, \
patch.object(signal, 'pidfd_send_signal', create=True) as sent:
reaper.signal(signal.SIGTERM)
self.assertEqual(sent.call_count, int(permitted))
closed.assert_called_once_with(99)
self.assertEqual(reaper.members[(123, 42)]['state'], 'S')
@unittest.skipUnless(sys.platform == 'linux', 'Linux comm and subreaper')
def test_non_utf8_child_name_is_still_identified_and_reaped(self):
with tempfile.TemporaryDirectory() as name:
root = Path(name)
owner = execution.Execution(root); owner.start(policy())
code = ("import ctypes,time;from pathlib import Path;"
"assert ctypes.CDLL(None).prctl(15,b'worker-\\xff',0,0,0)==0;"
"Path('armed').touch();time.sleep(30)")
result = watchdog.run([sys.executable, '-c', code], cwd=root,
stdout=root/'out', stderr=root/'err', execution=owner, poll_s=.01, hard_ceiling_s=3,
stop_check=lambda: 'fixture complete' if (root/'armed').exists() else None)
self.assertEqual(result.stalled_reason, 'fixture complete')
self.assertEqual(result.cleanup_state, 'VERIFIED')
for identity in owner.snapshot()['processes']:
self.assertNotEqual(execution.process_status(identity), 'ALIVE')
class ExecutionTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.rd = Path(self.tmp.name)
self.ex = execution.Execution(self.rd)
self.ex.start(policy())
if not Path('/proc').is_dir():
# Production admission requires Linux identity; mock only this known
# test controller for portable pure state-machine checks.
original = execution.process_status
owner = execution.process_key(self.ex.snapshot()['owner'])
override = patch.object(execution, 'process_status', side_effect=lambda identity:
'ALIVE' if execution.process_key(identity) == owner else original(identity))
override.start(); self.addCleanup(override.stop)
def test_cancel_fences_spawn_and_cannot_be_completed(self):
self.ex.cancel('test cancel')
with self.assertRaises(execution.ExecutionStopped):
with self.ex.admission():
self.fail('cancelled admission opened')
result = self.ex.finish(0, protocol_complete=True)
self.assertEqual(result['execution_outcome'], 'CANCELLED')
self.assertEqual(result['cleanup_state'], 'VERIFIED')
@unittest.skipUnless(sys.platform == 'linux', 'real owned process cleanup')
def test_worker_drain_uses_stage_deadline_not_cleanup_budget(self):
from harness.stageworker import Worker
owner = execution.Execution(self.rd / 'drain')
cfg = policy(); cfg.update(cleanup_deadline_s=.05, cancel_grace_s=.01)
owner.start(cfg)
worker = Worker(owner.rd, execution=owner)
started, results = threading.Event(), []
def stage():
with owner.scope('05-compile', 'h1'):
owner.begin_phase('h1', '05-compile'); owner.phase_running('h1')
results.append(watchdog.run([sys.executable, '-c', 'import time;time.sleep(.3)'],
cwd=owner.rd, stdout=owner.rd/'out', stderr=owner.rd/'err', execution=owner,
role='stage', poll_s=.01, on_start=lambda _: started.set()))
worker.thread = threading.Thread(target=stage)
with worker:
self.assertTrue(started.wait(3))
self.assertEqual(results[0].state, 'OK')
self.assertGreater(results[0].duration_s, cfg['cleanup_deadline_s'])
self.assertFalse(owner.snapshot()['cancel_requested'])
self.assertEqual(results[0].cleanup_state, 'VERIFIED')
@unittest.skipUnless(sys.platform == 'linux', 'real owned process cleanup')
def test_worker_drain_keeps_explicit_cancel_and_exception_cleanup(self):
from harness.stageworker import Worker
for explicit in (True, False):
with self.subTest(explicit=explicit):
owner = execution.Execution(self.rd / str(explicit)); owner.start(policy())
worker = Worker(owner.rd, execution=owner)
started, results = threading.Event(), []
def stage():
with owner.scope('05-compile', 'h1'):
owner.begin_phase('h1', '05-compile'); owner.phase_running('h1')
results.append(watchdog.run([sys.executable, '-c', 'import time;time.sleep(20)'],
cwd=owner.rd, stdout=owner.rd/'out', stderr=owner.rd/'err', execution=owner,
role='stage', poll_s=.01, on_start=lambda _: started.set()))
worker.thread = threading.Thread(target=stage)
try:
with worker:
self.assertTrue(started.wait(3))
if explicit:
owner.cancel('USER_STOP')
else:
raise ValueError('fixture context failed')
except ValueError as error:
self.assertFalse(explicit)
self.assertEqual(str(error), 'fixture context failed')
self.assertEqual(owner.snapshot()['cancel_requested']['reason'],
'USER_STOP' if explicit else 'WORKER_CONTEXT_ERROR')
self.assertEqual(results[0].cleanup_state, 'VERIFIED')
self.assertEqual(execution.owned_processes(owner.snapshot()), [])
def test_exit_and_protocol_are_independent(self):
self.assertEqual(self.ex.finish(143, protocol_complete=True)['execution_outcome'], 'AGENT_ERROR')
other = execution.Execution(self.rd / 'other'); other.start(policy())
self.assertEqual(other.finish(0, protocol_complete=False)['execution_outcome'], 'AGENT_ERROR')
def test_late_cancel_does_not_overwrite_completion(self):
self.ex.finish(0, protocol_complete=True)
self.assertEqual(self.ex.cancel('late')['reason'], 'ALREADY_TERMINAL')
self.assertEqual(self.ex.snapshot()['execution_outcome'], 'COMPLETED')
def test_old_epoch_is_fenced(self):
stale = execution.Execution(self.rd)
stale.epoch = self.ex.epoch - 1
with self.assertRaises(execution.ExecutionStopped):
with stale.admission():
self.fail('old owner admitted')
def test_queue_wait_is_bounded(self):
with patch.object(scheduler, '_try_slots', return_value=(None, None)), \
patch.object(scheduler, 'pool_size', return_value=1), \
patch.object(scheduler, 'pool_reserve', return_value=0), self.ex.scope('05-compile'):
before = time.monotonic()
with self.assertRaises(execution.ExecutionStopped):
with scheduler.acquire('cpu', poll_s=.01):
self.fail('unavailable slot acquired')
self.assertLess(time.monotonic() - before, 1)
def test_pid_reuse_is_not_alive(self):
ident = execution.process_identity(os.getpid())
if ident.get('status') == 'UNKNOWN':
self.skipTest('Linux proc identity required')
ident['proc_start_ticks'] += 1
self.assertEqual(execution.process_status(ident), 'DEAD')
def test_restart_does_not_reset_deadline_or_replay(self):
state = self.ex.snapshot()
duplicate = execution.Execution(self.rd)
with self.assertRaises(execution.ExecutionStopped):
duplicate.start(policy())
self.assertEqual(duplicate.snapshot()['deadline_monotonic'], state['deadline_monotonic'])
def test_completed_requires_verified_cleanup(self):
result = self.ex.finish(0, protocol_complete=True, cleanup_state='UNKNOWN')
self.assertEqual(result['cleanup_state'], 'QUARANTINED')
self.assertNotEqual(result['execution_outcome'], 'COMPLETED')
def test_cancel_serializes_with_spawn_admission(self):
started, committed = threading.Event(), threading.Event()
def cancel():
started.set()
self.ex.cancel('race fixture')
committed.set()
with self.ex.admission():
thread = threading.Thread(target=cancel)
thread.start(); started.wait(1)
self.assertFalse(committed.wait(.03))
thread.join(1)
self.assertTrue(committed.is_set())
with self.assertRaises(execution.ExecutionStopped):
with self.ex.admission():
self.fail('post-cancel spawn admitted')
def test_crashed_holder_lease_is_not_reallocated(self):
with patch.object(scheduler.paths, 'STATE', self.rd / 'state'), self.ex.scope('05-compile'):
lease, _ = scheduler._try_slots('cpu', [0])
self.assertIsNotNone(lease)
fcntl.flock(lease, fcntl.LOCK_UN); lease.close() # Simulate kernel lock release after a crash.
next_lease, _ = scheduler._try_slots('cpu', [0])
self.assertIsNone(next_lease)
self.assertTrue((self.rd / 'state/locks/cpu-0.lease.json').exists())
def test_unknown_telemetry_does_not_disable_deadline(self):
with patch.object(watchdog, '_proc_tree_stat', return_value=None):
before = time.monotonic()
result = watchdog.run([sys.executable, '-c', 'import time; time.sleep(60)'], cwd=self.rd,
stdout=self.rd/'out', stderr=self.rd/'err', execution=self.ex, poll_s=.01, hard_ceiling_s=.1)
self.assertLess(time.monotonic() - before, 3)
self.assertIn('DEADLINE', result.stalled_reason)
self.assertEqual(result.cleanup_state, 'VERIFIED')
def test_expired_start_bookkeeping_never_releases_command_gate(self):
marker = self.rd / 'must-not-start'
result = watchdog.run([sys.executable, '-c',
"from pathlib import Path;Path('must-not-start').touch()"], cwd=self.rd,
stdout=self.rd/'gate.out', stderr=self.rd/'gate.err', execution=self.ex,
poll_s=.01, hard_ceiling_s=.1, on_start=lambda _: time.sleep(.2))
self.assertIn('DEADLINE', result.stalled_reason)
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertFalse(marker.exists())
self.assertLess(result.duration_s, 3)
def test_predeadline_channel_loss_remains_infrastructure_error(self):
def stop_supervisor(proc):
execution.signal_process(proc.supervisor_identity, signal.SIGTERM)
proc.supervisor.wait(timeout=3)
result = watchdog.run([sys.executable, '-c',
"from pathlib import Path;Path('must-not-start').touch()"], cwd=self.rd,
stdout=self.rd/'closed.out', stderr=self.rd/'closed.err', execution=self.ex,
poll_s=.01, hard_ceiling_s=5, on_start=stop_supervisor)
self.assertIn('runner monitor failed:', result.stalled_reason)
self.assertNotIn('DEADLINE', result.stalled_reason)
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertFalse((self.rd/'must-not-start').exists())
def test_suspect_is_advisory_not_an_idle_kill(self):
with patch.object(watchdog, '_proc_tree_stat', return_value=(100, 0)), \
patch.object(watchdog, '_io_bytes', return_value=0):
result = watchdog.run([sys.executable, '-c', 'import time; time.sleep(.25)'], cwd=self.rd,
stdout=self.rd/'out', stderr=self.rd/'err', execution=self.ex, poll_s=.01)
self.assertEqual(result.state, 'OK')
self.assertTrue(result.suspect)
def worker_fixture(self):
from harness.stageworker import Worker
rd = self.rd / 'guard-smoke' / 'rollout'
(rd / 'handles/h1').mkdir(parents=True)
(rd / 'workspace').mkdir()
owner = execution.Execution(rd); owner.start(policy())
request = {'stage': 'noop', 'submitted_at': 1}
(rd / 'handles/h1/request.json').write_text(json.dumps(request))
return rd, owner, Worker(rd, test_noop=True, execution=owner)
def test_worker_restart_reuses_receipt_without_dispatch(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
worker.scan()
operation = rd / 'stages/_queue/h1/operation.json'
before = operation.read_bytes()
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(operation.read_bytes(), before)
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['result'], 'OK')
def test_worker_payload_conflict_preserves_original_receipt(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
worker.scan()
receipt = rd / 'stages/_queue/h1/status.json'
before = receipt.read_bytes()
(rd / 'handles/h1/request.json').write_text('{"stage":"noop","submitted_at":2}')
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(receipt.read_bytes(), before)
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['error_class'],
'OPERATION_PAYLOAD_CONFLICT')
def test_unknown_operation_is_quarantined_not_replayed(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
worker.scan()
(rd/'stages/_queue/h1/status.json').unlink()
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(owner.snapshot()['cleanup_state'], 'QUARANTINED')
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['error_class'],
'SIDE_EFFECT_UNKNOWN')
def checkpoint(self):
path = self.rd / 'session.jsonl'
path.write_text('{"message_id":"m1"}\n')
return {'complete': True, 'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}
def test_pause_resume_preserves_identity_returns_paused_time_and_fences_previous_epoch(self):
before = self.ex.snapshot()
stale = execution.Execution(self.rd).attach()
self.ex.pause('QUOTA_WAIT_BUDGET_EXHAUSTED', self.checkpoint())
with self.assertRaises(execution.ExecutionStopped):
with self.ex.admission():
self.fail('paused execution dispatched')
paused_at = self.ex.snapshot()['paused_at']
with patch.object(execution, 'marker_processes', return_value=[]), \
patch.object(execution.time, 'time', return_value=paused_at + 4):
state = self.ex.resume()
self.assertEqual(state['execution_id'], before['execution_id'])
self.assertEqual(state['deadline_monotonic'], before['deadline_monotonic'] + 4) # pause handed back
self.assertEqual((state['paused_s'], state.get('paused_at')), (4, None))
self.assertEqual(state['epoch'], before['epoch'] + 1)
self.assertEqual(state['auto_resume_count'], 1)
with self.assertRaises(execution.ExecutionStopped):
with stale.admission():
self.fail('old epoch survived takeover')
def test_long_pause_outlives_original_deadline_but_credit_is_capped(self):
self.ex.pause('CONNECT_FAILED', self.checkpoint())
paused_at = self.ex.snapshot()['paused_at']
with patch.object(execution.time, 'time', return_value=paused_at + 1000), \
patch.object(execution.time, 'monotonic', return_value=self.ex.snapshot()['deadline_monotonic'] + 5), \
patch.object(execution, 'marker_processes', return_value=[]):
self.assertNotEqual(self.ex.stop_reason(), 'EXECUTION_DEADLINE_EXCEEDED')
state = self.ex.resume(counted=False)
self.assertEqual(state['paused_s'], 10) # at most one execution budget (policy: 10 s)
def test_paused_past_deadline_after_reboot_is_deadline_not_resumable(self):
self.ex.pause('CONTROLLER_RESTARTED', self.checkpoint())
state = self.ex.snapshot()
with patch.object(execution, 'boot_id', return_value='next-boot'):
with patch.object(execution.time, 'time', return_value=state['paused_at'] + 1):
self.assertEqual(self.ex.stop_reason(), 'BOOT_CHANGED_RECONCILE_REQUIRED') # still has time
with patch.object(execution.time, 'time', return_value=state['started_at'] + 25): # 10 s budget + 10 s credit
self.assertEqual(self.ex.stop_reason(), 'EXECUTION_DEADLINE_EXCEEDED')
def test_incomplete_or_changed_checkpoint_cannot_resume(self):
with self.assertRaisesRegex(execution.ExecutionStopped, 'INCOMPLETE'):
self.ex.pause('quota', {})
checkpoint = self.checkpoint()
self.ex.pause('quota', checkpoint)
Path(checkpoint['path']).write_text('changed')
with self.assertRaisesRegex(execution.ExecutionStopped, 'HASH_MISMATCH'):
self.ex.resume()
def test_auto_resume_budget_survives_new_controller_objects(self):
checkpoint = self.checkpoint()
with patch.object(execution, 'marker_processes', return_value=[]):
for _ in range(3):
self.ex.pause('quota', checkpoint)
self.ex = execution.Execution(self.rd).attach()
self.ex.resume()
self.ex.pause('quota', checkpoint)
with self.assertRaisesRegex(execution.ExecutionStopped, 'AUTO_RESUME_BUDGET_EXHAUSTED'):
execution.Execution(self.rd).resume()
free = execution.Execution(self.rd)
free.resume(counted=False) # a pause that sent nothing (connect/DNS failure) is free
self.assertEqual(free.snapshot()['auto_resume_count'], 3)
@unittest.skipUnless(Path('/proc').is_dir(), 'Actual worker requires Linux owner identity')
def test_worker_executes_real_protected_stage_and_reuses_receipt(self):
from harness import stageworker, stagerun
import yaml
root = self.rd / 'fixture-project'
root.mkdir()
source = Path(__file__).resolve().parents[1]
(root/'harness').symlink_to(source/'harness', target_is_directory=True)
(root/'registry').symlink_to(source/'registry', target_is_directory=True)
cfg = {'paths': {'harness_python': sys.executable}, 'pools': {'cpu': {'workers': 1}}}
(root/'config.yaml').write_text(yaml.safe_dump(cfg))
weights = root/'cache/weights/fixture-sha'
weights.mkdir(parents=True)
(weights.parent/'index.json').write_text('{"fixture":{"sha256":"fixture-sha"}}')
rd = root/'runs/r/j/rollouts/00'
(rd/'handles/h1').mkdir(parents=True)
(rd/'workspace').mkdir()
(rd.parent.parent/'job.json').write_text(json.dumps({'run_id':'r', 'job_id':'j',
'model':'fixture', 'task':'cls', 'march':'nash-p', 'backend':'ptq'}))
(rd/'handles/h1/request.json').write_text('{"stage":"01-fetch"}')
owner = execution.Execution(rd)
fixture_policy = policy(); fixture_policy['phases']['01-fetch']['queue_deadline_s'] = 2
owner.start(fixture_policy)
connect = stagerun.db.connect
with patch.object(stagerun.db, 'connect', lambda **kw: connect(root/'state/jobs.sqlite', **kw)), \
patch.object(stagerun.paths, 'BENCH_ROOT', root), \
patch.object(stagerun.paths, 'config', return_value=cfg), \
patch.dict(os.environ, {'BENCH_CONFIG': str(root/'config.yaml'), 'BENCH_STATE_DIR': str(root/'state')}):
worker = stageworker.Worker(rd, execution=owner)
worker.scan()
status = json.loads((rd/'handles/h1/status.json').read_text())
self.assertEqual(status['result'], 'SKIPPED', status)
self.assertEqual(owner.snapshot()['operations']['h1']['state'], 'DONE')
# A later attempt may overwrite a settled attempt's outputs; a resumed worker must
# neither replay nor re-prove it (10-13: re-proof quarantined 19 rollouts).
with patch.object(stageworker.watchdog, 'run') as replay, \
patch.object(stageworker.Worker, '_reconcile_stage', side_effect=ValueError('completed artifact changed')):
stageworker.Worker(rd, execution=owner).scan()
replay.assert_not_called()
self.assertIsNone(owner.snapshot().get('stop_reason'))
self.assertFalse(list((root/'state/locks').glob('*.lease.json')))
def test_receipt_cannot_bind_reused_supervisor_pid(self):
binding = dict(proof='waitpid:ECHILD', supervisor=dict(pid=123, proc_start_ticks=1))
receipt = self.rd/'receipt.json'
execution.atomic_json(receipt, dict(binding=dict(binding, supervisor=dict(pid=123, proc_start_ticks=2)),
cleanup_state='VERIFIED', all_children_reaped=True, owner_lost=False, exit_code=0))
self.assertIsNone(execution.scope_receipt(dict(binding=binding, receipt_path=str(receipt))))
def test_board_exception_quarantines_execution_and_gives_board_back(self):
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock
from harness import stagerun
hd = self.rd/'stages/_queue/h1'; hd.mkdir(parents=True)
(hd/'request.json').write_text('{"stage":"07-board"}')
with self.ex.mutation() as state:
operation = dict(operation_id='h1', payload_sha256='fixture', execution_id=state['execution_id'],
epoch=state['epoch'], state='DISPATCHED')
state['operations']['h1'] = operation
self.ex._save(state, 'TEST_DISPATCHED')
(hd/'operation.json').write_text(json.dumps(operation))
class Board:
name, pool, level = '07-board', 'board', 'job'
def run(self, ctx):
raise RuntimeError('transport failed after dispatch')
ctx = SimpleNamespace(job={'model':'m','march':'nash-p','job_id':'j'},extra={},
run_id='r',rollout=0,lane='normal',agent_config=None)
with patch.object(stagerun, '_ctx_from_rollout', return_value=ctx), \
patch.dict(stagerun.BY_NAME, {'07-board':Board}), \
patch.object(stagerun.db, 'connect', side_effect=lambda:nullcontext(Mock())), \
patch.object(stagerun.db, 'stage_start'), \
patch.object(scheduler, 'board_slots', return_value={0:{'name':'fixture','march':'nash-p'}}), \
patch.object(scheduler.paths, 'STATE', self.rd/'state'), \
patch.object(scheduler.paths, 'config', return_value={'pools':{'board':{'reserve_for_agent':0}}}), \
patch.object(scheduler.board_lease, 'enabled', return_value=False), \
self.ex.scope('07-board', 'h1'):
with self.assertRaisesRegex(RuntimeError, 'transport'):
stagerun._exec_impl(hd, rd=self.rd, protected=True)
# The rollout stays quarantined; the board itself is given back (the platform reset cleans it, 11-07).
self.assertEqual(self.ex.snapshot()['cleanup_state'], 'QUARANTINED')
self.assertEqual(list((self.rd/'state/locks').glob('board-0.*.json')), [])
def test_stale_owner_cannot_mutate_or_cleanup(self):
old = execution.Execution(self.rd).attach()
self.ex.cancel('fixture')
state = self.ex.snapshot(); state['epoch'] += 1; self.ex.epoch = state['epoch']
execution.atomic_json(self.ex.path, state)
before = self.ex.path.read_bytes()
actions = [lambda:old.quarantine('late'), lambda:old.cleanup(), lambda:old.cancel('late'),
lambda:old.activity('late'), lambda:old.finish(0, protocol_complete=True)]
for action in actions:
with self.assertRaisesRegex(execution.ExecutionStopped, 'OWNER_FENCED'):
action()
self.assertEqual(self.ex.path.read_bytes(), before)
def test_unknown_owner_blocks_admission_but_deadline_still_expires(self):
with patch.object(execution, 'process_status', return_value='UNKNOWN'):
with self.assertRaisesRegex(execution.ExecutionStopped, 'OWNER_STATUS_UNKNOWN'):
with self.ex.admission():
self.fail('unknown owner admitted')
self.assertIsNone(self.ex.stop_reason())
state = self.ex.snapshot(); state['deadline_monotonic'] = time.monotonic() - 1
execution.atomic_json(self.ex.path, state)
self.assertEqual(self.ex.stop_reason(), 'EXECUTION_DEADLINE_EXCEEDED')
def test_phase_queue_and_execution_clocks_are_separate_and_supervised(self):
owner = execution.Execution(self.rd/'phase')
cfg = policy(); cfg['phases']['05-compile'].update(queue_deadline_s=2, execution_deadline_s=.1)
owner.start(cfg)
owner.begin_phase('h1', '05-compile')
self.assertIsNone(owner.snapshot()['phases']['h1']['execution_deadline_monotonic'])
time.sleep(.12) # More than the execution budget, still legitimately queued.
self.assertIsNone(owner.stop_reason('h1'))
owner.phase_running('h1')
with owner.scope('05-compile', 'h1'):
result = watchdog.run([sys.executable, '-c', 'import time;time.sleep(.4)'], cwd=self.rd,
stdout=self.rd/'phase.out', stderr=self.rd/'phase.err', execution=owner, role='stage',
env=dict(os.environ, BENCH_OPERATION_ID='h1'), poll_s=.02, hard_ceiling_s=5)
self.assertIn('PHASE_EXECUTION_DEADLINE', result.stalled_reason)
self.assertLess(result.duration_s, .35)
self.assertIsNotNone(owner.snapshot()['phases']['h1']['execution_deadline_monotonic'])
def test_typed_heartbeat_is_not_business_progress(self):
path = self.rd/'events'; path.touch()
tracker = watchdog.BusinessLog(path)
path.write_text('{"type":"heartbeat","state":"RUNNING"}\n')
self.assertEqual(tracker.sample(), 0)
with path.open('a') as out:
out.write('{"type":"message_end","message":{"role":"assistant","content":[]}}\n')
self.assertGreater(tracker.sample(), 0)
with path.open('a') as out:
out.write('unclassified operational text\n')
self.assertIsNone(tracker.sample())
def test_progress_health_is_persisted_and_unknown_not_zero(self):
signals = dict(cpu_ticks=1, io_bytes=0, business_log_bytes=0, stage_sequence=0)
self.ex.record_progress('p', signals)
time.sleep(.12)
self.ex.record_progress('p', signals)
self.assertEqual(self.ex.snapshot()['health'], 'SUSPECT')
self.ex.record_progress('p', dict(signals, io_bytes=None))
state = self.ex.snapshot()
self.assertEqual(state['health'], 'UNKNOWN')
self.assertIsNone(state['progress']['signals']['io_bytes'])
def test_submission_replay_retains_operation_identity(self):
from harness.stageworker import Worker
rd, owner, worker = self.worker_fixture()
source = rd/'stages/_queue/source'; source.mkdir(parents=True)
(source/'recipe.json').write_text('{}')
(source/'request.json').write_text(json.dumps({'recipe':str(source/'recipe.json')}))
(rd/'handles/h1/request.json').write_text('{"stage":"noop","submission":"source"}')
worker.scan()
record = json.loads((rd/'stages/_queue/h1/operation.json').read_text())
self.assertIn('payload_sha256', record)
self.assertEqual(record['execution_id'], owner.snapshot()['execution_id'])
Worker(rd, test_noop=True, execution=owner).scan()
self.assertEqual(json.loads((rd/'handles/h1/status.json').read_text())['result'], 'OK')
if __name__ == '__main__':
unittest.main()