data-gen-bpu/scripts/test_watchdog_stop.py
bench 0da66edde0 Resume rollouts after the collector dies (crash, kill, reboot, planned handover)
Processes are disposable; rollout state is resumable. When the collector dies,
the next one reconciles each running rollout from durable state instead of
quarantining or losing it:

- execution: a process of an earlier boot on this host is DEAD; a top-level
  reaper whose owner (the collector) died still proves its subtree empty
  (scopes record the owner that started them); reconcile() pauses a rollout
  with a proven-clean crash checkpoint as CONTROLLER_RESTARTED and marks its
  running tool calls INTERRUPTED; resume() across a reboot rebases monotonic
  clocks by wall time and credits the downtime as a pause
- stageworker.settle_interrupted: an admitted step whose executor is gone ends
  as STAGE_EXECUTOR_EXIT (free compile) and its handle is told directly; a
  board step without a clean board receipt still quarantines; nothing replays
- collection_runtime.crash_checkpoint: capture the real system prompt, freeze
  the session at the last saved message (mid-work phases only); configure_pi
  skips a generation ID the ledger already completed
- pi_session.js: checkpoint after every saved message (and compaction); on a
  recovered resume drop an unfinished trailing reply and answer open tool
  calls with an honest "interrupted by a platform restart" result
- scheduler: a paused rollout's local slot held by a dead holder is released
  once every process is proven gone
- pipeline: SIGUSR1 = planned handover (stop claiming, let board steps
  finish, shorten in-flight request holds, exit); SIGTERM still drains
- export: bench_recovery closes the cut segment; the dropped unfinished reply
  is excluded (UNFINISHED_BEFORE_RESTART)
- tests: test_controller_restart (real Pi on Linux: kill -9 mid-command,
  kill -9 mid-reply, SIGUSR1 handover, power loss + reboot), test_settle_
  interrupted; three tests of the old quarantine-everything rule now assert
  the new outcome while keeping their guarantees (no replay, unrelated PID
  untouched, board lease kept)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 09:29:26 +08:00

437 lines
24 KiB
Python

"""Exercise actual owned subprocess termination; no external workloads."""
import signal
import json
from contextlib import contextmanager
import os
import subprocess
import sys
import tempfile
import time
import unittest
from pathlib import Path
from unittest.mock import patch
from harness import watchdog
class ProcTelemetryTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.proc = Path(self.temp.name)
self.child_pids = {}
self.patch_path = patch.object(watchdog, 'Path', return_value=self.proc)
self.patch_path.start(); self.addCleanup(self.patch_path.stop)
self.patch_children = patch.object(watchdog, 'children', side_effect=lambda pid: self.child_pids[pid])
self.patch_children.start(); self.addCleanup(self.patch_children.stop)
@staticmethod
def stat(pid=1, comm=b'worker', state=b'S', ticks=(3, 4)):
fields = [state] + [b'0'] * 19
fields[11:13] = [str(n).encode() for n in ticks]
return str(pid).encode() + b' (' + comm + b') ' + b' '.join(fields) + b'\n'
def process(self, pid=1, **kwargs):
pdir = self.proc / str(pid)
tdir = pdir / 'task' / str(pid)
tdir.mkdir(parents=True)
raw = self.stat(pid, **kwargs)
(pdir / 'stat').write_bytes(raw)
(tdir / 'stat').write_bytes(raw)
self.child_pids[pid] = []
return pdir
def test_comm_is_bytes_including_truncated_utf8_spaces_and_parentheses(self):
pdir = self.process(state=b'D')
for comm in (b'raw\xff', '中文线程'.encode()[:-1], b'a ) ( b', b'a\nb'):
with self.subTest(comm=comm):
raw = self.stat(comm=comm, state=b'D')
(pdir / 'stat').write_bytes(raw)
(pdir / 'task' / '1' / 'stat').write_bytes(raw)
self.assertEqual(watchdog._proc_tree_stat(1), (7, 1))
def test_excluded_supervisor_still_walks_descendants_and_ignores_unrelated_pid(self):
self.process(1, ticks=(100, 100), state=b'D')
self.process(2, state=b'D')
self.child_pids[1] = [2]
(self.proc / '99').mkdir()
(self.proc / '99' / 'stat').write_bytes(b'broken unrelated process')
self.assertEqual(watchdog._proc_tree_stat(1, exclude={1}), (7, 1))
def test_missing_thread_is_ignored_but_empty_sample_is_unknown(self):
pdir = self.process(state=b'D')
(pdir / 'task' / '999').mkdir()
self.assertEqual(watchdog._d_threads(pdir), 1)
(pdir / 'task' / '1' / 'stat').unlink()
self.assertIsNone(watchdog._d_threads(pdir))
self.assertIsNone(watchdog._proc_tree_stat(1))
def test_truncated_or_bad_required_fields_are_unknown(self):
pdir = self.process()
target = pdir / 'stat'
for raw in (b'', b'1 (short', b'1 (short) S', self.stat(ticks=('bad', 1))):
with self.subTest(raw=raw):
target.write_bytes(raw)
self.assertIsNone(watchdog._proc_tree_stat(1))
target.write_bytes(self.stat())
for raw in (b'', b'1 (short)', self.stat(state=b'?')):
with self.subTest(thread_raw=raw):
(pdir / 'task' / '1' / 'stat').write_bytes(raw)
self.assertIsNone(watchdog._proc_tree_stat(1))
def test_target_read_failures_and_disappeared_child_are_unknown(self):
pdir = self.process()
self.process(2)
self.child_pids[1] = [2]
original = Path.read_bytes
for target in (pdir / 'stat', pdir / 'task' / '1' / 'stat'):
with self.subTest(target=target):
def denied(path):
if path == target:
raise PermissionError('fixture unreadable proc entry')
return original(path)
with patch.object(Path, 'read_bytes', denied):
self.assertIsNone(watchdog._proc_tree_stat(1))
(self.proc / '2' / 'stat').unlink()
self.assertIsNone(watchdog._proc_tree_stat(1))
def test_unreadable_children_and_inconsistent_tree_are_unknown(self):
self.process()
with patch.object(watchdog, 'children', side_effect=PermissionError('fixture')):
self.assertIsNone(watchdog._proc_tree_stat(1))
self.child_pids[1] = [1]
self.assertIsNone(watchdog._proc_tree_stat(1))
class WatchdogStopTests(unittest.TestCase):
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux process telemetry')
def test_supervisor_work_is_not_business_cpu_progress(self):
# Other tests may leave a live multiprocessing resource tracker in the
# suite process. A fresh child isolates the supervisor-only assertion.
code = ('import os;from harness import watchdog;'
'assert watchdog._proc_tree_stat(os.getpid())[0] > 0;'
'assert watchdog._proc_tree_stat(os.getpid(),exclude={os.getpid()}) == (0,0)')
result = subprocess.run([sys.executable, '-c', code],
cwd=Path(__file__).resolve().parents[1], capture_output=True, text=True, timeout=5)
self.assertEqual(result.returncode, 0, result.stderr)
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux subreaper verification')
def test_double_fork_cleared_environment_and_session_is_fully_reaped(self):
from harness import execution
from scripts.test_execution import policy
with tempfile.TemporaryDirectory() as name:
root = Path(name)
owner = execution.Execution(root); owner.start(policy())
# Both original parents disappear between watchdog observations.
code = """import os,signal,time
time.sleep(.15)
r,w=os.pipe()
if os.fork():
os.close(w)
open('child.pid','wb').write(os.read(r,100))
os._exit(0)
os.close(r); os.environ.clear(); os.setsid()
if os.fork(): os._exit(0)
signal.signal(signal.SIGTERM,signal.SIG_IGN)
os.write(w,str(os.getpid()).encode()); os.close(w)
time.sleep(30)
"""
identity = None
try:
result = watchdog.run([sys.executable, '-c', code], cwd=root,
stdout=root/'out', stderr=root/'err', execution=owner, poll_s=30)
identity = execution.process_identity(int((root/'child.pid').read_text()))
self.assertNotEqual(execution.process_status(identity), 'ALIVE')
self.assertEqual(result.cleanup_state, 'VERIFIED')
finally:
if identity and identity['status'] == 'KNOWN':
execution.signal_process(identity, signal.SIGKILL)
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux subreaper verification')
def test_sigkill_is_reaped_before_reporting(self):
from harness import execution
from scripts.test_execution import policy
with tempfile.TemporaryDirectory() as name:
root = Path(name); owner = execution.Execution(root); owner.start(policy())
result = watchdog.run([sys.executable, '-c',
'import signal,time;from pathlib import Path;'
'signal.signal(signal.SIGTERM,signal.SIG_IGN);Path("armed").touch();time.sleep(30)'],
cwd=root, stdout=root/'out', stderr=root/'err', execution=owner, hard_ceiling_s=2, poll_s=.01,
stop_check=lambda: 'fixture handler armed' if (root/'armed').exists() else None)
self.assertTrue((root/'armed').exists())
self.assertEqual(result.stalled_reason, 'fixture handler armed')
self.assertLess(result.duration_s, 3)
self.assertEqual(result.exit_code, -signal.SIGKILL)
self.assertEqual(result.cleanup_state, 'VERIFIED')
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux subreaper verification')
def test_monitor_exception_stops_only_owned_child_and_redacts_diagnostic(self):
with tempfile.TemporaryDirectory() as name:
root = Path(name)
secret = b'fixture-private-request-body-123'
def broken():
raise UnicodeDecodeError('utf-8', secret + b'\xff', len(secret), len(secret) + 1,
'invalid start byte')
with patch.object(watchdog, "_proc_tree_stat", return_value=None):
result = watchdog.run([sys.executable, "-c", "import time; time.sleep(60)"], cwd=root,
stdout=root / "out", stderr=root / "err", poll_s=0.01, stop_check=broken)
self.assertEqual(result.state, "STALLED")
self.assertIsNotNone(result.exit_code)
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertIn("monitor failed", result.stalled_reason)
stderr = (root / 'err').read_text()
self.assertNotIn(secret.decode(), stderr)
row = next(line for line in stderr.splitlines() if line.startswith('runner monitor diagnostic: '))
diagnostic = json.loads(row.split(': ', 1)[1])
self.assertEqual(diagnostic['encoding'], 'utf-8')
self.assertEqual(diagnostic['start'], len(secret))
self.assertEqual(diagnostic['end'], len(secret) + 1)
self.assertEqual(diagnostic['reason'], 'invalid start byte')
self.assertEqual([f['function'] for f in diagnostic['frames']], ['run', 'broken'])
self.assertTrue(all(set(f) == {'file', 'line', 'function'} for f in diagnostic['frames']))
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux subreaper verification')
def test_failed_diagnostic_write_does_not_block_cleanup(self):
with tempfile.TemporaryDirectory() as name:
root = Path(name)
attempted = []
class DiagnosticFailure:
def __init__(self, stream):
self.stream = stream
def __getattr__(self, name):
return getattr(self.stream, name)
def write(self, data):
if data.startswith(b'\nrunner monitor diagnostic: '):
attempted.append(True)
raise OSError('fixture diagnostic disk failure')
return self.stream.write(data)
@contextmanager
def output(path, mode):
with open(path, mode) as stream:
yield DiagnosticFailure(stream) if path == root / 'err' else stream
def broken():
raise ValueError('fixture private exception message')
with patch.object(watchdog, 'open', output, create=True):
result = watchdog.run([sys.executable, '-c', 'import time;time.sleep(60)'],
cwd=root, stdout=root / 'out', stderr=root / 'err', stop_check=broken)
self.assertEqual(attempted, [True])
self.assertEqual(result.state, 'STALLED')
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertIsNotNone(result.exit_code)
self.assertNotIn('fixture private exception message', (root / 'err').read_text())
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux subreaper verification')
def test_supervisor_sigkill_and_missing_receipt_are_quarantined(self):
from harness import execution
from scripts.test_execution import policy
with tempfile.TemporaryDirectory() as name:
root = Path(name); owner = execution.Execution(root); owner.start(policy())
def kill_supervisor():
scope = next(iter(owner.snapshot()['process_scopes'].values()))
execution.signal_process(scope['binding']['supervisor'], signal.SIGKILL)
try:
result = watchdog.run([sys.executable, '-c', 'import time;time.sleep(30)'],
cwd=root, stdout=root/'out', stderr=root/'err', execution=owner,
poll_s=.01, stop_check=kill_supervisor)
self.assertEqual(result.cleanup_state, 'QUARANTINED')
self.assertIsNone(execution.owned_processes(owner.snapshot()))
finally:
for identity in owner.snapshot()['processes']:
execution.signal_process(identity, signal.SIGKILL)
# No visible live PID does not manufacture the missing kernel proof.
self.assertEqual(owner.finish(0, protocol_complete=True)['cleanup_state'], 'QUARANTINED')
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux subreaper verification')
def test_controller_sigkill_cleans_children_and_its_receipt_proves_it(self):
from harness import execution
with tempfile.TemporaryDirectory() as name:
root = Path(name)
command = ("import subprocess,sys,time;from pathlib import Path;"
"p=subprocess.Popen([sys.executable,'-c','import time;time.sleep(30)'],"
"env={},start_new_session=True);Path('child.pid').write_text(str(p.pid));time.sleep(30)")
code = ("from pathlib import Path;import sys;from harness import execution,watchdog;"
"from scripts.test_execution import policy;"
f"root=Path({name!r});owner=execution.Execution(root);owner.start(policy());"
f"watchdog.run([sys.executable,'-c',{command!r}],cwd=root,stdout=root/'out',"
"stderr=root/'err',execution=owner,poll_s=.01)")
controller = subprocess.Popen([sys.executable, '-c', code])
child = None
try:
deadline = time.monotonic() + 3
while not (root/'child.pid').exists() and time.monotonic() < deadline:
time.sleep(.01)
child = execution.process_identity(int((root/'child.pid').read_text()))
owner = execution.Execution(root)
scope = next(iter(owner.snapshot()['process_scopes'].values()))
controller.kill(); controller.wait(timeout=3)
while not Path(scope['receipt_path']).exists() and time.monotonic() < deadline:
time.sleep(.01)
receipt = execution.scope_receipt(scope)
self.assertTrue(receipt['all_children_reaped'])
self.assertTrue(receipt['owner_lost'])
self.assertNotEqual(execution.process_status(child), 'ALIVE')
# The controller's own top-level reaper killed and reaped everything (ECHILD): that is proof, so the
# dead owner's execution ends as lost with verified cleanup instead of staying quarantined.
record = owner.reconcile()
self.assertEqual((record['execution_state'], record['execution_outcome'], record['cleanup_state']),
('TERMINAL', 'LOST', 'VERIFIED'))
finally:
if controller.poll() is None:
controller.kill(); controller.wait(timeout=3)
if child:
execution.signal_process(child, signal.SIGKILL)
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux subreaper verification')
def test_cancel_after_registration_never_releases_command_gate(self):
from harness import execution
from scripts.test_execution import policy
with tempfile.TemporaryDirectory() as name:
root = Path(name); owner = execution.Execution(root); owner.start(policy())
result = watchdog.run([sys.executable, '-c', "open('must-not-run','w').write('bad')"],
cwd=root, stdout=root/'out', stderr=root/'err', execution=owner,
on_start=lambda _proc: owner.cancel('cancel before exec'))
self.assertFalse((root/'must-not-run').exists())
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertEqual(result.stalled_reason, 'CANCEL_REQUESTED')
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux subreaper verification')
def test_namespace_exit_reaps_detached_child_with_same_host_identity(self):
from harness import execution
from scripts.test_execution import policy
with tempfile.TemporaryDirectory() as name:
root = Path(name); owner = execution.Execution(root); owner.start(policy())
code = ("import os,time;pid=os.fork();"
"os._exit(0) if pid else None;os.setsid();os.environ.clear();time.sleep(30)")
result = watchdog.run(['unshare', '-Urmpf', '--mount-proc', '--kill-child=KILL',
sys.executable, '-c', code], cwd=root, stdout=root/'out', stderr=root/'err', execution=owner)
self.assertEqual(result.state, 'OK', (root/'err').read_text())
self.assertEqual(result.cleanup_state, 'VERIFIED')
scopes = owner.snapshot()['process_scopes']
self.assertTrue(all(execution.scope_receipt(s)['all_children_reaped'] for s in scopes.values()))
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux identity verification')
def test_exited_leader_does_not_hide_detached_term_ignoring_child(self):
from harness import execution
from scripts.test_execution import policy
with tempfile.TemporaryDirectory() as name:
root = Path(name)
owner = execution.Execution(root); owner.start(policy())
code = """import os,signal,time
pid=os.fork()
if pid:
time.sleep(.1)
os._exit(0)
os.setsid()
signal.signal(signal.SIGTERM, signal.SIG_IGN)
open('child.pid','w').write(str(os.getpid()))
time.sleep(60)
"""
result = watchdog.run([sys.executable, '-c', code], cwd=root,
stdout=root/'out', stderr=root/'err', poll_s=.01, execution=owner)
child = execution.process_identity(int((root/'child.pid').read_text()))
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertNotEqual(execution.process_status(child), 'ALIVE')
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux identity verification')
def test_cleared_environment_descendants_are_tracked_and_reaped(self):
from harness import execution
from scripts.test_execution import policy
for detached in (False, True):
with self.subTest(detached=detached), tempfile.TemporaryDirectory() as name:
root = Path(name)
owner = execution.Execution(root); owner.start(policy())
code = ("import subprocess,sys,time;from pathlib import Path;"
"p=subprocess.Popen([sys.executable,'-c','import time;time.sleep(30)'],"
f"env={{}},start_new_session={detached!r});"
"Path('child.pid').write_text(str(p.pid));time.sleep(30)")
identity = None
try:
result = watchdog.run([sys.executable, '-c', code], cwd=root,
stdout=root/'out', stderr=root/'err', execution=owner, hard_ceiling_s=.25, poll_s=.02)
identity = execution.process_identity(int((root/'child.pid').read_text()))
self.assertNotEqual(execution.process_status(identity), 'ALIVE')
self.assertEqual(result.cleanup_state, 'VERIFIED')
self.assertTrue(any(p['pid'] == identity['pid'] for p in owner.snapshot()['processes']))
finally:
if identity and identity['status'] == 'KNOWN':
execution.signal_process(identity, signal.SIGKILL)
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux identity verification')
def test_unobserved_group_after_leader_exit_is_quarantined(self):
from harness import execution
from scripts.test_execution import policy
with tempfile.TemporaryDirectory() as name:
root = Path(name)
owner = execution.Execution(root); owner.start(policy())
code = ("import subprocess,sys,time;from pathlib import Path;time.sleep(.15);"
"p=subprocess.Popen([sys.executable,'-c','import time;time.sleep(30)'],env={});"
"Path('child.pid').write_text(str(p.pid))")
identity = None
try:
result = watchdog.run([sys.executable, '-c', code], cwd=root,
stdout=root/'out', stderr=root/'err', execution=owner, poll_s=30)
identity = execution.process_identity(int((root/'child.pid').read_text()))
if execution.process_status(identity) == 'ALIVE':
self.assertEqual(result.cleanup_state, 'QUARANTINED')
self.assertTrue(any(scope['unknown'] for scope in owner.snapshot()['process_sets'].values()))
else:
self.assertEqual(result.cleanup_state, 'VERIFIED') # A valid earlier observation may catch it.
finally:
if identity and identity['status'] == 'KNOWN':
execution.signal_process(identity, signal.SIGKILL)
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux identity verification')
def test_restart_uses_persisted_unmarked_child_identity(self):
from harness import execution
from scripts.test_execution import policy
with tempfile.TemporaryDirectory() as name:
owner = execution.Execution(name); owner.start(policy())
child = subprocess.Popen([sys.executable, '-c', 'import time;time.sleep(30)'], env={}, start_new_session=True)
try:
identity = execution.process_identity(child.pid)
owner.record_process_set('fixture', identity, [identity], [])
original = execution.process_status
with patch.object(execution, 'marker_processes', return_value=[]), \
patch.object(execution, 'process_status', side_effect=lambda value:
'DEAD' if value['pid'] == os.getpid() else original(value)):
result = execution.Execution(name).reconcile()
self.assertEqual(result['cleanup_state'], 'QUARANTINED')
self.assertNotEqual(result['execution_outcome'], 'LOST')
finally:
child.kill(); child.wait(timeout=3)
@unittest.skipUnless(Path('/proc').is_dir(), 'Linux identity verification')
def test_restart_quarantines_unobserved_old_group_member(self):
from harness import execution
from scripts.test_execution import policy
with tempfile.TemporaryDirectory() as name:
root = Path(name)
owner = execution.Execution(root); owner.start(policy())
code = ("import subprocess,sys,time;from pathlib import Path;time.sleep(.1);"
"p=subprocess.Popen([sys.executable,'-c','import time;time.sleep(30)'],env={});"
f"Path({str(root/'child.pid')!r}).write_text(str(p.pid))")
leader = subprocess.Popen([sys.executable, '-c', code], start_new_session=True)
identity = execution.process_identity(leader.pid)
owner.record_process_set('fixture', identity, [identity], [])
child_identity = None
try:
leader.wait(timeout=3)
child_identity = execution.process_identity(int((root/'child.pid').read_text()))
original = execution.process_status
with patch.object(execution, 'marker_processes', return_value=[]), \
patch.object(execution, 'process_status', side_effect=lambda value:
'DEAD' if value['pid'] == os.getpid() else original(value)):
result = execution.Execution(root).reconcile()
self.assertEqual(result['cleanup_state'], 'QUARANTINED')
self.assertNotEqual(result['execution_outcome'], 'LOST')
self.assertEqual(execution.process_status(child_identity), 'ALIVE')
finally:
if leader.poll() is None:
leader.kill(); leader.wait(timeout=3)
if child_identity:
execution.signal_process(child_identity, signal.SIGKILL)
if __name__ == "__main__":
unittest.main()