233 lines
9.2 KiB
Python
233 lines
9.2 KiB
Python
"""One Linux subreaper per command; ECHILD, not a /proc sample, proves cleanup.
|
|
|
|
The command keeps host PIDs and its normal environment. Forked, reparented and
|
|
setsid children cannot escape this dedicated reaper by clearing environment.
|
|
Only the trusted supervisor writes the receipt. Its death without a matching
|
|
receipt is uncertainty, never an empty process set.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import ctypes
|
|
import json
|
|
import os
|
|
import select
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
if __package__ in (None, ''):
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from harness import execution
|
|
from harness.process_reaper import Reaper, children
|
|
|
|
|
|
def _send(channel, value):
|
|
channel.sendall(json.dumps(value).encode() + b'\n')
|
|
|
|
|
|
def _receive(channel):
|
|
data = bytearray()
|
|
while not data.endswith(b'\n'):
|
|
chunk = channel.recv(1)
|
|
if not chunk:
|
|
raise EOFError('scope controller disconnected')
|
|
data.extend(chunk)
|
|
return json.loads(data)
|
|
|
|
|
|
class Scope:
|
|
"""Popen-shaped command handle; the separate supervisor owns/reaps it."""
|
|
def __init__(self, cmd, *, cwd, stdout, stderr, env, shell, token, receipt,
|
|
binding, grace_s, cleanup_s, deadline):
|
|
self.path, self.token = Path(receipt), token
|
|
self.returncode = None
|
|
self.channel, peer = socket.socketpair()
|
|
self.channel.settimeout(5)
|
|
self.supervisor = subprocess.Popen(
|
|
[sys.executable, str(Path(__file__).resolve()), str(peer.fileno())],
|
|
pass_fds=(peer.fileno(),), stdout=stdout, stderr=stderr, start_new_session=True)
|
|
peer.close()
|
|
self.supervisor_identity = execution.process_identity(self.supervisor.pid)
|
|
self.binding = None
|
|
try:
|
|
_send(self.channel, dict(command=cmd, cwd=str(cwd), env=env, shell=shell,
|
|
token=token, receipt=str(receipt), binding=binding,
|
|
supervisor=self.supervisor_identity, controller=execution.process_identity(os.getpid()),
|
|
grace_s=grace_s, cleanup_s=cleanup_s, deadline=deadline))
|
|
ready = _receive(self.channel)
|
|
self.binding = ready['binding']
|
|
self.leader = self.binding['leader']
|
|
self.pid = self.leader['pid']
|
|
self.known = {execution.process_key(p): p for p in (self.leader, self.supervisor_identity)}
|
|
except BaseException:
|
|
self.channel.close()
|
|
execution.signal_process(self.supervisor_identity, signal.SIGTERM)
|
|
try:
|
|
self.supervisor.wait(timeout=cleanup_s + .5)
|
|
except subprocess.TimeoutExpired:
|
|
execution.signal_process(self.supervisor_identity, signal.SIGKILL)
|
|
self.supervisor.wait(timeout=1)
|
|
raise
|
|
|
|
def start(self):
|
|
_send(self.channel, {'start': True})
|
|
|
|
def receipt(self):
|
|
return execution.scope_receipt(dict(binding=self.binding, receipt_path=str(self.path)))
|
|
|
|
def poll(self):
|
|
if self.returncode is not None:
|
|
return self.returncode
|
|
if self.supervisor.poll() is None:
|
|
return None
|
|
receipt = self.receipt()
|
|
self.returncode = receipt.get('exit_code') if receipt else self.supervisor.returncode
|
|
return self.returncode
|
|
|
|
def wait(self, timeout=None):
|
|
self.supervisor.wait(timeout=timeout)
|
|
return self.poll()
|
|
|
|
def observe(self):
|
|
# Sampling provides monitor detail only. ECHILD receipt is the cleanup gate.
|
|
pending = [self.supervisor_identity]
|
|
seen = set()
|
|
while pending:
|
|
parent = pending.pop()
|
|
key = execution.process_key(parent)
|
|
if key in seen or execution.process_status(parent) != 'ALIVE':
|
|
continue
|
|
seen.add(key)
|
|
try:
|
|
pids = children(parent['pid'])
|
|
except OSError:
|
|
continue
|
|
for pid in pids:
|
|
child = execution.process_identity(pid)
|
|
if (child['status'] == 'KNOWN' and child['ppid'] == parent['pid'] and
|
|
execution.process_status(parent) == 'ALIVE'):
|
|
self.known[execution.process_key(child)] = child
|
|
pending.append(child)
|
|
receipt = self.receipt()
|
|
for child in receipt.get('members', []) if receipt else []:
|
|
self.known[execution.process_key(child)] = child
|
|
return [p for p in self.known.values() if execution.process_status(p) != 'DEAD']
|
|
|
|
def persist(self, owner):
|
|
if owner:
|
|
owner.record_process_set(self.token, self.leader, list(self.known.values()), [])
|
|
|
|
def empty(self):
|
|
receipt = self.receipt()
|
|
return bool(receipt and receipt['cleanup_state'] == 'VERIFIED' and self.supervisor.poll() is not None)
|
|
|
|
def cleanup(self, timeout):
|
|
if self.supervisor.poll() is None:
|
|
execution.signal_process(self.supervisor_identity, signal.SIGTERM)
|
|
try:
|
|
self.supervisor.wait(timeout=timeout + .5)
|
|
except subprocess.TimeoutExpired:
|
|
execution.signal_process(self.supervisor_identity, signal.SIGKILL)
|
|
try:
|
|
self.supervisor.wait(timeout=.5)
|
|
except subprocess.TimeoutExpired:
|
|
pass
|
|
self.channel.close()
|
|
self.observe()
|
|
self.poll()
|
|
return self.empty()
|
|
|
|
|
|
def supervise(fd):
|
|
channel = socket.socket(fileno=fd)
|
|
cfg = _receive(channel)
|
|
stopping = False
|
|
|
|
def stop(_sig, _frame):
|
|
nonlocal stopping
|
|
stopping = True
|
|
|
|
signal.signal(signal.SIGTERM, stop)
|
|
signal.signal(signal.SIGINT, stop)
|
|
reaper = Reaper()
|
|
libc = ctypes.CDLL(None, use_errno=True)
|
|
if libc.prctl(1, signal.SIGTERM, 0, 0, 0) != 0: # PR_SET_PDEATHSIG
|
|
raise OSError(ctypes.get_errno(), 'cannot establish parent death signal')
|
|
if execution.process_key(execution.process_identity(os.getpid())) != execution.process_key(cfg['supervisor']):
|
|
raise RuntimeError('supervisor birth identity changed')
|
|
gate_read, gate_write = os.pipe()
|
|
pid = os.fork()
|
|
if pid == 0:
|
|
try:
|
|
channel.close(); os.close(gate_write)
|
|
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
|
signal.signal(signal.SIGINT, signal.SIG_DFL)
|
|
os.setsid()
|
|
if os.read(gate_read, 1) != b'1':
|
|
os._exit(125)
|
|
os.close(gate_read)
|
|
os.chdir(cfg['cwd'])
|
|
command = cfg['command']
|
|
command = [command] if isinstance(command, str) else command
|
|
if cfg['shell']:
|
|
command = ['/bin/sh', '-c', *command]
|
|
os.execvpe(command[0], command, cfg['env'])
|
|
except BaseException as exc:
|
|
print(f'command startup failed: {exc}', file=sys.stderr, flush=True)
|
|
os._exit(127)
|
|
os.close(gate_read)
|
|
leader = execution.process_identity(pid)
|
|
binding = dict(cfg['binding'], token=cfg['token'], supervisor=cfg['supervisor'], leader=leader,
|
|
proof='waitpid:ECHILD', version=1)
|
|
started, cleanup_at, exit_code, empty, owner_lost = False, None, None, False, False
|
|
failure = None
|
|
try:
|
|
_send(channel, {'binding': binding})
|
|
while True:
|
|
# waitpid is authoritative even when all processes changed env/session
|
|
# and disappeared from every observation between two polls.
|
|
empty = reaper.poll()
|
|
exit_code = reaper.exits.get(pid)
|
|
now = time.monotonic()
|
|
owner_lost |= (os.getppid() != cfg['controller']['pid'] or
|
|
execution.process_status(cfg['controller']) != 'ALIVE')
|
|
if empty:
|
|
break
|
|
if owner_lost or stopping or exit_code is not None or now >= cfg['deadline']:
|
|
cleanup_at = cleanup_at or now
|
|
if cleanup_at is None and not started and select.select([channel], [], [], 0)[0]:
|
|
try:
|
|
started = _receive(channel).get('start') is True
|
|
except (OSError, EOFError, ValueError):
|
|
stopping = True
|
|
if started:
|
|
os.write(gate_write, b'1')
|
|
os.close(gate_write); gate_write = -1
|
|
else:
|
|
cleanup_at = now
|
|
reaper.signal(None if cleanup_at is None else (
|
|
signal.SIGTERM if now-cleanup_at < cfg['grace_s'] else signal.SIGKILL))
|
|
if cleanup_at is not None and now-cleanup_at >= cfg['cleanup_s']:
|
|
break
|
|
time.sleep(.005)
|
|
except BaseException as exc:
|
|
failure = type(exc).__name__
|
|
result = reaper.cleanup(None, cfg['grace_s'], cfg['cleanup_s'])
|
|
empty = result['all_children_reaped']
|
|
exit_code = reaper.exits.get(pid)
|
|
finally:
|
|
if gate_write >= 0:
|
|
os.close(gate_write)
|
|
channel.close()
|
|
execution.atomic_json(Path(cfg['receipt']), dict(binding=binding, members=list(reaper.members.values()),
|
|
cleanup_state='VERIFIED' if empty and not owner_lost else 'QUARANTINED',
|
|
all_children_reaped=empty, owner_lost=owner_lost, exit_code=exit_code,
|
|
supervisor_error=failure, finished_monotonic=time.monotonic()))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
supervise(int(sys.argv[1]))
|