data-gen-bpu/harness/process_scope.py

233 lines
9.2 KiB
Python

"""One Linux subreaper per command; ECHILD, not a /proc sample, proves cleanup.
The command keeps host PIDs and its normal environment. Forked, reparented and
setsid children cannot escape this dedicated reaper by clearing environment.
Only the trusted supervisor writes the receipt. Its death without a matching
receipt is uncertainty, never an empty process set.
"""
from __future__ import annotations
import ctypes
import json
import os
import select
import signal
import socket
import subprocess
import sys
import time
from pathlib import Path
if __package__ in (None, ''):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from harness import execution
from harness.process_reaper import Reaper, children
def _send(channel, value):
channel.sendall(json.dumps(value).encode() + b'\n')
def _receive(channel):
data = bytearray()
while not data.endswith(b'\n'):
chunk = channel.recv(1)
if not chunk:
raise EOFError('scope controller disconnected')
data.extend(chunk)
return json.loads(data)
class Scope:
"""Popen-shaped command handle; the separate supervisor owns/reaps it."""
def __init__(self, cmd, *, cwd, stdout, stderr, env, shell, token, receipt,
binding, grace_s, cleanup_s, deadline):
self.path, self.token = Path(receipt), token
self.returncode = None
self.channel, peer = socket.socketpair()
self.channel.settimeout(5)
self.supervisor = subprocess.Popen(
[sys.executable, str(Path(__file__).resolve()), str(peer.fileno())],
pass_fds=(peer.fileno(),), stdout=stdout, stderr=stderr, start_new_session=True)
peer.close()
self.supervisor_identity = execution.process_identity(self.supervisor.pid)
self.binding = None
try:
_send(self.channel, dict(command=cmd, cwd=str(cwd), env=env, shell=shell,
token=token, receipt=str(receipt), binding=binding,
supervisor=self.supervisor_identity, controller=execution.process_identity(os.getpid()),
grace_s=grace_s, cleanup_s=cleanup_s, deadline=deadline))
ready = _receive(self.channel)
self.binding = ready['binding']
self.leader = self.binding['leader']
self.pid = self.leader['pid']
self.known = {execution.process_key(p): p for p in (self.leader, self.supervisor_identity)}
except BaseException:
self.channel.close()
execution.signal_process(self.supervisor_identity, signal.SIGTERM)
try:
self.supervisor.wait(timeout=cleanup_s + .5)
except subprocess.TimeoutExpired:
execution.signal_process(self.supervisor_identity, signal.SIGKILL)
self.supervisor.wait(timeout=1)
raise
def start(self):
_send(self.channel, {'start': True})
def receipt(self):
return execution.scope_receipt(dict(binding=self.binding, receipt_path=str(self.path)))
def poll(self):
if self.returncode is not None:
return self.returncode
if self.supervisor.poll() is None:
return None
receipt = self.receipt()
self.returncode = receipt.get('exit_code') if receipt else self.supervisor.returncode
return self.returncode
def wait(self, timeout=None):
self.supervisor.wait(timeout=timeout)
return self.poll()
def observe(self):
# Sampling provides monitor detail only. ECHILD receipt is the cleanup gate.
pending = [self.supervisor_identity]
seen = set()
while pending:
parent = pending.pop()
key = execution.process_key(parent)
if key in seen or execution.process_status(parent) != 'ALIVE':
continue
seen.add(key)
try:
pids = children(parent['pid'])
except OSError:
continue
for pid in pids:
child = execution.process_identity(pid)
if (child['status'] == 'KNOWN' and child['ppid'] == parent['pid'] and
execution.process_status(parent) == 'ALIVE'):
self.known[execution.process_key(child)] = child
pending.append(child)
receipt = self.receipt()
for child in receipt.get('members', []) if receipt else []:
self.known[execution.process_key(child)] = child
return [p for p in self.known.values() if execution.process_status(p) != 'DEAD']
def persist(self, owner):
if owner:
owner.record_process_set(self.token, self.leader, list(self.known.values()), [])
def empty(self):
receipt = self.receipt()
return bool(receipt and receipt['cleanup_state'] == 'VERIFIED' and self.supervisor.poll() is not None)
def cleanup(self, timeout):
if self.supervisor.poll() is None:
execution.signal_process(self.supervisor_identity, signal.SIGTERM)
try:
self.supervisor.wait(timeout=timeout + .5)
except subprocess.TimeoutExpired:
execution.signal_process(self.supervisor_identity, signal.SIGKILL)
try:
self.supervisor.wait(timeout=.5)
except subprocess.TimeoutExpired:
pass
self.channel.close()
self.observe()
self.poll()
return self.empty()
def supervise(fd):
channel = socket.socket(fileno=fd)
cfg = _receive(channel)
stopping = False
def stop(_sig, _frame):
nonlocal stopping
stopping = True
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
reaper = Reaper()
libc = ctypes.CDLL(None, use_errno=True)
if libc.prctl(1, signal.SIGTERM, 0, 0, 0) != 0: # PR_SET_PDEATHSIG
raise OSError(ctypes.get_errno(), 'cannot establish parent death signal')
if execution.process_key(execution.process_identity(os.getpid())) != execution.process_key(cfg['supervisor']):
raise RuntimeError('supervisor birth identity changed')
gate_read, gate_write = os.pipe()
pid = os.fork()
if pid == 0:
try:
channel.close(); os.close(gate_write)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
signal.signal(signal.SIGINT, signal.SIG_DFL)
os.setsid()
if os.read(gate_read, 1) != b'1':
os._exit(125)
os.close(gate_read)
os.chdir(cfg['cwd'])
command = cfg['command']
command = [command] if isinstance(command, str) else command
if cfg['shell']:
command = ['/bin/sh', '-c', *command]
os.execvpe(command[0], command, cfg['env'])
except BaseException as exc:
print(f'command startup failed: {exc}', file=sys.stderr, flush=True)
os._exit(127)
os.close(gate_read)
leader = execution.process_identity(pid)
binding = dict(cfg['binding'], token=cfg['token'], supervisor=cfg['supervisor'], leader=leader,
proof='waitpid:ECHILD', version=1)
started, cleanup_at, exit_code, empty, owner_lost = False, None, None, False, False
failure = None
try:
_send(channel, {'binding': binding})
while True:
# waitpid is authoritative even when all processes changed env/session
# and disappeared from every observation between two polls.
empty = reaper.poll()
exit_code = reaper.exits.get(pid)
now = time.monotonic()
owner_lost |= (os.getppid() != cfg['controller']['pid'] or
execution.process_status(cfg['controller']) != 'ALIVE')
if empty:
break
if owner_lost or stopping or exit_code is not None or now >= cfg['deadline']:
cleanup_at = cleanup_at or now
if cleanup_at is None and not started and select.select([channel], [], [], 0)[0]:
try:
started = _receive(channel).get('start') is True
except (OSError, EOFError, ValueError):
stopping = True
if started:
os.write(gate_write, b'1')
os.close(gate_write); gate_write = -1
else:
cleanup_at = now
reaper.signal(None if cleanup_at is None else (
signal.SIGTERM if now-cleanup_at < cfg['grace_s'] else signal.SIGKILL))
if cleanup_at is not None and now-cleanup_at >= cfg['cleanup_s']:
break
time.sleep(.005)
except BaseException as exc:
failure = type(exc).__name__
result = reaper.cleanup(None, cfg['grace_s'], cfg['cleanup_s'])
empty = result['all_children_reaped']
exit_code = reaper.exits.get(pid)
finally:
if gate_write >= 0:
os.close(gate_write)
channel.close()
execution.atomic_json(Path(cfg['receipt']), dict(binding=binding, members=list(reaper.members.values()),
cleanup_state='VERIFIED' if empty and not owner_lost else 'QUARANTINED',
all_children_reaped=empty, owner_lost=owner_lost, exit_code=exit_code,
supervisor_error=failure, finished_monotonic=time.monotonic()))
if __name__ == '__main__':
supervise(int(sys.argv[1]))