One user-space repair task per eligible HWE debug-diag / compile-deploy base ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content filter. Each ships a public spec with a check-to-basis table, at least two defects with single-fix fixtures, a symptom-masking fixture and an alternative correct implementation. Claude re-checked all 320: structure, the HWE quality items and trace6 Docker calibration receipts against current digests (320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120 ad-hoc family names; the next batch maps them to the canonical list. The batch's 1,256-line progress log stays in its factory report and the 36-local branch batch15-hwe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
905 lines
35 KiB
Python
905 lines
35 KiB
Python
"""Private independent plan oracle and same-identity execution supervisor."""
|
|
import base64
|
|
from collections import Counter
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import zlib
|
|
|
|
CANDIDATES = ("Makefile", "src/main.cpp", "src/planner.cpp", "src/planner.hpp")
|
|
COMPILER_DEADLINE = 20.0
|
|
CANDIDATE_DEADLINE = 14.0
|
|
CLEANUP_DEADLINE = 6.0
|
|
ABSENCE_DEADLINE = 6.0
|
|
RECOVERY_ALLOWANCE = 3.0
|
|
|
|
SUPERVISOR = r'''
|
|
import base64, ctypes, hashlib, json, os, selectors, signal, subprocess, sys, time
|
|
|
|
BRIDGE = r"""
|
|
import base64, json, os, sys
|
|
argv = json.loads(base64.b64decode(sys.argv[1], validate=True))
|
|
fd = int(sys.argv[2])
|
|
raw = open('/proc/self/stat', 'rb').read()
|
|
fields = raw[raw.rfind(b')')+2:].split()
|
|
report = {'pid':os.getpid(),'start':int(fields[19]),'argv':argv,
|
|
'cwd':os.getcwd(),'uid':os.getuid(),'gid':os.getgid(),
|
|
'groups':os.getgroups()}
|
|
os.write(fd, json.dumps(report, separators=(',', ':')).encode('ascii'))
|
|
os.close(fd)
|
|
os.execvp(argv[0], argv)
|
|
"""
|
|
|
|
def sha(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
def proc_one(pid):
|
|
try:
|
|
data = open('/proc/%d/stat' % pid, 'rb').read()
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
return None
|
|
close = data.rfind(b')')
|
|
if close < 0:
|
|
raise RuntimeError('proc stat missing comm end')
|
|
tail = data[close + 2:].split()
|
|
if len(tail) < 20:
|
|
raise RuntimeError('proc stat too short')
|
|
return {'pid':pid, 'state':tail[0].decode('ascii'), 'ppid':int(tail[1]),
|
|
'pgid':int(tail[2]), 'start':int(tail[19])}
|
|
|
|
def proc_table():
|
|
table = {}
|
|
for item in os.listdir('/proc'):
|
|
if item.isdecimal():
|
|
pid = int(item)
|
|
entry = proc_one(pid)
|
|
if entry is not None:
|
|
table[pid] = entry
|
|
return table
|
|
|
|
def descendants(table, supervisor):
|
|
owned = set()
|
|
active = {supervisor}
|
|
while active:
|
|
next_active = set()
|
|
for pid, row in table.items():
|
|
if pid not in owned and row['ppid'] in active and row['state'] not in ('Z', 'X'):
|
|
owned.add(pid)
|
|
next_active.add(pid)
|
|
active = next_active
|
|
return {pid:table[pid] for pid in owned}
|
|
|
|
def fresh_signal(row, sig):
|
|
fresh = proc_one(row['pid'])
|
|
if fresh is None or fresh['start'] != row['start'] or fresh['state'] in ('Z', 'X'):
|
|
return False
|
|
try:
|
|
os.kill(row['pid'], sig)
|
|
except ProcessLookupError:
|
|
return False
|
|
return True
|
|
|
|
def reap_adopted(table, leader_pid, errors):
|
|
for pid, row in table.items():
|
|
if row['ppid'] != os.getpid() or pid == leader_pid:
|
|
continue
|
|
try:
|
|
os.waitpid(pid, os.WNOHANG)
|
|
except ChildProcessError:
|
|
pass
|
|
except ProcessLookupError:
|
|
pass
|
|
except Exception as exc:
|
|
errors.append('adopted_wait:' + repr(exc))
|
|
|
|
def cleanup_children(proc, leader_pid, errors):
|
|
stopped = False
|
|
end = time.monotonic() + 2.0
|
|
last = {}
|
|
while time.monotonic() < end:
|
|
if proc is not None:
|
|
proc.poll()
|
|
table = proc_table()
|
|
last = descendants(table, os.getpid())
|
|
for row in last.values():
|
|
fresh_signal(row, signal.SIGKILL)
|
|
if proc is not None and proc.poll() is None:
|
|
own = proc_one(leader_pid)
|
|
if own is not None:
|
|
fresh_signal(own, signal.SIGKILL)
|
|
if proc is not None:
|
|
proc.poll()
|
|
reap_adopted(proc_table(), leader_pid, errors)
|
|
again = descendants(proc_table(), os.getpid())
|
|
if not again and (proc is None or proc.poll() is not None):
|
|
stopped = True
|
|
break
|
|
time.sleep(0.015)
|
|
if proc is not None:
|
|
try:
|
|
proc.wait(timeout=0.2)
|
|
except Exception as exc:
|
|
errors.append('leader_wait:' + repr(exc))
|
|
if not stopped:
|
|
errors.append('descendants_remaining:' + str(sorted(last)))
|
|
return stopped
|
|
|
|
def transfer(proc, writer, payload, deadline):
|
|
selector = selectors.DefaultSelector()
|
|
output = bytearray()
|
|
diagnostics = bytearray()
|
|
sent = bytearray()
|
|
closed = False
|
|
broken = False
|
|
os.set_blocking(writer, False)
|
|
for stream in (proc.stdout, proc.stderr):
|
|
os.set_blocking(stream.fileno(), False)
|
|
selector.register(proc.stdout, selectors.EVENT_READ, output)
|
|
selector.register(proc.stderr, selectors.EVENT_READ, diagnostics)
|
|
if payload:
|
|
selector.register(writer, selectors.EVENT_WRITE, None)
|
|
else:
|
|
os.close(writer)
|
|
closed = True
|
|
end = time.monotonic() + deadline
|
|
timed_out = False
|
|
while selector.get_map():
|
|
remaining = end - time.monotonic()
|
|
if remaining <= 0:
|
|
timed_out = True
|
|
break
|
|
for key, events in selector.select(min(remaining, 0.05)):
|
|
stream = key.fileobj
|
|
if stream == writer:
|
|
try:
|
|
n = os.write(writer, payload[len(sent):len(sent)+65536])
|
|
sent.extend(payload[len(sent):len(sent)+n])
|
|
except BrokenPipeError:
|
|
broken = True
|
|
n = 0
|
|
if broken or len(sent) == len(payload):
|
|
selector.unregister(stream)
|
|
os.close(writer)
|
|
closed = True
|
|
else:
|
|
chunk = os.read(stream.fileno(), 65536)
|
|
if chunk:
|
|
key.data.extend(chunk)
|
|
else:
|
|
selector.unregister(stream)
|
|
stream.close()
|
|
selector.close()
|
|
if proc.poll() is None:
|
|
remaining = end - time.monotonic()
|
|
if remaining <= 0:
|
|
timed_out = True
|
|
else:
|
|
try:
|
|
proc.wait(timeout=remaining)
|
|
except subprocess.TimeoutExpired:
|
|
timed_out = True
|
|
return bytes(output), bytes(diagnostics), bytes(sent), closed, broken, timed_out
|
|
|
|
def drain_unread(fd, errors):
|
|
unread = bytearray()
|
|
try:
|
|
os.set_blocking(fd, False)
|
|
while True:
|
|
try:
|
|
chunk = os.read(fd, 65536)
|
|
except BlockingIOError:
|
|
errors.append('stdin_pipe_writer_alive')
|
|
break
|
|
if not chunk:
|
|
break
|
|
unread.extend(chunk)
|
|
finally:
|
|
os.close(fd)
|
|
return bytes(unread)
|
|
|
|
def read_preexec(fd, deadline):
|
|
os.set_blocking(fd, False)
|
|
data = bytearray()
|
|
end = time.monotonic() + min(deadline, 2.0)
|
|
try:
|
|
while time.monotonic() < end:
|
|
try:
|
|
part = os.read(fd, 8192)
|
|
except BlockingIOError:
|
|
time.sleep(0.002)
|
|
continue
|
|
if not part:
|
|
break
|
|
data.extend(part)
|
|
if len(data) > 8192:
|
|
raise RuntimeError('preexec report too large')
|
|
if not data:
|
|
raise RuntimeError('missing preexec report')
|
|
return json.loads(data)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
def main():
|
|
raw_request = sys.stdin.buffer.read()
|
|
request = json.loads(raw_request)
|
|
stage = request['stage']
|
|
argv = request['argv']
|
|
cwd = request['cwd']
|
|
deadline = request['deadline']
|
|
payload = base64.b64decode(request['input_b64'], validate=True)
|
|
errors = []
|
|
supervisor = os.getpid()
|
|
own = proc_one(supervisor)
|
|
meta = {'stage':stage,'supervisor_pid':supervisor,'supervisor_start':own['start'],
|
|
'uid':os.getuid(),'gid':os.getgid(),'groups':os.getgroups(),
|
|
'argv':argv,'cwd':cwd,'deadline_seconds':deadline,
|
|
'request_wire_len':len(raw_request),'request_wire_sha256':sha(raw_request),
|
|
'stdin_len':len(payload),'stdin_sha256':sha(payload),
|
|
'stdin_sent_len':0,'stdin_sent_sha256':sha(b''),
|
|
'stdin_read_len':0,'stdin_read_sha256':sha(b''),
|
|
'stdin_unread_len':0,'stdin_unread_sha256':sha(b''),
|
|
'stdin_closed':False,'stdin_broken_pipe':False,
|
|
'preexec':None,
|
|
'leader_pid':None,'leader_start':None,'leader_rc':None,
|
|
'controlled_rc':None,'leader_reaped':False,'timeout':False,
|
|
'pg_absent':False,'descendants_absent':False,'elapsed_seconds':0.0,
|
|
'stdout_len':0,'stdout_sha256':sha(b''),
|
|
'stderr_len':0,'stderr_sha256':sha(b''),'exceptions':errors}
|
|
stdout = b''
|
|
stderr = b''
|
|
sent = b''
|
|
unread = b''
|
|
writer = None
|
|
drain_fd = None
|
|
reader = None
|
|
report_read = None
|
|
report_write = None
|
|
proc = None
|
|
start = time.monotonic()
|
|
try:
|
|
libc = ctypes.CDLL(None, use_errno=True)
|
|
if libc.prctl(36, 1, 0, 0, 0) != 0:
|
|
raise OSError(ctypes.get_errno(), 'PR_SET_CHILD_SUBREAPER')
|
|
reader, writer = os.pipe()
|
|
drain_fd = os.dup(reader)
|
|
report_read, report_write = os.pipe()
|
|
bridge_argv = [sys.executable, '-I', '-c', BRIDGE,
|
|
base64.b64encode(json.dumps(argv).encode('utf-8')).decode('ascii'),
|
|
str(report_write)]
|
|
proc = subprocess.Popen(bridge_argv, cwd=cwd, stdin=reader,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
start_new_session=True, pass_fds=(report_write,))
|
|
os.close(report_write)
|
|
report_write = None
|
|
os.close(reader)
|
|
reader = None
|
|
meta['leader_pid'] = proc.pid
|
|
child = proc_one(proc.pid)
|
|
if child is None:
|
|
raise RuntimeError('leader disappeared before start binding')
|
|
meta['leader_start'] = child['start']
|
|
meta['preexec'] = read_preexec(report_read, deadline)
|
|
report_read = None
|
|
stdout, stderr, sent, closed, broken, timed_out = transfer(
|
|
proc, writer, payload, deadline)
|
|
writer = None
|
|
meta['stdin_closed'] = closed
|
|
meta['stdin_broken_pipe'] = broken
|
|
meta['timeout'] = timed_out
|
|
meta['controlled_rc'] = 124 if timed_out else proc.returncode
|
|
meta['leader_rc'] = proc.returncode
|
|
except BaseException as exc:
|
|
errors.append('execute:' + repr(exc))
|
|
finally:
|
|
try:
|
|
absent = cleanup_children(proc, meta['leader_pid'] or -1, errors)
|
|
meta['descendants_absent'] = bool(absent)
|
|
except BaseException as exc:
|
|
errors.append('cleanup_children:' + repr(exc))
|
|
for label, fd in (('writer', writer), ('reader', reader),
|
|
('report_read', report_read), ('report_write', report_write)):
|
|
if fd is not None:
|
|
try:
|
|
os.close(fd)
|
|
if label == 'writer':
|
|
meta['stdin_closed'] = True
|
|
except OSError as exc:
|
|
errors.append('close_' + label + ':' + repr(exc))
|
|
if drain_fd is not None:
|
|
try:
|
|
unread = drain_unread(drain_fd, errors)
|
|
except BaseException as exc:
|
|
errors.append('drain_unread:' + repr(exc))
|
|
try:
|
|
if proc is not None:
|
|
meta['leader_reaped'] = proc.returncode is not None
|
|
if meta['leader_rc'] is None and proc.returncode is not None:
|
|
meta['leader_rc'] = proc.returncode
|
|
table = proc_table()
|
|
meta['pg_absent'] = not any(
|
|
row['pgid'] == proc.pid and row['state'] not in ('Z', 'X')
|
|
for row in table.values())
|
|
else:
|
|
meta['pg_absent'] = bool(meta['descendants_absent'])
|
|
except BaseException as exc:
|
|
errors.append('pg_absence:' + repr(exc))
|
|
meta['elapsed_seconds'] = time.monotonic() - start
|
|
meta['stdin_sent_len'], meta['stdin_sent_sha256'] = len(sent), sha(sent)
|
|
meta['stdin_unread_len'], meta['stdin_unread_sha256'] = len(unread), sha(unread)
|
|
if not sent.endswith(unread):
|
|
errors.append('stdin_unread_not_suffix')
|
|
consumed = sent[:len(sent)-len(unread)]
|
|
meta['stdin_read_len'], meta['stdin_read_sha256'] = len(consumed), sha(consumed)
|
|
meta['stdout_len'], meta['stdout_sha256'] = len(stdout), sha(stdout)
|
|
meta['stderr_len'], meta['stderr_sha256'] = len(stderr), sha(stderr)
|
|
wire = {'meta':meta,'sent_b64':base64.b64encode(sent).decode('ascii'),
|
|
'unread_b64':base64.b64encode(unread).decode('ascii'),
|
|
'stdout_b64':base64.b64encode(stdout).decode('ascii'),
|
|
'stderr_b64':base64.b64encode(stderr).decode('ascii')}
|
|
sys.stdout.write(json.dumps(wire, separators=(',', ':')) + '\n')
|
|
|
|
if __name__ == '__main__':
|
|
main()
|
|
'''
|
|
|
|
CHILD_ACTION = r'''
|
|
import os, shutil, sys
|
|
kind, path = sys.argv[1:3]
|
|
if kind == 'cleanup':
|
|
shutil.rmtree(path)
|
|
elif kind == 'absence':
|
|
try:
|
|
os.lstat(path)
|
|
except FileNotFoundError:
|
|
sys.stdout.buffer.write(b'ABSENT\n')
|
|
else:
|
|
sys.stdout.buffer.write(b'PRESENT\n')
|
|
else:
|
|
raise SystemExit(4)
|
|
'''
|
|
|
|
WATCHDOG = r'''
|
|
import json, os, select, signal, sys, time
|
|
|
|
def one(pid):
|
|
try:
|
|
data = open('/proc/%d/stat' % pid, 'rb').read()
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
return None
|
|
fields = data[data.rfind(b')')+2:].split()
|
|
if len(fields) < 20:
|
|
raise RuntimeError('short proc stat')
|
|
return {'pid':pid,'state':fields[0].decode('ascii'),'ppid':int(fields[1]),
|
|
'start':int(fields[19])}
|
|
|
|
def table():
|
|
result = {}
|
|
for name in os.listdir('/proc'):
|
|
if name.isdecimal():
|
|
row = one(int(name))
|
|
if row is not None:
|
|
result[row['pid']] = row
|
|
return result
|
|
|
|
def closure(rows, root):
|
|
found = {}
|
|
wave = {root}
|
|
while wave:
|
|
next_wave = set()
|
|
for pid, row in rows.items():
|
|
if pid not in found and row['ppid'] in wave and row['state'] not in ('Z','X'):
|
|
found[pid] = row
|
|
next_wave.add(pid)
|
|
wave = next_wave
|
|
return found
|
|
|
|
def fresh_signal(row, sig):
|
|
current = one(row['pid'])
|
|
if current is None or current['start'] != row['start'] or current['state'] in ('Z','X'):
|
|
return False
|
|
try:
|
|
os.kill(row['pid'], sig)
|
|
except ProcessLookupError:
|
|
return False
|
|
return True
|
|
|
|
def alive_token(row):
|
|
current = one(row['pid'])
|
|
return (current is not None and current['start'] == row['start']
|
|
and current['state'] not in ('Z','X'))
|
|
|
|
def main():
|
|
pid = int(sys.argv[1]); start = int(sys.argv[2]); deadline = float(sys.argv[3])
|
|
root = {'pid':pid,'start':start}
|
|
until = time.monotonic() + deadline
|
|
while True:
|
|
remaining = until - time.monotonic()
|
|
ready, _, _ = select.select([sys.stdin], [], [], max(0, min(remaining, 0.1)))
|
|
if ready:
|
|
command = sys.stdin.buffer.readline()
|
|
if command == b'DONE\n':
|
|
print(json.dumps({'status':'finished','target':pid}))
|
|
return 0
|
|
if command == b'KILL\n':
|
|
break
|
|
if not command:
|
|
break
|
|
if remaining <= 0:
|
|
break
|
|
current = one(pid)
|
|
if current is None or current['start'] != start or current['state'] in ('Z','X'):
|
|
print(json.dumps({'status':'already_exited','target':pid}))
|
|
return 0
|
|
fresh_signal(root, signal.SIGSTOP)
|
|
tracked = closure(table(), pid)
|
|
end = time.monotonic() + 2.0
|
|
while time.monotonic() < end:
|
|
for row in closure(table(), pid).values():
|
|
tracked[row['pid']] = row
|
|
for row in tracked.values():
|
|
fresh_signal(row, signal.SIGKILL)
|
|
if not any(alive_token(row) for row in tracked.values()):
|
|
break
|
|
time.sleep(0.015)
|
|
fresh_signal(root, signal.SIGKILL)
|
|
alive = [pid for pid,row in tracked.items() if alive_token(row)]
|
|
print(json.dumps({'status':'forced','target':pid,'tracked':len(tracked),'alive':alive}))
|
|
return 1 if alive else 0
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|
|
'''
|
|
|
|
|
|
def sha(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def process_start(pid):
|
|
try:
|
|
raw = Path('/proc/%d/stat' % pid).read_bytes()
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
return None
|
|
fields = raw[raw.rfind(b')') + 2:].split()
|
|
if len(fields) < 20:
|
|
raise RuntimeError('short supervisor proc stat')
|
|
return int(fields[19])
|
|
|
|
|
|
def exact_int(value, minimum=None):
|
|
return type(value) is int and (minimum is None or value >= minimum)
|
|
|
|
|
|
def valid_meta(meta, stage, argv, cwd, deadline, supervisor_pid, supervisor_start, wire_input,
|
|
payload, sent, unread, stdout, stderr):
|
|
required = ('stage', 'supervisor_pid', 'supervisor_start', 'uid', 'gid',
|
|
'groups', 'argv', 'cwd', 'deadline_seconds', 'stdin_len',
|
|
'stdin_sha256', 'request_wire_len', 'request_wire_sha256',
|
|
'stdin_sent_len', 'stdin_sent_sha256', 'stdin_closed',
|
|
'stdin_read_len', 'stdin_read_sha256', 'stdin_unread_len',
|
|
'stdin_unread_sha256', 'preexec',
|
|
'stdin_broken_pipe', 'leader_pid', 'leader_start', 'leader_rc',
|
|
'controlled_rc', 'leader_reaped', 'timeout', 'pg_absent',
|
|
'descendants_absent', 'elapsed_seconds', 'stdout_len',
|
|
'stdout_sha256', 'stderr_len', 'stderr_sha256', 'exceptions')
|
|
if not isinstance(meta, dict) or any(key not in meta for key in required):
|
|
return False
|
|
if meta['stage'] != stage or meta['argv'] != argv or meta['cwd'] != cwd:
|
|
return False
|
|
if not isinstance(meta['argv'], list) or any(type(x) is not str for x in meta['argv']):
|
|
return False
|
|
if type(meta['deadline_seconds']) not in (int, float) or meta['deadline_seconds'] != deadline:
|
|
return False
|
|
if not exact_int(meta['supervisor_pid'], 1) or not exact_int(meta['leader_pid'], 1):
|
|
return False
|
|
if meta['supervisor_pid'] != supervisor_pid:
|
|
return False
|
|
if meta['supervisor_pid'] == meta['leader_pid']:
|
|
return False
|
|
if not exact_int(meta['supervisor_start'], 1) or not exact_int(meta['leader_start'], 1):
|
|
return False
|
|
if meta['supervisor_start'] != supervisor_start:
|
|
return False
|
|
isolated = os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534'
|
|
expected_uid = 65534 if isolated else os.getuid()
|
|
expected_gid = 65534 if isolated else os.getgid()
|
|
if not exact_int(meta['uid']) or meta['uid'] != expected_uid:
|
|
return False
|
|
if not exact_int(meta['gid']) or meta['gid'] != expected_gid:
|
|
return False
|
|
if type(meta['groups']) is not list or any(not exact_int(x) for x in meta['groups']):
|
|
return False
|
|
if isolated and meta['groups'] != []:
|
|
return False
|
|
preexec = meta['preexec']
|
|
if type(preexec) is not dict:
|
|
return False
|
|
if set(preexec) != {'pid', 'start', 'argv', 'cwd', 'uid', 'gid', 'groups'}:
|
|
return False
|
|
if not exact_int(preexec['pid'], 1) or preexec['pid'] != meta['leader_pid']:
|
|
return False
|
|
if not exact_int(preexec['start'], 1) or preexec['start'] != meta['leader_start']:
|
|
return False
|
|
if preexec['argv'] != argv or preexec['cwd'] != cwd:
|
|
return False
|
|
if (not exact_int(preexec['uid']) or preexec['uid'] != expected_uid or
|
|
not exact_int(preexec['gid']) or preexec['gid'] != expected_gid):
|
|
return False
|
|
if type(preexec['groups']) is not list or any(not exact_int(x) for x in preexec['groups']):
|
|
return False
|
|
if isolated and preexec['groups'] != []:
|
|
return False
|
|
if type(meta['timeout']) is not bool or type(meta['leader_reaped']) is not bool:
|
|
return False
|
|
if type(meta['pg_absent']) is not bool or type(meta['descendants_absent']) is not bool:
|
|
return False
|
|
if not meta['leader_reaped'] or not meta['pg_absent'] or not meta['descendants_absent']:
|
|
return False
|
|
if not exact_int(meta['leader_rc']) or not exact_int(meta['controlled_rc']):
|
|
return False
|
|
if meta['timeout']:
|
|
if meta['controlled_rc'] != 124:
|
|
return False
|
|
elif meta['controlled_rc'] != meta['leader_rc']:
|
|
return False
|
|
elapsed = meta['elapsed_seconds']
|
|
if type(elapsed) not in (int, float) or not math.isfinite(elapsed):
|
|
return False
|
|
if not 0 <= elapsed <= deadline + RECOVERY_ALLOWANCE:
|
|
return False
|
|
if type(meta['exceptions']) is not list or meta['exceptions']:
|
|
return False
|
|
if not exact_int(meta['stdin_len'], 0) or meta['stdin_len'] != len(payload):
|
|
return False
|
|
if meta['stdin_sha256'] != sha(payload):
|
|
return False
|
|
if not exact_int(meta['request_wire_len'], 0) or meta['request_wire_len'] != len(wire_input):
|
|
return False
|
|
if meta['request_wire_sha256'] != sha(wire_input):
|
|
return False
|
|
if type(meta['stdin_closed']) is not bool or not meta['stdin_closed']:
|
|
return False
|
|
if type(meta['stdin_broken_pipe']) is not bool:
|
|
return False
|
|
if not exact_int(meta['stdin_sent_len'], 0) or meta['stdin_sent_len'] != len(sent):
|
|
return False
|
|
if meta['stdin_sent_sha256'] != sha(sent) or sent != payload[:len(sent)]:
|
|
return False
|
|
if not sent.endswith(unread):
|
|
return False
|
|
consumed = sent[:len(sent)-len(unread)]
|
|
for label, raw in (('stdin_read', consumed), ('stdin_unread', unread)):
|
|
if not exact_int(meta[label + '_len'], 0) or meta[label + '_len'] != len(raw):
|
|
return False
|
|
if meta[label + '_sha256'] != sha(raw):
|
|
return False
|
|
for prefix, raw in (('stdout', stdout), ('stderr', stderr)):
|
|
if not exact_int(meta[prefix + '_len'], 0) or meta[prefix + '_len'] != len(raw):
|
|
return False
|
|
if meta[prefix + '_sha256'] != sha(raw):
|
|
return False
|
|
return True
|
|
|
|
|
|
def execute(stage, argv, cwd, deadline, payload, evidence):
|
|
request = {'stage': stage, 'argv': argv, 'cwd': cwd,
|
|
'deadline': deadline,
|
|
'input_b64': base64.b64encode(payload).decode('ascii')}
|
|
wire_input = json.dumps(request, separators=(',', ':')).encode('ascii')
|
|
record = {'stage': stage}
|
|
evidence.append(record)
|
|
try:
|
|
supervisor = subprocess.Popen([sys.executable, '-I', '-c', SUPERVISOR],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
start = process_start(supervisor.pid)
|
|
if start is None:
|
|
raise RuntimeError('supervisor start token unavailable')
|
|
watcher = subprocess.Popen(
|
|
[sys.executable, '-I', '-c', WATCHDOG, str(supervisor.pid),
|
|
str(start), str(deadline + RECOVERY_ALLOWANCE + 0.5)],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
watcher_start = process_start(watcher.pid)
|
|
try:
|
|
raw, wrapper_stderr = supervisor.communicate(
|
|
input=wire_input, timeout=deadline + RECOVERY_ALLOWANCE + 2)
|
|
watchdog_raw, watchdog_stderr = watcher.communicate(
|
|
input=b'DONE\n', timeout=3)
|
|
except subprocess.TimeoutExpired:
|
|
record['outer_timeout'] = True
|
|
watchdog_raw, watchdog_stderr = watcher.communicate(
|
|
input=b'KILL\n', timeout=3)
|
|
try:
|
|
raw, wrapper_stderr = supervisor.communicate(timeout=2)
|
|
except subprocess.TimeoutExpired:
|
|
record['watchdog_rc'] = watcher.returncode
|
|
record['watchdog_stdout'] = watchdog_raw.decode('utf-8', 'replace')
|
|
record['wrapper_exception'] = 'supervisor remained after same-uid kill'
|
|
return None
|
|
record['watchdog_rc'] = watcher.returncode
|
|
record['watchdog_stdout'] = watchdog_raw.decode('utf-8', 'replace')
|
|
record['watchdog_stderr_tail'] = watchdog_stderr[-300:].decode('utf-8', 'replace')
|
|
record['watchdog_pid'] = watcher.pid
|
|
record['watchdog_start'] = watcher_start
|
|
record['supervisor_pid'] = supervisor.pid
|
|
record['supervisor_start'] = start
|
|
record['wrapper_rc'] = supervisor.returncode
|
|
record['wire_len'] = len(raw)
|
|
record['wire_sha256'] = sha(raw)
|
|
except BaseException as exc:
|
|
record['wrapper_exception'] = repr(exc)
|
|
return None
|
|
if record.get('outer_timeout'):
|
|
return None
|
|
try:
|
|
watchdog_status = json.loads(record['watchdog_stdout'])
|
|
except (ValueError, TypeError):
|
|
return None
|
|
if (record['watchdog_rc'] != 0 or watchdog_status !=
|
|
{'status': 'finished', 'target': supervisor.pid}):
|
|
return None
|
|
if supervisor.returncode != 0:
|
|
record['wrapper_stderr_tail'] = wrapper_stderr[-300:].decode('utf-8', 'replace')
|
|
return None
|
|
try:
|
|
wire = json.loads(raw)
|
|
sent = base64.b64decode(wire['sent_b64'], validate=True)
|
|
unread = base64.b64decode(wire['unread_b64'], validate=True)
|
|
stdout = base64.b64decode(wire['stdout_b64'], validate=True)
|
|
stderr = base64.b64decode(wire['stderr_b64'], validate=True)
|
|
meta = wire['meta']
|
|
except (ValueError, KeyError, TypeError) as exc:
|
|
record['parse_error'] = repr(exc)
|
|
return None
|
|
record['meta'] = meta
|
|
if not valid_meta(meta, stage, argv, cwd, deadline,
|
|
supervisor.pid, start,
|
|
wire_input, payload, sent, unread, stdout, stderr):
|
|
record['validation_error'] = 'metadata inconsistent'
|
|
return None
|
|
return meta, stdout, stderr
|
|
|
|
|
|
def token(name):
|
|
return bool(name) and len(name) <= 32 and name[0].isalpha() and name[0].isascii() and all(
|
|
(ch.isascii() and (ch.isalnum() or ch in '_.-')) for ch in name)
|
|
|
|
|
|
def oracle(config, units, root):
|
|
if root not in units:
|
|
return ('ERROR', 'NO_ROOT')
|
|
seen, visiting, edges = set(), set(), set()
|
|
depth = {}
|
|
cpp = False
|
|
|
|
def walk(name):
|
|
nonlocal cpp
|
|
if name not in units:
|
|
raise ValueError('MISSING')
|
|
if name in visiting:
|
|
raise ValueError('CYCLE')
|
|
if name in seen:
|
|
return
|
|
language, origin, deps = units[name]
|
|
if origin == 'HOST':
|
|
raise ValueError('HOST_DEP')
|
|
visiting.add(name)
|
|
cpp = cpp or language == 'CPP'
|
|
for dep in deps:
|
|
edges.add(('EDGE', name, dep))
|
|
walk(dep)
|
|
depth[name] = max((depth[dep] for dep in deps), default=-1) + 1
|
|
visiting.remove(name)
|
|
seen.add(name)
|
|
|
|
try:
|
|
walk(root)
|
|
except ValueError as exc:
|
|
return ('ERROR', str(exc))
|
|
target, sysroot, cc, cxx, cr, cppr = config
|
|
prefix = '' if sysroot == '/' else sysroot
|
|
rows = [('PLAN', target, root), ('LINK', cxx if cpp else cc),
|
|
('RUNTIME', cr, prefix + '/lib/' + cr)]
|
|
if cpp:
|
|
rows.append(('RUNTIME', cppr, prefix + '/lib/' + cppr))
|
|
rows.extend(edges)
|
|
for name in seen:
|
|
lang = units[name][0]
|
|
rows.append(('TOOL', name, cxx if lang == 'CPP' else cc))
|
|
rows.append(('DEPTH', name, str(depth[name])))
|
|
return rows
|
|
|
|
|
|
def encode_case(config, units, root):
|
|
lines = ['CONFIG ' + ' '.join(config)]
|
|
for name, (language, origin, deps) in units.items():
|
|
lines.append('UNIT %s %s %s %s' % (
|
|
name, language, origin, ','.join(deps) if deps else '-'))
|
|
lines.extend(['ROOT ' + root, 'END'])
|
|
return ('\n'.join(lines) + '\n').encode('ascii')
|
|
|
|
|
|
def cases():
|
|
base = ('aarch64-linux-gnu', '/sdk/a64', 'aarch64-linux-gnu-gcc',
|
|
'aarch64-linux-gnu-g++', 'libc.so.6', 'libstdc++.so.6')
|
|
data = []
|
|
data.append((base, {'app': ('CPP', 'TARGET', ['core']),
|
|
'core': ('C', 'TARGET', [])}, 'app'))
|
|
data.append((base, {'front': ('C', 'TARGET', ['model']),
|
|
'model': ('CPP', 'TARGET', ['math']),
|
|
'math': ('C', 'TARGET', [])}, 'front'))
|
|
data.append((base, {'plain': ('C', 'TARGET', [])}, 'plain'))
|
|
data.append((base, {'app': ('C', 'TARGET', ['adapter']),
|
|
'adapter': ('C', 'TARGET', ['engine']),
|
|
'engine': ('CPP', 'TARGET', [])}, 'app'))
|
|
changed = ('riscv64-linux-gnu', '/', 'tool/gcc', 'tool/g++', 'libc_custom.so',
|
|
'libcxx_custom.so')
|
|
data.append((changed, {'service': ('C', 'TARGET', ['helper']),
|
|
'helper': ('CPP', 'TARGET', []),
|
|
'host_tool': ('CPP', 'HOST', [])}, 'service'))
|
|
same_runtime = ('aarch64-linux-gnu', '/same', 'ccA', 'cxxA',
|
|
'libshared.so', 'libshared.so')
|
|
data.append((same_runtime, {'same': ('CPP', 'TARGET', [])}, 'same'))
|
|
rng = random.Random(61723)
|
|
for index in range(8):
|
|
nodes = {}
|
|
for n in range(1 + index % 5):
|
|
deps = ['u' + str(i) for i in range(n) if rng.randrange(3) == 0]
|
|
nodes['u' + str(n)] = (rng.choice(('C', 'CPP')), 'TARGET', deps)
|
|
last = 'u' + str(len(nodes) - 1)
|
|
data.append((base, nodes, last))
|
|
data.append((base, {'app': ('C', 'TARGET', ['host']),
|
|
'host': ('CPP', 'HOST', [])}, 'app'))
|
|
data.append((base, {'app': ('CPP', 'TARGET', ['absent'])}, 'app'))
|
|
data.append((base, {'left': ('C', 'TARGET', ['right']),
|
|
'right': ('CPP', 'TARGET', ['left'])}, 'left'))
|
|
return data
|
|
|
|
|
|
def expected_stream():
|
|
chunks, expected = [], []
|
|
for config, units, root in cases():
|
|
chunks.append(encode_case(config, units, root))
|
|
expected.append(oracle(config, units, root))
|
|
chunks.append(b'CONFIG a /x cc cxx cr cppr\nUNIT a C TARGET -\nROOT a\nEND\n')
|
|
expected.append(oracle(('a', '/x', 'cc', 'cxx', 'cr', 'cppr'),
|
|
{'a': ('C', 'TARGET', [])}, 'a'))
|
|
chunks.append(b'CONFIG a /x cc cxx cr cppr\nUNIT a C TARGET b,b\nROOT a\nEND\n')
|
|
expected.append(('ERROR', 'DUPLICATE'))
|
|
chunks.append(b'CONFIG a /x cc cxx cr cppr\nUNIT a CPP TARGET -\nROOT a\nEND\n')
|
|
expected.append(oracle(('a', '/x', 'cc', 'cxx', 'cr', 'cppr'),
|
|
{'a': ('CPP', 'TARGET', [])}, 'a'))
|
|
chunks.append(b'CONFIG a /x cc cxx cr cppr\nUNIT a C TARGET -\nROOT a\nEND')
|
|
expected.append(('ERROR', 'FORMAT'))
|
|
return b''.join(chunks), expected
|
|
|
|
|
|
def decode_output(raw, expected):
|
|
if not raw.endswith(b'\n') or b'\r' in raw or b'\0' in raw:
|
|
return False, 'line ending'
|
|
try:
|
|
text = raw.decode('ascii')
|
|
except UnicodeDecodeError:
|
|
return False, 'non ascii'
|
|
lines = text.split('\n')
|
|
if lines[-1] != '':
|
|
return False, 'unterminated'
|
|
lines.pop()
|
|
blocks = []
|
|
active = []
|
|
for line in lines:
|
|
if not line or line.startswith(' ') or line.endswith(' ') or ' ' in line or '\t' in line:
|
|
return False, 'malformed output line'
|
|
if line == 'END':
|
|
blocks.append(active)
|
|
active = []
|
|
else:
|
|
active.append(tuple(line.split(' ')))
|
|
if active or len(blocks) != len(expected):
|
|
return False, 'block count'
|
|
for index, (block, wanted) in enumerate(zip(blocks, expected)):
|
|
if isinstance(wanted, tuple):
|
|
if block != [wanted]:
|
|
return False, 'error block %d' % index
|
|
else:
|
|
if Counter(block) != Counter(wanted):
|
|
return False, 'plan block %d' % index
|
|
return True, 'ok'
|
|
|
|
|
|
def check(candidate):
|
|
evidence = []
|
|
scratch = None
|
|
candidate_result = None
|
|
compile_result = None
|
|
cleanup_result = None
|
|
absence_result = None
|
|
failure = None
|
|
try:
|
|
scratch = Path(tempfile.mkdtemp(prefix='cc-meson-cpp-grade-'))
|
|
work = scratch / 'work'
|
|
work.mkdir(mode=0o777)
|
|
work.chmod(0o777)
|
|
for relative in CANDIDATES:
|
|
source = candidate / relative
|
|
if not stat.S_ISREG(source.lstat().st_mode):
|
|
raise ValueError('missing or unsafe candidate file')
|
|
target = work / relative
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(source.read_bytes())
|
|
target.chmod(0o644)
|
|
(work / 'src').chmod(0o777)
|
|
compile_argv = ['make', '-C', str(work), 'all']
|
|
compile_result = execute('compiler', compile_argv, str(work),
|
|
COMPILER_DEADLINE, b'', evidence)
|
|
if compile_result is None:
|
|
failure = 'compiler wire/metadata failure'
|
|
elif compile_result[0]['controlled_rc'] != 0 or compile_result[0]['timeout']:
|
|
failure = 'compiler rc/timeout %s' % compile_result[0]['controlled_rc']
|
|
else:
|
|
payload, expected = expected_stream()
|
|
executable = str(work / 'build' / 'plan')
|
|
candidate_result = execute('candidate', [executable], str(work),
|
|
CANDIDATE_DEADLINE, payload, evidence)
|
|
if candidate_result is None:
|
|
failure = 'candidate wire/metadata failure'
|
|
elif candidate_result[0]['controlled_rc'] != 0 or candidate_result[0]['timeout']:
|
|
failure = 'candidate rc/timeout %s' % candidate_result[0]['controlled_rc']
|
|
elif (candidate_result[0]['stdin_sent_len'] != len(payload) or
|
|
candidate_result[0]['stdin_read_len'] != len(payload) or
|
|
candidate_result[0]['stdin_broken_pipe']):
|
|
failure = 'candidate input delivery incomplete'
|
|
else:
|
|
good, detail = decode_output(candidate_result[1], expected)
|
|
if not good:
|
|
failure = 'behavior: ' + detail
|
|
except BaseException as exc:
|
|
failure = 'preparation/check exception: ' + repr(exc)
|
|
finally:
|
|
if scratch is not None:
|
|
try:
|
|
cleanup_argv = [sys.executable, '-I', '-c', CHILD_ACTION,
|
|
'cleanup', str(scratch)]
|
|
cleanup_result = execute('cleanup', cleanup_argv, '/tmp',
|
|
CLEANUP_DEADLINE, b'', evidence)
|
|
if cleanup_result is None or cleanup_result[0]['controlled_rc'] != 0:
|
|
failure = failure or 'cleanup failed'
|
|
except BaseException as exc:
|
|
failure = failure or 'cleanup exception: ' + repr(exc)
|
|
try:
|
|
absence_argv = [sys.executable, '-I', '-c', CHILD_ACTION,
|
|
'absence', str(scratch)]
|
|
absence_result = execute('absence', absence_argv, '/tmp',
|
|
ABSENCE_DEADLINE, b'', evidence)
|
|
if absence_result is None or absence_result[0]['controlled_rc'] != 0:
|
|
failure = failure or 'absence failed'
|
|
if absence_result is not None and absence_result[1] != b'ABSENT\n':
|
|
failure = failure or 'scratch still present'
|
|
except BaseException as exc:
|
|
failure = failure or 'absence exception: ' + repr(exc)
|
|
metadata = []
|
|
for item in evidence:
|
|
row = {key: value for key, value in item.items() if key != 'meta'}
|
|
if 'meta' in item:
|
|
row['meta'] = item['meta']
|
|
metadata.append(row)
|
|
packed = base64.b64encode(zlib.compress(json.dumps(metadata, separators=(',', ':')).encode('utf-8')))
|
|
print('CC_MESON_EVIDENCE_Z ' + packed.decode('ascii'), file=sys.stderr)
|
|
if failure is None and compile_result and candidate_result and cleanup_result and absence_result:
|
|
print('PASS')
|
|
return 0
|
|
print(failure or 'incomplete evidence')
|
|
return 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(check(Path(sys.argv[1])))
|