amos 9d633e9e67 Batch 15 (20261023-C-hwe-scale-15): 320 HWE-derived repair tasks
One user-space repair task per eligible HWE debug-diag / compile-deploy base
ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content
filter. Each ships a public spec with a check-to-basis table, at least two
defects with single-fix fixtures, a symptom-masking fixture and an alternative
correct implementation. Claude re-checked all 320: structure, the HWE quality
items and trace6 Docker calibration receipts against current digests
(320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120
ad-hoc family names; the next batch maps them to the canonical list.
The batch's 1,256-line progress log stays in its factory report and the
36-local branch batch15-hwe.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 01:55:56 +08:00

905 lines
35 KiB
Python

"""Private independent plan oracle and same-identity execution supervisor."""
import base64
from collections import Counter
import hashlib
import json
import math
import os
from pathlib import Path
import random
import shutil
import stat
import subprocess
import sys
import tempfile
import zlib
CANDIDATES = ("Makefile", "src/main.cpp", "src/planner.cpp", "src/planner.hpp")
COMPILER_DEADLINE = 20.0
CANDIDATE_DEADLINE = 14.0
CLEANUP_DEADLINE = 6.0
ABSENCE_DEADLINE = 6.0
RECOVERY_ALLOWANCE = 3.0
SUPERVISOR = r'''
import base64, ctypes, hashlib, json, os, selectors, signal, subprocess, sys, time
BRIDGE = r"""
import base64, json, os, sys
argv = json.loads(base64.b64decode(sys.argv[1], validate=True))
fd = int(sys.argv[2])
raw = open('/proc/self/stat', 'rb').read()
fields = raw[raw.rfind(b')')+2:].split()
report = {'pid':os.getpid(),'start':int(fields[19]),'argv':argv,
'cwd':os.getcwd(),'uid':os.getuid(),'gid':os.getgid(),
'groups':os.getgroups()}
os.write(fd, json.dumps(report, separators=(',', ':')).encode('ascii'))
os.close(fd)
os.execvp(argv[0], argv)
"""
def sha(data):
return hashlib.sha256(data).hexdigest()
def proc_one(pid):
try:
data = open('/proc/%d/stat' % pid, 'rb').read()
except (FileNotFoundError, ProcessLookupError):
return None
close = data.rfind(b')')
if close < 0:
raise RuntimeError('proc stat missing comm end')
tail = data[close + 2:].split()
if len(tail) < 20:
raise RuntimeError('proc stat too short')
return {'pid':pid, 'state':tail[0].decode('ascii'), 'ppid':int(tail[1]),
'pgid':int(tail[2]), 'start':int(tail[19])}
def proc_table():
table = {}
for item in os.listdir('/proc'):
if item.isdecimal():
pid = int(item)
entry = proc_one(pid)
if entry is not None:
table[pid] = entry
return table
def descendants(table, supervisor):
owned = set()
active = {supervisor}
while active:
next_active = set()
for pid, row in table.items():
if pid not in owned and row['ppid'] in active and row['state'] not in ('Z', 'X'):
owned.add(pid)
next_active.add(pid)
active = next_active
return {pid:table[pid] for pid in owned}
def fresh_signal(row, sig):
fresh = proc_one(row['pid'])
if fresh is None or fresh['start'] != row['start'] or fresh['state'] in ('Z', 'X'):
return False
try:
os.kill(row['pid'], sig)
except ProcessLookupError:
return False
return True
def reap_adopted(table, leader_pid, errors):
for pid, row in table.items():
if row['ppid'] != os.getpid() or pid == leader_pid:
continue
try:
os.waitpid(pid, os.WNOHANG)
except ChildProcessError:
pass
except ProcessLookupError:
pass
except Exception as exc:
errors.append('adopted_wait:' + repr(exc))
def cleanup_children(proc, leader_pid, errors):
stopped = False
end = time.monotonic() + 2.0
last = {}
while time.monotonic() < end:
if proc is not None:
proc.poll()
table = proc_table()
last = descendants(table, os.getpid())
for row in last.values():
fresh_signal(row, signal.SIGKILL)
if proc is not None and proc.poll() is None:
own = proc_one(leader_pid)
if own is not None:
fresh_signal(own, signal.SIGKILL)
if proc is not None:
proc.poll()
reap_adopted(proc_table(), leader_pid, errors)
again = descendants(proc_table(), os.getpid())
if not again and (proc is None or proc.poll() is not None):
stopped = True
break
time.sleep(0.015)
if proc is not None:
try:
proc.wait(timeout=0.2)
except Exception as exc:
errors.append('leader_wait:' + repr(exc))
if not stopped:
errors.append('descendants_remaining:' + str(sorted(last)))
return stopped
def transfer(proc, writer, payload, deadline):
selector = selectors.DefaultSelector()
output = bytearray()
diagnostics = bytearray()
sent = bytearray()
closed = False
broken = False
os.set_blocking(writer, False)
for stream in (proc.stdout, proc.stderr):
os.set_blocking(stream.fileno(), False)
selector.register(proc.stdout, selectors.EVENT_READ, output)
selector.register(proc.stderr, selectors.EVENT_READ, diagnostics)
if payload:
selector.register(writer, selectors.EVENT_WRITE, None)
else:
os.close(writer)
closed = True
end = time.monotonic() + deadline
timed_out = False
while selector.get_map():
remaining = end - time.monotonic()
if remaining <= 0:
timed_out = True
break
for key, events in selector.select(min(remaining, 0.05)):
stream = key.fileobj
if stream == writer:
try:
n = os.write(writer, payload[len(sent):len(sent)+65536])
sent.extend(payload[len(sent):len(sent)+n])
except BrokenPipeError:
broken = True
n = 0
if broken or len(sent) == len(payload):
selector.unregister(stream)
os.close(writer)
closed = True
else:
chunk = os.read(stream.fileno(), 65536)
if chunk:
key.data.extend(chunk)
else:
selector.unregister(stream)
stream.close()
selector.close()
if proc.poll() is None:
remaining = end - time.monotonic()
if remaining <= 0:
timed_out = True
else:
try:
proc.wait(timeout=remaining)
except subprocess.TimeoutExpired:
timed_out = True
return bytes(output), bytes(diagnostics), bytes(sent), closed, broken, timed_out
def drain_unread(fd, errors):
unread = bytearray()
try:
os.set_blocking(fd, False)
while True:
try:
chunk = os.read(fd, 65536)
except BlockingIOError:
errors.append('stdin_pipe_writer_alive')
break
if not chunk:
break
unread.extend(chunk)
finally:
os.close(fd)
return bytes(unread)
def read_preexec(fd, deadline):
os.set_blocking(fd, False)
data = bytearray()
end = time.monotonic() + min(deadline, 2.0)
try:
while time.monotonic() < end:
try:
part = os.read(fd, 8192)
except BlockingIOError:
time.sleep(0.002)
continue
if not part:
break
data.extend(part)
if len(data) > 8192:
raise RuntimeError('preexec report too large')
if not data:
raise RuntimeError('missing preexec report')
return json.loads(data)
finally:
os.close(fd)
def main():
raw_request = sys.stdin.buffer.read()
request = json.loads(raw_request)
stage = request['stage']
argv = request['argv']
cwd = request['cwd']
deadline = request['deadline']
payload = base64.b64decode(request['input_b64'], validate=True)
errors = []
supervisor = os.getpid()
own = proc_one(supervisor)
meta = {'stage':stage,'supervisor_pid':supervisor,'supervisor_start':own['start'],
'uid':os.getuid(),'gid':os.getgid(),'groups':os.getgroups(),
'argv':argv,'cwd':cwd,'deadline_seconds':deadline,
'request_wire_len':len(raw_request),'request_wire_sha256':sha(raw_request),
'stdin_len':len(payload),'stdin_sha256':sha(payload),
'stdin_sent_len':0,'stdin_sent_sha256':sha(b''),
'stdin_read_len':0,'stdin_read_sha256':sha(b''),
'stdin_unread_len':0,'stdin_unread_sha256':sha(b''),
'stdin_closed':False,'stdin_broken_pipe':False,
'preexec':None,
'leader_pid':None,'leader_start':None,'leader_rc':None,
'controlled_rc':None,'leader_reaped':False,'timeout':False,
'pg_absent':False,'descendants_absent':False,'elapsed_seconds':0.0,
'stdout_len':0,'stdout_sha256':sha(b''),
'stderr_len':0,'stderr_sha256':sha(b''),'exceptions':errors}
stdout = b''
stderr = b''
sent = b''
unread = b''
writer = None
drain_fd = None
reader = None
report_read = None
report_write = None
proc = None
start = time.monotonic()
try:
libc = ctypes.CDLL(None, use_errno=True)
if libc.prctl(36, 1, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'PR_SET_CHILD_SUBREAPER')
reader, writer = os.pipe()
drain_fd = os.dup(reader)
report_read, report_write = os.pipe()
bridge_argv = [sys.executable, '-I', '-c', BRIDGE,
base64.b64encode(json.dumps(argv).encode('utf-8')).decode('ascii'),
str(report_write)]
proc = subprocess.Popen(bridge_argv, cwd=cwd, stdin=reader,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
start_new_session=True, pass_fds=(report_write,))
os.close(report_write)
report_write = None
os.close(reader)
reader = None
meta['leader_pid'] = proc.pid
child = proc_one(proc.pid)
if child is None:
raise RuntimeError('leader disappeared before start binding')
meta['leader_start'] = child['start']
meta['preexec'] = read_preexec(report_read, deadline)
report_read = None
stdout, stderr, sent, closed, broken, timed_out = transfer(
proc, writer, payload, deadline)
writer = None
meta['stdin_closed'] = closed
meta['stdin_broken_pipe'] = broken
meta['timeout'] = timed_out
meta['controlled_rc'] = 124 if timed_out else proc.returncode
meta['leader_rc'] = proc.returncode
except BaseException as exc:
errors.append('execute:' + repr(exc))
finally:
try:
absent = cleanup_children(proc, meta['leader_pid'] or -1, errors)
meta['descendants_absent'] = bool(absent)
except BaseException as exc:
errors.append('cleanup_children:' + repr(exc))
for label, fd in (('writer', writer), ('reader', reader),
('report_read', report_read), ('report_write', report_write)):
if fd is not None:
try:
os.close(fd)
if label == 'writer':
meta['stdin_closed'] = True
except OSError as exc:
errors.append('close_' + label + ':' + repr(exc))
if drain_fd is not None:
try:
unread = drain_unread(drain_fd, errors)
except BaseException as exc:
errors.append('drain_unread:' + repr(exc))
try:
if proc is not None:
meta['leader_reaped'] = proc.returncode is not None
if meta['leader_rc'] is None and proc.returncode is not None:
meta['leader_rc'] = proc.returncode
table = proc_table()
meta['pg_absent'] = not any(
row['pgid'] == proc.pid and row['state'] not in ('Z', 'X')
for row in table.values())
else:
meta['pg_absent'] = bool(meta['descendants_absent'])
except BaseException as exc:
errors.append('pg_absence:' + repr(exc))
meta['elapsed_seconds'] = time.monotonic() - start
meta['stdin_sent_len'], meta['stdin_sent_sha256'] = len(sent), sha(sent)
meta['stdin_unread_len'], meta['stdin_unread_sha256'] = len(unread), sha(unread)
if not sent.endswith(unread):
errors.append('stdin_unread_not_suffix')
consumed = sent[:len(sent)-len(unread)]
meta['stdin_read_len'], meta['stdin_read_sha256'] = len(consumed), sha(consumed)
meta['stdout_len'], meta['stdout_sha256'] = len(stdout), sha(stdout)
meta['stderr_len'], meta['stderr_sha256'] = len(stderr), sha(stderr)
wire = {'meta':meta,'sent_b64':base64.b64encode(sent).decode('ascii'),
'unread_b64':base64.b64encode(unread).decode('ascii'),
'stdout_b64':base64.b64encode(stdout).decode('ascii'),
'stderr_b64':base64.b64encode(stderr).decode('ascii')}
sys.stdout.write(json.dumps(wire, separators=(',', ':')) + '\n')
if __name__ == '__main__':
main()
'''
CHILD_ACTION = r'''
import os, shutil, sys
kind, path = sys.argv[1:3]
if kind == 'cleanup':
shutil.rmtree(path)
elif kind == 'absence':
try:
os.lstat(path)
except FileNotFoundError:
sys.stdout.buffer.write(b'ABSENT\n')
else:
sys.stdout.buffer.write(b'PRESENT\n')
else:
raise SystemExit(4)
'''
WATCHDOG = r'''
import json, os, select, signal, sys, time
def one(pid):
try:
data = open('/proc/%d/stat' % pid, 'rb').read()
except (FileNotFoundError, ProcessLookupError):
return None
fields = data[data.rfind(b')')+2:].split()
if len(fields) < 20:
raise RuntimeError('short proc stat')
return {'pid':pid,'state':fields[0].decode('ascii'),'ppid':int(fields[1]),
'start':int(fields[19])}
def table():
result = {}
for name in os.listdir('/proc'):
if name.isdecimal():
row = one(int(name))
if row is not None:
result[row['pid']] = row
return result
def closure(rows, root):
found = {}
wave = {root}
while wave:
next_wave = set()
for pid, row in rows.items():
if pid not in found and row['ppid'] in wave and row['state'] not in ('Z','X'):
found[pid] = row
next_wave.add(pid)
wave = next_wave
return found
def fresh_signal(row, sig):
current = one(row['pid'])
if current is None or current['start'] != row['start'] or current['state'] in ('Z','X'):
return False
try:
os.kill(row['pid'], sig)
except ProcessLookupError:
return False
return True
def alive_token(row):
current = one(row['pid'])
return (current is not None and current['start'] == row['start']
and current['state'] not in ('Z','X'))
def main():
pid = int(sys.argv[1]); start = int(sys.argv[2]); deadline = float(sys.argv[3])
root = {'pid':pid,'start':start}
until = time.monotonic() + deadline
while True:
remaining = until - time.monotonic()
ready, _, _ = select.select([sys.stdin], [], [], max(0, min(remaining, 0.1)))
if ready:
command = sys.stdin.buffer.readline()
if command == b'DONE\n':
print(json.dumps({'status':'finished','target':pid}))
return 0
if command == b'KILL\n':
break
if not command:
break
if remaining <= 0:
break
current = one(pid)
if current is None or current['start'] != start or current['state'] in ('Z','X'):
print(json.dumps({'status':'already_exited','target':pid}))
return 0
fresh_signal(root, signal.SIGSTOP)
tracked = closure(table(), pid)
end = time.monotonic() + 2.0
while time.monotonic() < end:
for row in closure(table(), pid).values():
tracked[row['pid']] = row
for row in tracked.values():
fresh_signal(row, signal.SIGKILL)
if not any(alive_token(row) for row in tracked.values()):
break
time.sleep(0.015)
fresh_signal(root, signal.SIGKILL)
alive = [pid for pid,row in tracked.items() if alive_token(row)]
print(json.dumps({'status':'forced','target':pid,'tracked':len(tracked),'alive':alive}))
return 1 if alive else 0
if __name__ == '__main__':
raise SystemExit(main())
'''
def sha(data):
return hashlib.sha256(data).hexdigest()
def process_start(pid):
try:
raw = Path('/proc/%d/stat' % pid).read_bytes()
except (FileNotFoundError, ProcessLookupError):
return None
fields = raw[raw.rfind(b')') + 2:].split()
if len(fields) < 20:
raise RuntimeError('short supervisor proc stat')
return int(fields[19])
def exact_int(value, minimum=None):
return type(value) is int and (minimum is None or value >= minimum)
def valid_meta(meta, stage, argv, cwd, deadline, supervisor_pid, supervisor_start, wire_input,
payload, sent, unread, stdout, stderr):
required = ('stage', 'supervisor_pid', 'supervisor_start', 'uid', 'gid',
'groups', 'argv', 'cwd', 'deadline_seconds', 'stdin_len',
'stdin_sha256', 'request_wire_len', 'request_wire_sha256',
'stdin_sent_len', 'stdin_sent_sha256', 'stdin_closed',
'stdin_read_len', 'stdin_read_sha256', 'stdin_unread_len',
'stdin_unread_sha256', 'preexec',
'stdin_broken_pipe', 'leader_pid', 'leader_start', 'leader_rc',
'controlled_rc', 'leader_reaped', 'timeout', 'pg_absent',
'descendants_absent', 'elapsed_seconds', 'stdout_len',
'stdout_sha256', 'stderr_len', 'stderr_sha256', 'exceptions')
if not isinstance(meta, dict) or any(key not in meta for key in required):
return False
if meta['stage'] != stage or meta['argv'] != argv or meta['cwd'] != cwd:
return False
if not isinstance(meta['argv'], list) or any(type(x) is not str for x in meta['argv']):
return False
if type(meta['deadline_seconds']) not in (int, float) or meta['deadline_seconds'] != deadline:
return False
if not exact_int(meta['supervisor_pid'], 1) or not exact_int(meta['leader_pid'], 1):
return False
if meta['supervisor_pid'] != supervisor_pid:
return False
if meta['supervisor_pid'] == meta['leader_pid']:
return False
if not exact_int(meta['supervisor_start'], 1) or not exact_int(meta['leader_start'], 1):
return False
if meta['supervisor_start'] != supervisor_start:
return False
isolated = os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534'
expected_uid = 65534 if isolated else os.getuid()
expected_gid = 65534 if isolated else os.getgid()
if not exact_int(meta['uid']) or meta['uid'] != expected_uid:
return False
if not exact_int(meta['gid']) or meta['gid'] != expected_gid:
return False
if type(meta['groups']) is not list or any(not exact_int(x) for x in meta['groups']):
return False
if isolated and meta['groups'] != []:
return False
preexec = meta['preexec']
if type(preexec) is not dict:
return False
if set(preexec) != {'pid', 'start', 'argv', 'cwd', 'uid', 'gid', 'groups'}:
return False
if not exact_int(preexec['pid'], 1) or preexec['pid'] != meta['leader_pid']:
return False
if not exact_int(preexec['start'], 1) or preexec['start'] != meta['leader_start']:
return False
if preexec['argv'] != argv or preexec['cwd'] != cwd:
return False
if (not exact_int(preexec['uid']) or preexec['uid'] != expected_uid or
not exact_int(preexec['gid']) or preexec['gid'] != expected_gid):
return False
if type(preexec['groups']) is not list or any(not exact_int(x) for x in preexec['groups']):
return False
if isolated and preexec['groups'] != []:
return False
if type(meta['timeout']) is not bool or type(meta['leader_reaped']) is not bool:
return False
if type(meta['pg_absent']) is not bool or type(meta['descendants_absent']) is not bool:
return False
if not meta['leader_reaped'] or not meta['pg_absent'] or not meta['descendants_absent']:
return False
if not exact_int(meta['leader_rc']) or not exact_int(meta['controlled_rc']):
return False
if meta['timeout']:
if meta['controlled_rc'] != 124:
return False
elif meta['controlled_rc'] != meta['leader_rc']:
return False
elapsed = meta['elapsed_seconds']
if type(elapsed) not in (int, float) or not math.isfinite(elapsed):
return False
if not 0 <= elapsed <= deadline + RECOVERY_ALLOWANCE:
return False
if type(meta['exceptions']) is not list or meta['exceptions']:
return False
if not exact_int(meta['stdin_len'], 0) or meta['stdin_len'] != len(payload):
return False
if meta['stdin_sha256'] != sha(payload):
return False
if not exact_int(meta['request_wire_len'], 0) or meta['request_wire_len'] != len(wire_input):
return False
if meta['request_wire_sha256'] != sha(wire_input):
return False
if type(meta['stdin_closed']) is not bool or not meta['stdin_closed']:
return False
if type(meta['stdin_broken_pipe']) is not bool:
return False
if not exact_int(meta['stdin_sent_len'], 0) or meta['stdin_sent_len'] != len(sent):
return False
if meta['stdin_sent_sha256'] != sha(sent) or sent != payload[:len(sent)]:
return False
if not sent.endswith(unread):
return False
consumed = sent[:len(sent)-len(unread)]
for label, raw in (('stdin_read', consumed), ('stdin_unread', unread)):
if not exact_int(meta[label + '_len'], 0) or meta[label + '_len'] != len(raw):
return False
if meta[label + '_sha256'] != sha(raw):
return False
for prefix, raw in (('stdout', stdout), ('stderr', stderr)):
if not exact_int(meta[prefix + '_len'], 0) or meta[prefix + '_len'] != len(raw):
return False
if meta[prefix + '_sha256'] != sha(raw):
return False
return True
def execute(stage, argv, cwd, deadline, payload, evidence):
request = {'stage': stage, 'argv': argv, 'cwd': cwd,
'deadline': deadline,
'input_b64': base64.b64encode(payload).decode('ascii')}
wire_input = json.dumps(request, separators=(',', ':')).encode('ascii')
record = {'stage': stage}
evidence.append(record)
try:
supervisor = subprocess.Popen([sys.executable, '-I', '-c', SUPERVISOR],
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
start = process_start(supervisor.pid)
if start is None:
raise RuntimeError('supervisor start token unavailable')
watcher = subprocess.Popen(
[sys.executable, '-I', '-c', WATCHDOG, str(supervisor.pid),
str(start), str(deadline + RECOVERY_ALLOWANCE + 0.5)],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
watcher_start = process_start(watcher.pid)
try:
raw, wrapper_stderr = supervisor.communicate(
input=wire_input, timeout=deadline + RECOVERY_ALLOWANCE + 2)
watchdog_raw, watchdog_stderr = watcher.communicate(
input=b'DONE\n', timeout=3)
except subprocess.TimeoutExpired:
record['outer_timeout'] = True
watchdog_raw, watchdog_stderr = watcher.communicate(
input=b'KILL\n', timeout=3)
try:
raw, wrapper_stderr = supervisor.communicate(timeout=2)
except subprocess.TimeoutExpired:
record['watchdog_rc'] = watcher.returncode
record['watchdog_stdout'] = watchdog_raw.decode('utf-8', 'replace')
record['wrapper_exception'] = 'supervisor remained after same-uid kill'
return None
record['watchdog_rc'] = watcher.returncode
record['watchdog_stdout'] = watchdog_raw.decode('utf-8', 'replace')
record['watchdog_stderr_tail'] = watchdog_stderr[-300:].decode('utf-8', 'replace')
record['watchdog_pid'] = watcher.pid
record['watchdog_start'] = watcher_start
record['supervisor_pid'] = supervisor.pid
record['supervisor_start'] = start
record['wrapper_rc'] = supervisor.returncode
record['wire_len'] = len(raw)
record['wire_sha256'] = sha(raw)
except BaseException as exc:
record['wrapper_exception'] = repr(exc)
return None
if record.get('outer_timeout'):
return None
try:
watchdog_status = json.loads(record['watchdog_stdout'])
except (ValueError, TypeError):
return None
if (record['watchdog_rc'] != 0 or watchdog_status !=
{'status': 'finished', 'target': supervisor.pid}):
return None
if supervisor.returncode != 0:
record['wrapper_stderr_tail'] = wrapper_stderr[-300:].decode('utf-8', 'replace')
return None
try:
wire = json.loads(raw)
sent = base64.b64decode(wire['sent_b64'], validate=True)
unread = base64.b64decode(wire['unread_b64'], validate=True)
stdout = base64.b64decode(wire['stdout_b64'], validate=True)
stderr = base64.b64decode(wire['stderr_b64'], validate=True)
meta = wire['meta']
except (ValueError, KeyError, TypeError) as exc:
record['parse_error'] = repr(exc)
return None
record['meta'] = meta
if not valid_meta(meta, stage, argv, cwd, deadline,
supervisor.pid, start,
wire_input, payload, sent, unread, stdout, stderr):
record['validation_error'] = 'metadata inconsistent'
return None
return meta, stdout, stderr
def token(name):
return bool(name) and len(name) <= 32 and name[0].isalpha() and name[0].isascii() and all(
(ch.isascii() and (ch.isalnum() or ch in '_.-')) for ch in name)
def oracle(config, units, root):
if root not in units:
return ('ERROR', 'NO_ROOT')
seen, visiting, edges = set(), set(), set()
depth = {}
cpp = False
def walk(name):
nonlocal cpp
if name not in units:
raise ValueError('MISSING')
if name in visiting:
raise ValueError('CYCLE')
if name in seen:
return
language, origin, deps = units[name]
if origin == 'HOST':
raise ValueError('HOST_DEP')
visiting.add(name)
cpp = cpp or language == 'CPP'
for dep in deps:
edges.add(('EDGE', name, dep))
walk(dep)
depth[name] = max((depth[dep] for dep in deps), default=-1) + 1
visiting.remove(name)
seen.add(name)
try:
walk(root)
except ValueError as exc:
return ('ERROR', str(exc))
target, sysroot, cc, cxx, cr, cppr = config
prefix = '' if sysroot == '/' else sysroot
rows = [('PLAN', target, root), ('LINK', cxx if cpp else cc),
('RUNTIME', cr, prefix + '/lib/' + cr)]
if cpp:
rows.append(('RUNTIME', cppr, prefix + '/lib/' + cppr))
rows.extend(edges)
for name in seen:
lang = units[name][0]
rows.append(('TOOL', name, cxx if lang == 'CPP' else cc))
rows.append(('DEPTH', name, str(depth[name])))
return rows
def encode_case(config, units, root):
lines = ['CONFIG ' + ' '.join(config)]
for name, (language, origin, deps) in units.items():
lines.append('UNIT %s %s %s %s' % (
name, language, origin, ','.join(deps) if deps else '-'))
lines.extend(['ROOT ' + root, 'END'])
return ('\n'.join(lines) + '\n').encode('ascii')
def cases():
base = ('aarch64-linux-gnu', '/sdk/a64', 'aarch64-linux-gnu-gcc',
'aarch64-linux-gnu-g++', 'libc.so.6', 'libstdc++.so.6')
data = []
data.append((base, {'app': ('CPP', 'TARGET', ['core']),
'core': ('C', 'TARGET', [])}, 'app'))
data.append((base, {'front': ('C', 'TARGET', ['model']),
'model': ('CPP', 'TARGET', ['math']),
'math': ('C', 'TARGET', [])}, 'front'))
data.append((base, {'plain': ('C', 'TARGET', [])}, 'plain'))
data.append((base, {'app': ('C', 'TARGET', ['adapter']),
'adapter': ('C', 'TARGET', ['engine']),
'engine': ('CPP', 'TARGET', [])}, 'app'))
changed = ('riscv64-linux-gnu', '/', 'tool/gcc', 'tool/g++', 'libc_custom.so',
'libcxx_custom.so')
data.append((changed, {'service': ('C', 'TARGET', ['helper']),
'helper': ('CPP', 'TARGET', []),
'host_tool': ('CPP', 'HOST', [])}, 'service'))
same_runtime = ('aarch64-linux-gnu', '/same', 'ccA', 'cxxA',
'libshared.so', 'libshared.so')
data.append((same_runtime, {'same': ('CPP', 'TARGET', [])}, 'same'))
rng = random.Random(61723)
for index in range(8):
nodes = {}
for n in range(1 + index % 5):
deps = ['u' + str(i) for i in range(n) if rng.randrange(3) == 0]
nodes['u' + str(n)] = (rng.choice(('C', 'CPP')), 'TARGET', deps)
last = 'u' + str(len(nodes) - 1)
data.append((base, nodes, last))
data.append((base, {'app': ('C', 'TARGET', ['host']),
'host': ('CPP', 'HOST', [])}, 'app'))
data.append((base, {'app': ('CPP', 'TARGET', ['absent'])}, 'app'))
data.append((base, {'left': ('C', 'TARGET', ['right']),
'right': ('CPP', 'TARGET', ['left'])}, 'left'))
return data
def expected_stream():
chunks, expected = [], []
for config, units, root in cases():
chunks.append(encode_case(config, units, root))
expected.append(oracle(config, units, root))
chunks.append(b'CONFIG a /x cc cxx cr cppr\nUNIT a C TARGET -\nROOT a\nEND\n')
expected.append(oracle(('a', '/x', 'cc', 'cxx', 'cr', 'cppr'),
{'a': ('C', 'TARGET', [])}, 'a'))
chunks.append(b'CONFIG a /x cc cxx cr cppr\nUNIT a C TARGET b,b\nROOT a\nEND\n')
expected.append(('ERROR', 'DUPLICATE'))
chunks.append(b'CONFIG a /x cc cxx cr cppr\nUNIT a CPP TARGET -\nROOT a\nEND\n')
expected.append(oracle(('a', '/x', 'cc', 'cxx', 'cr', 'cppr'),
{'a': ('CPP', 'TARGET', [])}, 'a'))
chunks.append(b'CONFIG a /x cc cxx cr cppr\nUNIT a C TARGET -\nROOT a\nEND')
expected.append(('ERROR', 'FORMAT'))
return b''.join(chunks), expected
def decode_output(raw, expected):
if not raw.endswith(b'\n') or b'\r' in raw or b'\0' in raw:
return False, 'line ending'
try:
text = raw.decode('ascii')
except UnicodeDecodeError:
return False, 'non ascii'
lines = text.split('\n')
if lines[-1] != '':
return False, 'unterminated'
lines.pop()
blocks = []
active = []
for line in lines:
if not line or line.startswith(' ') or line.endswith(' ') or ' ' in line or '\t' in line:
return False, 'malformed output line'
if line == 'END':
blocks.append(active)
active = []
else:
active.append(tuple(line.split(' ')))
if active or len(blocks) != len(expected):
return False, 'block count'
for index, (block, wanted) in enumerate(zip(blocks, expected)):
if isinstance(wanted, tuple):
if block != [wanted]:
return False, 'error block %d' % index
else:
if Counter(block) != Counter(wanted):
return False, 'plan block %d' % index
return True, 'ok'
def check(candidate):
evidence = []
scratch = None
candidate_result = None
compile_result = None
cleanup_result = None
absence_result = None
failure = None
try:
scratch = Path(tempfile.mkdtemp(prefix='cc-meson-cpp-grade-'))
work = scratch / 'work'
work.mkdir(mode=0o777)
work.chmod(0o777)
for relative in CANDIDATES:
source = candidate / relative
if not stat.S_ISREG(source.lstat().st_mode):
raise ValueError('missing or unsafe candidate file')
target = work / relative
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(source.read_bytes())
target.chmod(0o644)
(work / 'src').chmod(0o777)
compile_argv = ['make', '-C', str(work), 'all']
compile_result = execute('compiler', compile_argv, str(work),
COMPILER_DEADLINE, b'', evidence)
if compile_result is None:
failure = 'compiler wire/metadata failure'
elif compile_result[0]['controlled_rc'] != 0 or compile_result[0]['timeout']:
failure = 'compiler rc/timeout %s' % compile_result[0]['controlled_rc']
else:
payload, expected = expected_stream()
executable = str(work / 'build' / 'plan')
candidate_result = execute('candidate', [executable], str(work),
CANDIDATE_DEADLINE, payload, evidence)
if candidate_result is None:
failure = 'candidate wire/metadata failure'
elif candidate_result[0]['controlled_rc'] != 0 or candidate_result[0]['timeout']:
failure = 'candidate rc/timeout %s' % candidate_result[0]['controlled_rc']
elif (candidate_result[0]['stdin_sent_len'] != len(payload) or
candidate_result[0]['stdin_read_len'] != len(payload) or
candidate_result[0]['stdin_broken_pipe']):
failure = 'candidate input delivery incomplete'
else:
good, detail = decode_output(candidate_result[1], expected)
if not good:
failure = 'behavior: ' + detail
except BaseException as exc:
failure = 'preparation/check exception: ' + repr(exc)
finally:
if scratch is not None:
try:
cleanup_argv = [sys.executable, '-I', '-c', CHILD_ACTION,
'cleanup', str(scratch)]
cleanup_result = execute('cleanup', cleanup_argv, '/tmp',
CLEANUP_DEADLINE, b'', evidence)
if cleanup_result is None or cleanup_result[0]['controlled_rc'] != 0:
failure = failure or 'cleanup failed'
except BaseException as exc:
failure = failure or 'cleanup exception: ' + repr(exc)
try:
absence_argv = [sys.executable, '-I', '-c', CHILD_ACTION,
'absence', str(scratch)]
absence_result = execute('absence', absence_argv, '/tmp',
ABSENCE_DEADLINE, b'', evidence)
if absence_result is None or absence_result[0]['controlled_rc'] != 0:
failure = failure or 'absence failed'
if absence_result is not None and absence_result[1] != b'ABSENT\n':
failure = failure or 'scratch still present'
except BaseException as exc:
failure = failure or 'absence exception: ' + repr(exc)
metadata = []
for item in evidence:
row = {key: value for key, value in item.items() if key != 'meta'}
if 'meta' in item:
row['meta'] = item['meta']
metadata.append(row)
packed = base64.b64encode(zlib.compress(json.dumps(metadata, separators=(',', ':')).encode('utf-8')))
print('CC_MESON_EVIDENCE_Z ' + packed.decode('ascii'), file=sys.stderr)
if failure is None and compile_result and candidate_result and cleanup_result and absence_result:
print('PASS')
return 0
print(failure or 'incomplete evidence')
return 1
if __name__ == '__main__':
raise SystemExit(check(Path(sys.argv[1])))