amos 9d633e9e67 Batch 15 (20261023-C-hwe-scale-15): 320 HWE-derived repair tasks
One user-space repair task per eligible HWE debug-diag / compile-deploy base
ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content
filter. Each ships a public spec with a check-to-basis table, at least two
defects with single-fix fixtures, a symptom-masking fixture and an alternative
correct implementation. Claude re-checked all 320: structure, the HWE quality
items and trace6 Docker calibration receipts against current digests
(320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120
ad-hoc family names; the next batch maps them to the canonical list.
The batch's 1,256-line progress log stays in its factory report and the
36-local branch batch15-hwe.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 01:55:56 +08:00

409 lines
18 KiB
Python

"""Independent software-port oracle and candidate-identity process supervisor."""
import base64
import hashlib
import io
import json
import math
import os
from pathlib import Path
import random
import shutil
import subprocess
import sys
import tarfile
import tempfile
import time
SUPERVISOR = r'''
import base64, ctypes, hashlib, json, os, signal, subprocess, sys, time
def exact_stat(pid):
try:
raw = open('/proc/%d/stat' % pid, 'rb').read().decode('ascii')
except (FileNotFoundError, ProcessLookupError):
return None
close = raw.rfind(')')
if close < 0: raise RuntimeError('bad proc stat')
fields = raw[close + 2:].split()
return (fields[0], int(fields[1]), int(fields[2]), int(fields[19]))
def table():
found = {}
for name in os.listdir('/proc'):
if name.isdecimal():
pid = int(name)
item = exact_stat(pid)
if item is not None: found[pid] = item
return found
def living(item):
return item is not None and item[0] not in ('Z', 'X')
def closure(snapshot, self_pid, known):
mine = {self_pid}
out = {}
changed = True
while changed:
changed = False
for pid, item in snapshot.items():
if pid == self_pid or not living(item): continue
old = known.get(pid)
if old is not None and old == item[3] and pid not in out:
out[pid] = item[3]
mine.add(pid)
changed = True
if item[1] in mine and pid not in out:
out[pid] = item[3]
mine.add(pid)
changed = True
return out
def fresh_kill(pid, start):
current = exact_stat(pid)
if current is None or not living(current) or current[3] != start: return False
try:
os.kill(pid, signal.SIGKILL)
except ProcessLookupError:
return False
return True
def reap(proc, known, exceptions, leader_pid):
self_pid = os.getpid()
leader_reaped = proc is None
pg_absent = False
desc_absent = False
adopted = []
quiet = 0
for turn in range(120):
try:
if proc is not None:
proc.poll()
leader_reaped = proc.returncode is not None
snap = table()
seen = closure(snap, self_pid, known)
known.update(seen)
for pid, start in seen.items():
if pid != leader_pid: fresh_kill(pid, start)
if proc is not None and proc.returncode is None:
token = known.get(leader_pid)
if token is not None: fresh_kill(leader_pid, token)
if proc is not None:
proc.poll()
leader_reaped = proc.returncode is not None
if leader_reaped:
while True:
try: pid, status = os.waitpid(-1, os.WNOHANG)
except ChildProcessError: break
if pid == 0: break
if pid == leader_pid:
exceptions.append('leader adopted wait conflict')
else:
adopted.append([pid, status])
snap = table()
live = closure(snap, self_pid, known)
known.update(live)
desc_absent = not any(pid != leader_pid for pid in live)
pg_absent = not any(living(item) and item[2] == leader_pid for item in snap.values())
if leader_reaped and desc_absent and pg_absent:
quiet += 1
else:
quiet = 0
if quiet >= 3: break
time.sleep(0.025)
except BaseException as error:
exceptions.append(type(error).__name__ + ':' + str(error)[:80])
time.sleep(0.025)
return leader_reaped, desc_absent, pg_absent, adopted
def main():
request = json.loads(sys.stdin.buffer.read())
argv = request['argv']
cwd = request['cwd']
stdin = base64.b64decode(request['stdin'])
input_length = len(stdin)
input_sha256 = hashlib.sha256(stdin).hexdigest()
deadline = request['deadline']
started = time.monotonic()
me = os.getpid()
own = exact_stat(me)
libc = ctypes.CDLL(None, use_errno=True)
if libc.prctl(36, 1, 0, 0, 0) != 0: raise OSError(ctypes.get_errno(), 'subreaper')
proc = None
stdout = b''
stderr = b''
timed_out = False
errors = []
known = {}
leader_pid = 0
leader_start = 0
leader_rc = None
reaped = False
desc_absent = False
pg_absent = False
adopted = []
try:
proc = subprocess.Popen(argv, cwd=cwd, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE,
start_new_session=True)
leader_pid = proc.pid
item = exact_stat(leader_pid)
if item is None: raise RuntimeError('leader vanished before identity capture')
leader_start = item[3]
known[leader_pid] = leader_start
try:
stdout, stderr = proc.communicate(stdin, timeout=deadline)
except subprocess.TimeoutExpired:
timed_out = True
try:
snap = table()
live = closure(snap, me, known)
known.update(live)
for pid, start in live.items(): fresh_kill(pid, start)
stdout, stderr = proc.communicate(timeout=1)
except subprocess.TimeoutExpired:
errors.append('pipe remained open after deadline')
leader_rc = proc.poll()
except BaseException as error:
errors.append(type(error).__name__ + ':' + str(error)[:100])
finally:
reaped, desc_absent, pg_absent, adopted = reap(proc, known, errors, leader_pid)
if proc is not None:
proc.poll()
leader_rc = proc.returncode
elapsed = time.monotonic() - started
wire = dict(argv=list(proc.args) if proc is not None else argv,
cwd=cwd, input_length=input_length, input_sha256=input_sha256,
supervisor_pid=me, supervisor_start=own[3], leader_pid=leader_pid,
leader_start=leader_start, uid=os.getuid(), gid=os.getgid(),
groups=os.getgroups(), elapsed=elapsed, deadline=deadline,
timeout=timed_out, leader_rc=leader_rc,
rc=124 if timed_out else leader_rc, leader_reaped=reaped,
descendants_absent=desc_absent, pg_absent=pg_absent,
adopted=adopted, exceptions=errors,
stdout=base64.b64encode(stdout).decode('ascii'),
stderr=base64.b64encode(stderr).decode('ascii'),
stdout_length=len(stdout), stderr_length=len(stderr),
stdout_sha256=hashlib.sha256(stdout).hexdigest(),
stderr_sha256=hashlib.sha256(stderr).hexdigest())
sys.stdout.write(json.dumps(wire, separators=(',', ':')))
if __name__ == '__main__': main()
'''
PUBLIC = ('Makefile', 'observer.h', 'port.c', 'parser.c', 'report.c', 'main.c',
'cli.c', 'cli.h', 'stats.c', 'stats.h', 'audit.c', 'audit.h',
'text.c', 'text.h', 'trace.c', 'trace.h', 'json.c', 'json.h')
def sha(raw):
return hashlib.sha256(raw).hexdigest()
def valid_number(value, minimum=None):
return type(value) is int and (minimum is None or value >= minimum)
def stage(argv, cwd, data, deadline, label):
payload = json.dumps(dict(argv=argv, cwd=str(cwd), stdin=base64.b64encode(data).decode(), deadline=deadline)).encode()
started = time.monotonic()
wrapper = subprocess.run([sys.executable, '-I', '-c', SUPERVISOR], input=payload,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=deadline + 8)
raw = wrapper.stdout
wrapper_error = ('%d/%s' % (len(wrapper.stderr), sha(wrapper.stderr))
if wrapper.stderr else '0')
print('STAGE %s deadline=%d request_sha=%s wrapper_rc=%d wire_len=%d wire_sha=%s wrapper_err=%s' %
(label, deadline, sha(payload), wrapper.returncode, len(raw), sha(raw),
wrapper_error), file=sys.stderr)
if wrapper.returncode != 0: raise RuntimeError('%s supervisor rc %d' % (label, wrapper.returncode))
try:
wire = json.loads(raw)
if not isinstance(wire, dict): raise ValueError('wire object')
required = ('argv', 'cwd', 'input_length', 'input_sha256',
'supervisor_pid', 'supervisor_start', 'leader_pid', 'leader_start',
'uid', 'gid', 'groups', 'elapsed', 'deadline', 'timeout',
'leader_rc', 'rc', 'leader_reaped', 'descendants_absent',
'pg_absent', 'adopted', 'exceptions', 'stdout', 'stderr',
'stdout_length', 'stderr_length', 'stdout_sha256', 'stderr_sha256')
if any(key not in wire for key in required): raise ValueError('missing wire field')
if (type(wire['argv']) is not list or
any(type(arg) is not str for arg in wire['argv']) or wire['argv'] != argv):
raise ValueError('actual argv mismatch')
if type(wire['cwd']) is not str or wire['cwd'] != str(cwd):
raise ValueError('actual cwd mismatch')
if (not valid_number(wire['input_length'], 0) or
wire['input_length'] != len(data) or
type(wire['input_sha256']) is not str or
wire['input_sha256'] != sha(data)):
raise ValueError('actual stdin mismatch')
if not all(valid_number(wire[key], 1) for key in
('supervisor_pid', 'supervisor_start', 'leader_pid', 'leader_start')):
raise ValueError('invalid pid/start')
if wire['supervisor_pid'] == wire['leader_pid']: raise ValueError('same pid')
if not all(valid_number(wire[key], 0) for key in ('uid', 'gid')): raise ValueError('identity scalar')
if type(wire['groups']) is not list or not all(valid_number(x, 0) for x in wire['groups']):
raise ValueError('groups type')
isolated = os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534'
identity = (65534, 65534, []) if isolated else (os.getuid(), os.getgid(), os.getgroups())
if (wire['uid'], wire['gid'], wire['groups']) != identity: raise ValueError('identity mismatch')
if type(wire['elapsed']) not in (int, float) or not math.isfinite(wire['elapsed']):
raise ValueError('elapsed type')
if not 0 <= wire['elapsed'] <= deadline + 4: raise ValueError('elapsed range')
if (type(wire['deadline']) not in (int, float) or
not math.isfinite(wire['deadline']) or wire['deadline'] != deadline or
type(wire['timeout']) is not bool):
raise ValueError('time fields')
if type(wire['leader_rc']) is not int or type(wire['rc']) is not int: raise ValueError('rc type')
if wire['rc'] != (124 if wire['timeout'] else wire['leader_rc']): raise ValueError('rc conflict')
if any(type(wire[k]) is not bool or not wire[k] for k in
('leader_reaped', 'descendants_absent', 'pg_absent')): raise ValueError('reap/absence')
if type(wire['exceptions']) is not list or wire['exceptions']: raise ValueError('supervisor exception')
if type(wire['adopted']) is not list: raise ValueError('adopted type')
for item in wire['adopted']:
if type(item) is not list or len(item) != 2 or not all(valid_number(x, 0) for x in item):
raise ValueError('adopted item')
stdout = base64.b64decode(wire['stdout'], validate=True)
stderr = base64.b64decode(wire['stderr'], validate=True)
for key, content in (('stdout', stdout), ('stderr', stderr)):
if not valid_number(wire[key+'_length'], 0) or wire[key+'_length'] != len(content):
raise ValueError('io length')
if wire[key+'_sha256'] != sha(content): raise ValueError('io sha')
print('META %s argv=%s cwd=%s stdin=%d/%s sup=%d/%d leader=%d/%d uid=%d gid=%d groups=%s rc=%d actual=%d timeout=%s elapsed=%.4f reaped=%s pg=%s desc=%s adopted=%s exceptions=%s out=%d/%s err=%d/%s' %
(label, wire['argv'], wire['cwd'], wire['input_length'], wire['input_sha256'],
wire['supervisor_pid'], wire['supervisor_start'], wire['leader_pid'],
wire['leader_start'], wire['uid'], wire['gid'], wire['groups'], wire['rc'],
wire['leader_rc'], wire['timeout'], wire['elapsed'], wire['leader_reaped'],
wire['pg_absent'], wire['descendants_absent'], wire['adopted'],
wire['exceptions'], len(stdout), sha(stdout),
len(stderr), sha(stderr)), file=sys.stderr)
return wire, stdout, stderr
except (ValueError, TypeError, KeyError, UnicodeError) as error:
raise RuntimeError('%s wire invalid: %s' % (label, error)) from error
def cases():
rng = random.Random(173058)
jobs = []
def add(mask, budget, hold, frames):
jobs.append((rng.randrange(100000, 2000000000), mask, budget, hold, frames))
add(1, 1, 1, [1])
add(1, 1, 1, [0])
add(1, 5, 1, [0, 0, 1, 0, 0])
add(3, 5, 2, [1, 3, 0, 3, 3])
add(3, 5, 2, [3, 0, 3, 0, 3])
add(4, 6, 3, [4, 4, 0, 4, 4, 4])
add(9, 6, 2, [8, 9, 1, 9, 9, 0])
for _ in range(28):
budget = rng.randrange(1, 40)
hold = rng.randrange(1, budget + 1)
mask = rng.randrange(1, 65536)
frames = [rng.randrange(65536) for _ in range(budget)]
if rng.randrange(2):
for at in range(rng.randrange(budget), budget, rng.randrange(1, 4)):
frames[at] |= mask
add(mask, budget, hold, frames)
return jobs
def input_and_expected(jobs):
lines = ['0' * 5000 + str(len(jobs))]
expected = []
for index, (ident, mask, budget, hold, frames) in enumerate(jobs):
if index == 0:
lines.append('%s %d %d %d %d' % ('0' * 5000 + str(ident), mask, budget, hold, budget))
lines.extend(('0' * 5000 + str(value) if at == 0 else str(value))
for at, value in enumerate(frames))
else:
lines.append('%d %d %d %d %d' % (ident, mask, budget, hold, budget))
lines.extend(map(str, frames))
streak = 0
outcome = 0
for step, value in enumerate(frames):
if value & mask == mask: streak += 1
else: streak = 0
if streak == hold:
outcome = 1
break
expected.append('%d %d %d %d %d %d' %
(ident, outcome, step + 1, step, value, streak))
return ('\n'.join(lines) + '\n').encode(), ('\n'.join(expected) + '\n').encode()
def main(candidate):
scratch = None
result_ok = False
problem = 'not started'
cleanup_ok = False
absence_ok = False
exceptions = []
try:
scratch = Path(tempfile.mkdtemp(prefix='ready-observer-'))
work = scratch / 'work'
work.mkdir()
if os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534':
os.chown(work, 65534, 65534)
archive = Path(__file__).parent / 'public.tar'
with tarfile.open(archive, 'r') as package:
for member in package.getmembers():
if member.name not in PUBLIC or not member.isfile(): raise RuntimeError('public bundle member')
content = package.extractfile(member).read()
(work / member.name).write_bytes(content)
(work / 'observer.c').write_bytes((candidate / 'observer.c').read_bytes())
jobs = cases()
stdin, expected = input_and_expected(jobs)
invalid = b'1\n7 1 1 1 1\n0\nEXTRA\n'
build_argv = ['gcc', '-std=c11', '-O2', '-Wall', '-Wextra', '-Werror', '-pedantic',
'main.c', 'parser.c', 'port.c', 'report.c', 'observer.c',
'cli.c', 'stats.c', 'audit.c', 'text.c', 'trace.c', 'json.c',
'-o', 'ready_replay']
built, _, build_err = stage(build_argv, work, b'', 18, 'compile')
if built['timeout'] or built['rc'] != 0:
problem = ('compile rc=%d timeout=%s stderr=%s' %
(built['rc'], built['timeout'], build_err[-180:].decode('utf-8', 'replace')))
else:
ran, actual, _ = stage([str(work / 'ready_replay')], work, stdin, 18, 'candidate')
if ran['timeout'] or ran['rc'] != 0:
problem = 'candidate rc=%d timeout=%s' % (ran['rc'], ran['timeout'])
elif actual != expected:
a = actual.splitlines()
e = expected.splitlines()
index = next((i for i in range(max(len(a), len(e)))
if (a[i] if i < len(a) else None) != (e[i] if i < len(e) else None)), 0)
problem = ('observation %d got=%r expected=%r' %
(index, a[index] if index < len(a) else None,
e[index] if index < len(e) else None))
else:
rejected, _, _ = stage([str(work / 'ready_replay')], work,
invalid, 18, 'invalid_input')
if rejected['timeout'] or rejected['rc'] != 2:
problem = 'invalid input rc=%d timeout=%s' % (rejected['rc'], rejected['timeout'])
else:
result_ok = True
problem = 'all %d observations match' % len(jobs)
except BaseException as error:
exceptions.append('main:' + type(error).__name__)
problem = 'exception %s: %s' % (type(error).__name__, str(error)[:180])
finally:
if scratch is not None:
try:
cleaned, _, _ = stage(['rm', '-rf', '--', str(scratch)], scratch.parent, b'', 8, 'cleanup')
cleanup_ok = cleaned['rc'] == 0 and not cleaned['timeout']
except BaseException as error:
exceptions.append('cleanup:' + type(error).__name__)
print('cleanup exception %s' % str(error)[:120], file=sys.stderr)
try:
script = 'import os,sys\ntry: os.lstat(sys.argv[1])\nexcept FileNotFoundError: sys.exit(0)\nsys.exit(1)\n'
absent, _, _ = stage([sys.executable, '-I', '-c', script, str(scratch)],
scratch.parent, b'', 8, 'absence')
absence_ok = absent['rc'] == 0 and not absent['timeout']
except BaseException as error:
exceptions.append('absence:' + type(error).__name__)
print('absence exception %s' % str(error)[:120], file=sys.stderr)
if result_ok and cleanup_ok and absence_ok and not exceptions:
print('PASS')
return 0
print('FAIL ' + problem + ' cleanup=%s absence=%s exceptions=%s' % (cleanup_ok, absence_ok, exceptions))
return 1
if __name__ == '__main__':
raise SystemExit(main(Path(sys.argv[1])))