amos 9d633e9e67 Batch 15 (20261023-C-hwe-scale-15): 320 HWE-derived repair tasks
One user-space repair task per eligible HWE debug-diag / compile-deploy base
ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content
filter. Each ships a public spec with a check-to-basis table, at least two
defects with single-fix fixtures, a symptom-masking fixture and an alternative
correct implementation. Claude re-checked all 320: structure, the HWE quality
items and trace6 Docker calibration receipts against current digests
(320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120
ad-hoc family names; the next batch maps them to the canonical list.
The batch's 1,256-line progress log stays in its factory report and the
36-local branch batch15-hwe.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 01:55:56 +08:00

535 lines
23 KiB
Python

"""Independent publication oracle and isolated process supervisor."""
import base64
import hashlib
import json
import math
import os
from pathlib import Path
import random
import shutil
import subprocess
import sys
import tempfile
import time
import zlib
GRADER = Path(__file__).resolve().parent
COMPILER_DEADLINE = 25.0
CANDIDATE_DEADLINE = 25.0
CLEANUP_DEADLINE = 12.0
ABSENCE_DEADLINE = 10.0
REAP_MARGIN = 2.0
# This text runs as the candidate identity, and carries no test expectations.
SUPERVISOR = r'''
import base64, ctypes, hashlib, json, math, os, pathlib, select, signal, subprocess, sys, time
def proc(pid):
try:
raw = pathlib.Path('/proc/%d/stat' % pid).read_text()
except (FileNotFoundError, ProcessLookupError):
return None
right = raw.rfind(')')
if right < 0:
raise ValueError('malformed proc stat')
tail = raw[right + 2:].split()
return {'pid':pid, 'state':tail[0], 'ppid':int(tail[1]), 'pgid':int(tail[2]), 'start':int(tail[19])}
def table():
result = {}
for name in os.listdir('/proc'):
if name.isdecimal():
value = proc(int(name))
if value is not None:
result[value['pid']] = value
return result
def closure(owner):
snapshot = table()
root = snapshot.get(owner['pid'])
if root is None or root['start'] != owner['start'] or root['state'] in 'ZX':
return []
found = {owner['pid']}
changed = True
while changed:
changed = False
for value in snapshot.values():
if value['ppid'] in found and value['pid'] not in found and value['state'] not in 'ZX':
found.add(value['pid'])
changed = True
return [snapshot[pid] for pid in found if pid != owner['pid']]
def fresh_signal(token, sig, errors):
try:
now = proc(token['pid'])
if now is not None and now['start'] == token['start'] and now['state'] not in 'ZX':
os.kill(token['pid'], sig)
except (FileNotFoundError, ProcessLookupError):
pass
except Exception as exc:
errors.append('signal:' + type(exc).__name__ + ':' + str(exc))
def reap_adopted(errors):
while True:
try:
child, _ = os.waitpid(-1, os.WNOHANG)
except ChildProcessError:
return
except Exception as exc:
errors.append('waitpid:' + type(exc).__name__ + ':' + str(exc))
return
if child == 0:
return
def owned_identity(pid):
path = pathlib.Path('/proc/%d/status' % pid)
data = path.read_text()
fields = {}
for line in data.splitlines():
if line.startswith(('Uid:', 'Gid:', 'Groups:')):
name, values = line.split(':', 1)
fields[name] = [int(piece) for piece in values.split()]
return {'uid':fields['Uid'][0], 'gid':fields['Gid'][0], 'groups':fields['Groups']}
def run(payload):
started = time.monotonic()
phase = payload['phase']
deadline = float(payload['deadline'])
errors = []
libc = ctypes.CDLL(None, use_errno=True)
if libc.prctl(36, 1, 0, 0, 0) != 0:
raise OSError(ctypes.get_errno(), 'subreaper registration failed')
mine = proc(os.getpid())
owner = {'pid':os.getpid(), 'start':mine['start']}
leader = None
leader_token = None
leader_identity = None
leader_rc = None
leader_reaped = False
timed_out = False
stdout = b''
stderr = b''
child_input = base64.b64decode(payload.get('input_b64', ''))
known = {}
try:
leader = subprocess.Popen(payload['argv'], cwd=payload['cwd'],
stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, start_new_session=True)
found = proc(leader.pid)
if found is None:
raise RuntimeError('leader identity disappeared before capture')
leader_token = {'pid':leader.pid, 'start':found['start']}
leader_identity = owned_identity(leader.pid)
remaining = max(0.0, deadline - (time.monotonic() - started))
try:
stdout, stderr = leader.communicate(input=child_input, timeout=remaining)
leader_rc = leader.returncode
leader_reaped = True
except subprocess.TimeoutExpired:
timed_out = True
# A quick first signal closes pipes before the bounded reap loop.
try:
targets = closure(owner)
known.update((item['pid'], item) for item in targets)
except Exception as exc:
errors.append('timeout-discover:' + type(exc).__name__ + ':' + str(exc))
targets = list(known.values())
if leader_token is not None:
targets.append(leader_token)
for token in targets:
fresh_signal(token, signal.SIGKILL, errors)
try:
stdout, stderr = leader.communicate(timeout=REAP_SECONDS)
leader_rc = leader.returncode
leader_reaped = True
except subprocess.TimeoutExpired:
errors.append('leader pipe still open after timeout kill')
except Exception as exc:
errors.append('run:' + type(exc).__name__ + ':' + str(exc))
finally:
# This loop also runs on Popen, communicate, and metadata exceptions.
stop = time.monotonic() + REAP_SECONDS
while True:
if leader is not None and not leader_reaped:
try:
leader_rc = leader.poll()
if leader_rc is not None:
leader.wait(timeout=0)
leader_reaped = True
except Exception as exc:
errors.append('leader-wait:' + type(exc).__name__ + ':' + str(exc))
if leader_reaped:
reap_adopted(errors)
try:
descendants = closure(owner)
known.update((item['pid'], item) for item in descendants)
except Exception as exc:
errors.append('reap-discover:' + type(exc).__name__ + ':' + str(exc))
descendants = list(known.values())
if leader_token is not None and not leader_reaped:
descendants.append(leader_token)
if not descendants and (leader is None or leader_reaped):
break
for token in descendants:
fresh_signal(token, signal.SIGKILL, errors)
if time.monotonic() >= stop:
errors.append('reap deadline with live descendants or leader')
break
time.sleep(0.02)
if leader is not None and not leader_reaped:
try:
fresh_signal(leader_token, signal.SIGKILL, errors)
leader_rc = leader.wait(timeout=0.2)
leader_reaped = True
except Exception as exc:
errors.append('final-wait:' + type(exc).__name__ + ':' + str(exc))
if leader_reaped:
reap_adopted(errors)
try:
absent = not closure(owner)
except Exception as exc:
errors.append('absence:' + type(exc).__name__ + ':' + str(exc))
absent = False
elapsed = time.monotonic() - started
self_identity = {'uid':os.getuid(), 'gid':os.getgid(), 'groups':os.getgroups()}
return {'phase':phase, 'supervisor_pid':owner['pid'], 'supervisor_start':owner['start'],
'supervisor_identity':self_identity, 'leader_pid':leader_token['pid'] if leader_token else None,
'leader_start':leader_token['start'] if leader_token else None,
'leader_identity':leader_identity, 'leader_rc':leader_rc, 'rc':leader_rc,
'reaped':leader_reaped, 'timeout':timed_out, 'absence':absent,
'exceptions':errors, 'elapsed':elapsed, 'deadline':deadline,
'stdout_len':len(stdout), 'stdout_sha256':hashlib.sha256(stdout).hexdigest(),
'stderr_len':len(stderr), 'stderr_sha256':hashlib.sha256(stderr).hexdigest(),
'stdout_b64':base64.b64encode(stdout).decode('ascii'),
'stderr_b64':base64.b64encode(stderr).decode('ascii')}
REAP_SECONDS = 2.0
try:
answer = run(json.loads(sys.argv[1]))
print(json.dumps(answer, sort_keys=True, separators=(',', ':')))
except Exception as exc:
print(json.dumps({'wire_error':type(exc).__name__ + ':' + str(exc)}))
raise
'''
def identity_ok(value, expected):
if not isinstance(value, dict) or set(value) != {'uid', 'gid', 'groups'}:
return False
if type(value['uid']) is not int or type(value['gid']) is not int:
return False
if type(value['groups']) is not list or any(type(item) is not int for item in value['groups']):
return False
return value['uid'] == expected['uid'] and value['gid'] == expected['gid'] and value['groups'] == expected['groups']
def strict_meta(wire, phase, deadline, expected_identity, wrapper_rc, wrapper_stderr):
keys = {'phase','supervisor_pid','supervisor_start','supervisor_identity',
'leader_pid','leader_start','leader_identity','leader_rc','rc','reaped',
'timeout','absence','exceptions','elapsed','deadline','stdout_len',
'stdout_sha256','stderr_len','stderr_sha256','stdout_b64','stderr_b64'}
if wrapper_rc != 0 or wrapper_stderr:
raise ValueError('supervisor wrapper failed')
if not isinstance(wire, dict) or set(wire) != keys:
raise ValueError('supervisor metadata keys missing or extra')
if wire['phase'] != phase:
raise ValueError('supervisor phase mismatch')
for name in ('supervisor_pid','supervisor_start','leader_pid','leader_start'):
if type(wire[name]) is not int or wire[name] <= 0:
raise ValueError('invalid ' + name)
if wire['supervisor_pid'] == wire['leader_pid']:
raise ValueError('leader and supervisor PID collide')
if not identity_ok(wire['supervisor_identity'], expected_identity) or not identity_ok(wire['leader_identity'], expected_identity):
raise ValueError('execution identity mismatch')
for name in ('reaped','timeout','absence'):
if type(wire[name]) is not bool:
raise ValueError('invalid ' + name)
if type(wire['leader_rc']) is not int or type(wire['rc']) is not int or wire['rc'] != wire['leader_rc']:
raise ValueError('leader rc mismatch')
if type(wire['elapsed']) not in (float,int) or not math.isfinite(wire['elapsed']) or not 0 <= wire['elapsed'] <= deadline + REAP_MARGIN + 1.0:
raise ValueError('invalid elapsed')
if type(wire['deadline']) not in (float,int) or wire['deadline'] != deadline:
raise ValueError('deadline mismatch')
if not isinstance(wire['exceptions'], list) or any(type(item) is not str for item in wire['exceptions']):
raise ValueError('invalid exception list')
raw = {}
for stream in ('stdout','stderr'):
length = wire[stream + '_len']
digest = wire[stream + '_sha256']
if type(length) is not int or length < 0 or not isinstance(digest, str) or len(digest) != 64:
raise ValueError('invalid IO metadata')
encoded = wire[stream + '_b64']
if type(encoded) is not str:
raise ValueError('invalid IO wire')
data = base64.b64decode(encoded, validate=True)
if len(data) != length or hashlib.sha256(data).hexdigest() != digest:
raise ValueError('IO digest mismatch')
raw[stream] = data
return raw
def supervise(phase, argv, cwd, deadline, identity, input_bytes=b''):
payload = {'phase':phase,'argv':argv,'cwd':str(cwd),'deadline':deadline,
'input_b64':base64.b64encode(input_bytes).decode('ascii')}
completed = subprocess.run([sys.executable, '-I', '-c', SUPERVISOR, json.dumps(payload)],
capture_output=True, timeout=deadline + REAP_MARGIN + 4.0)
# Preserve actual wrapper facts before trusting its wire. This short summary is safe for Bos's tail.
wrapper = {'phase':phase,'wrapper_rc':completed.returncode,
'wire_len':len(completed.stdout),
'wire_sha256':hashlib.sha256(completed.stdout).hexdigest(),
'wrapper_stderr_len':len(completed.stderr),
'wrapper_stderr_sha256':hashlib.sha256(completed.stderr).hexdigest(),
'argv':argv}
try:
wire = json.loads(completed.stdout)
raw = strict_meta(wire, phase, deadline, identity, completed.returncode, completed.stderr)
except Exception as exc:
return {'ok':False,'wrapper':wrapper,'failure':'wire:'+type(exc).__name__+':'+str(exc)}
compact = {key:value for key,value in wire.items() if not key.endswith('_b64')}
success = (wire['rc'] == 0 and not wire['timeout'] and wire['reaped'] and
wire['absence'] and not wire['exceptions'])
return {'ok':success,'wrapper':wrapper,'meta':compact,'raw':raw,
'failure':None if success else 'stage:'+phase+':rc='+str(wire['rc'])+
':timeout='+str(wire['timeout'])+':absence='+str(wire['absence'])+
':exceptions='+str(wire['exceptions'][:2])}
def fnv(data):
value = 2166136261
for byte in data:
value = ((value ^ byte) * 16777619) & 0xffffffff
return value
def hx(data):
return bytes(data).hex() or '-'
def cases():
rng = random.Random(0xCC006E1)
events = []
def add(*args):
events.append(tuple(args))
a = [rng.randrange(256) for _ in range(37)]
b = [rng.randrange(256) for _ in range(11)]
add('O',0,rng.randrange(0,1<<30))
add('A',0,len(a),*a)
add('S',0)
add('P',0,3,3,*b[:3])
add('I',0,5,4,*b[3:7])
add('R',0,1,17)
add('Y',0,0,14,5)
add('S',0)
add('U',0,rng.randrange(0,1<<30),len(b),*b)
add('S',0)
add('F',0,2,4,194)
add('X',0,0,3,23)
add('E',0,5,2)
add('T',0,6)
add('S',0)
add('C',0,1,81)
add('K',0,1,1,2,3)
add('S',1)
add('D',0)
add('O',0,0)
add('S',0)
add('U',1,0,0)
add('O',4,123)
add('A',2,1,5)
add('C',1,1,8)
add('I',1,999,1,4)
add('T',1,999)
add('Y',1,0,0,3)
add('K',1,0,2,0,1)
add('S',4)
add('D',4)
add('U',4,7,1,8)
add('A',0,256,*[rng.randrange(256) for _ in range(256)])
add('S',0)
add('U',0,93,17,*[rng.randrange(256) for _ in range(17)])
for bank in range(2,4):
add('O',bank,rng.randrange(1000))
count = rng.randrange(1,65)
add('A',bank,count,*[rng.randrange(256) for _ in range(count)])
add('S',bank)
add('U',bank,rng.randrange(1000),count//2,*[rng.randrange(256) for _ in range(count//2)])
for _ in range(60):
bank = rng.randrange(4)
op = rng.choice(('A','P','I','E','F','X','R','Y','T','S','U','Z'))
length = rng.randrange(0,23)
at = rng.randrange(0,24)
if op == 'A': add(op,bank,length,*[rng.randrange(256) for _ in range(length)])
elif op in ('P','I'): add(op,bank,at,length,*[rng.randrange(256) for _ in range(length)])
elif op in ('E','R'): add(op,bank,at,length)
elif op in ('F','X','Y'): add(op,bank,at,length,rng.randrange(256))
elif op == 'T': add(op,bank,at)
elif op == 'U': add(op,bank,rng.randrange(1000),length,*[rng.randrange(256) for _ in range(length)])
else: add(op,bank)
return events
def expected(events):
banks = [None] * 4
visible = [[0] * 256 for _ in range(4)]
output = []
for event in events:
op,*p = event
rc = 0
valid_bank = lambda b: 0 <= b < 4
if op == 'O':
bank,rev = p
if not valid_bank(bank): rc = -1
elif banks[bank] is not None: rc = -2
else: banks[bank] = {'data':[], 'rev':rev}
elif op in ('D','Z','S'):
bank = p[0]
if not valid_bank(bank): rc = -1
elif banks[bank] is None: rc = -2
elif op == 'D': banks[bank] = None
elif op == 'Z': banks[bank]['data'] = []
else:
data = banks[bank]['data']
visible[bank][:len(data)] = data
output.append('E %d %d %d %d %s' % (bank,banks[bank]['rev'],len(data),fnv(data),hx(visible[bank][:len(data)])))
elif op == 'C':
source,target,rev = p
if not valid_bank(source) or not valid_bank(target): rc = -1
elif source == target or banks[source] is None or banks[target] is not None: rc = -2
else: banks[target] = {'data':banks[source]['data'][:], 'rev':rev}
elif op == 'K':
source,start,target,at,count = p
if not valid_bank(source) or not valid_bank(target): rc = -1
elif banks[source] is None or banks[target] is None: rc = -2
elif start+count > len(banks[source]['data']) or at+count > len(banks[target]['data']): rc = -1
else: banks[target]['data'][at:at+count] = banks[source]['data'][start:start+count]
elif op == 'U':
bank,rev,count,*data = p
if not valid_bank(bank) or count > 256: rc = -1
elif banks[bank] is None: rc = -2
else:
banks[bank] = {'data':data[:], 'rev':rev}
visible[bank][:count] = data
output.append('E %d %d %d %d %s' % (bank,rev,count,fnv(data),hx(visible[bank][:count])))
else:
bank = p[0]
if not valid_bank(bank): rc = -1
elif banks[bank] is None: rc = -2
else:
data = banks[bank]['data']
if op == 'A':
count = p[1]
if len(data)+count > 256: rc = -1
else: data.extend(p[2:])
elif op == 'I':
at,count = p[1:3]
if at > len(data) or len(data)+count > 256: rc = -1
else: data[at:at] = p[3:]
elif op in ('P','E','F','X','R','Y'):
at,count = p[1:3]
if at+count > len(data): rc = -1
elif op == 'P': data[at:at+count] = p[3:]
elif op == 'E': del data[at:at+count]
elif op == 'F': data[at:at+count] = [p[3]]*count
elif op == 'X': data[at:at+count] = [x ^ p[3] for x in data[at:at+count]]
elif op == 'R': data[at:at+count] = data[at:at+count][::-1]
elif count == 0:
if p[3] != 0: rc = -1
else:
shift = p[3] % count
data[at:at+count] = data[at:at+count][-shift:] + data[at:at+count][:-shift] if shift else data[at:at+count]
elif op == 'T':
length = p[1]
if length > len(data): rc = -1
else: del data[length:]
else: raise AssertionError(op)
state = 'S %d' % rc
for bank in banks:
content = [] if bank is None else bank['data']
state += ' %d:%s' % (len(content),hx(content))
output.append(state)
return output
def main():
identity = {'uid':os.getuid(), 'gid':os.getgid(), 'groups':os.getgroups()}
if os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534':
identity = {'uid':65534,'gid':65534,'groups':[]}
records = []
failure = None
scratch = None
try:
scratch = Path(tempfile.mkdtemp(prefix='dev-flush-'))
work = scratch / 'work'
work.mkdir(mode=0o700)
# Only this new, still-owned child directory is changed by the check.
os.chown(work, identity['uid'], identity['gid'])
for name in ('store.c','store.h'):
shutil.copyfile(Path(sys.argv[1]) / name, work / name)
os.chown(work / name, identity['uid'], identity['gid'])
shutil.copyfile(GRADER / 'harness.c', work / 'harness.c')
os.chown(work / 'harness.c', identity['uid'], identity['gid'])
events = cases()
expected_lines = expected(events)
candidate_input = ''.join(' '.join(map(str,event))+'\n' for event in events).encode('ascii')
executable = work / 'observer'
compiler = supervise('compiler', ['gcc','-std=c11','-O2','-Wall','-Wextra','-Werror','-pedantic',
'-I',str(work),str(work/'harness.c'),str(work/'store.c'),'-o',str(executable)],
work, COMPILER_DEADLINE, identity)
records.append(compiler)
if not compiler['ok']:
failure = 'compiler: ' + str(compiler['failure'])
else:
candidate = supervise('candidate', [str(executable)], work, CANDIDATE_DEADLINE, identity, candidate_input)
records.append(candidate)
if not candidate['ok']:
failure = 'candidate: ' + str(candidate['failure'])
else:
observed = candidate['raw']['stdout'].decode('utf-8', 'replace').splitlines()
if observed != expected_lines:
at = next((i for i in range(max(len(observed),len(expected_lines)))
if (observed[i] if i < len(observed) else None) !=
(expected_lines[i] if i < len(expected_lines) else None)), 0)
failure = 'behavior line %d: got %r expected %r' % (at+1,
(observed[at] if at < len(observed) else '<missing>')[:100],
(expected_lines[at] if at < len(expected_lines) else '<extra>')[:100])
except Exception as exc:
failure = 'check: ' + type(exc).__name__ + ': ' + str(exc)
finally:
if scratch is not None:
# Candidate-owned cleanup and independent lstat absence are separate deadline stages.
cleanup_code = 'import shutil,sys; shutil.rmtree(sys.argv[1])'
try:
cleanup = supervise('cleanup',[sys.executable,'-I','-c',cleanup_code,str(scratch)],
'/',CLEANUP_DEADLINE,identity)
records.append(cleanup)
if not cleanup['ok']:
failure = (failure or '') + ' cleanup: ' + str(cleanup['failure'])
except Exception as exc:
failure = (failure or '') + ' cleanup exception: ' + type(exc).__name__ + ': ' + str(exc)
absence_code = ('import os,sys\ntry: os.lstat(sys.argv[1])\n'
'except FileNotFoundError: raise SystemExit(0)\n'
'except Exception: raise SystemExit(2)\n'
'raise SystemExit(1)\n')
try:
absence = supervise('absence',[sys.executable,'-I','-c',absence_code,str(scratch)],
'/',ABSENCE_DEADLINE,identity)
records.append(absence)
if not absence['ok']:
failure = (failure or '') + ' absence: ' + str(absence['failure'])
except Exception as exc:
failure = (failure or '') + ' absence exception: ' + type(exc).__name__ + ': ' + str(exc)
# Keep complete structured phase metadata inside Bos's 4000-character stderr tail.
summary = []
for record in records:
summary.append({key:value for key,value in record.items() if key != 'raw'})
packed = base64.b64encode(zlib.compress(json.dumps(summary,sort_keys=True,separators=(',',':')).encode(),9)).decode()
print('META-Z:'+packed,file=sys.stderr)
if failure:
print('FAIL '+failure[:300])
return 1
print('PASS')
return 0
if __name__ == '__main__':
raise SystemExit(main())