amos 9d633e9e67 Batch 15 (20261023-C-hwe-scale-15): 320 HWE-derived repair tasks
One user-space repair task per eligible HWE debug-diag / compile-deploy base
ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content
filter. Each ships a public spec with a check-to-basis table, at least two
defects with single-fix fixtures, a symptom-masking fixture and an alternative
correct implementation. Claude re-checked all 320: structure, the HWE quality
items and trace6 Docker calibration receipts against current digests
(320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120
ad-hoc family names; the next batch maps them to the canonical list.
The batch's 1,256-line progress log stays in its factory report and the
36-local branch batch15-hwe.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 01:55:56 +08:00

255 lines
19 KiB
Python

"""Independent byte-record oracle and candidate-identity process supervision."""
import base64
import hashlib
import json
import math
import os
from pathlib import Path
import random
import re
import subprocess
import sys
import tempfile
import uuid
import zlib
SUPERVISOR = "import base64,ctypes,hashlib,json,os,selectors,signal,subprocess,sys,time\npayload=json.loads(sys.stdin.buffer.read())\nstart=time.monotonic()\nerrors=[]\ntracked={}\nsignals=[]\nadopted=[]\nleader=None\nleader_start=None\nleader_rc=None\ntimed_out=False\nout=bytearray();err=bytearray()\nuid=os.getuid();gid=os.getgid();groups=os.getgroups()\n\ndef stat_pid(pid):\n try:\n data=open('/proc/%d/stat'%pid,'rb').read()\n tail=data[data.rfind(b')')+2:].split()\n return (int(pid),int(tail[1]),int(tail[2]),int(tail[19]),tail[0].decode('ascii'))\n except (FileNotFoundError,ProcessLookupError):\n return None\n except BaseException as exc:\n errors.append(['stat',int(pid),type(exc).__name__,str(exc)])\n return None\n\ndef closure(table,root,known):\n roots={root}\n for pid,token in known.items():\n row=table.get(pid)\n if row and row[3]==token and row[4] not in ('Z','X'):\n roots.add(pid)\n found=set()\n changed=True\n while changed:\n changed=False\n for pid,row in table.items():\n if pid!=root and row[1] in roots and pid not in found:\n found.add(pid); roots.add(pid); changed=True\n found.update(pid for pid in roots if pid!=root)\n return found\n\ndef discover():\n table={}\n try:\n names=os.listdir('/proc')\n except BaseException as exc:\n errors.append(['proc',type(exc).__name__,str(exc)]);return {}\n for name in names:\n if name.isascii() and name.isdigit():\n row=stat_pid(int(name))\n if row:table[row[0]]=row\n found=closure(table,os.getpid(),tracked)\n for pid in found:tracked[pid]=table[pid][3]\n return {pid:table[pid] for pid in found}\n\ndef send(pid,token,sig):\n fresh=stat_pid(pid)\n if fresh is None or fresh[3]!=token or fresh[4] in ('Z','X'):\n return\n try:\n os.kill(pid,sig)\n signals.append([pid,token,sig])\n except ProcessLookupError:\n return\n except BaseException as exc:\n errors.append(['signal',pid,token,type(exc).__name__,str(exc)])\n\ndef reap():\n # Never waitpid(-1) until Popen has obtained its own leader status.\n if leader is not None and leader.poll() is None:return\n while True:\n try:\n pid,status=os.waitpid(-1,os.WNOHANG)\n if not pid:break\n adopted.append([pid,status])\n except ChildProcessError:break\n except BaseException as exc:\n errors.append(['wait',type(exc).__name__,str(exc)]);break\n\ndef drain(selector):\n for key,events in selector.select(0.02):\n file=key.fileobj\n if key.data=='in':\n try:\n sent=os.write(file.fileno(),pending[0][:65536])\n pending[0]=pending[0][sent:]\n except BrokenPipeError:pending[0]=b''\n if not pending[0]:selector.unregister(file);file.close()\n else:\n try:data=os.read(file.fileno(),65536)\n except BlockingIOError:continue\n if not data:selector.unregister(file);file.close()\n elif key.data=='out':out.extend(data)\n else:err.extend(data)\n\ntry:\n if ctypes.CDLL(None,use_errno=True).prctl(36,1,0,0,0)!=0:\n raise OSError(ctypes.get_errno(),'subreaper')\n command=payload['argv']\n raw=base64.b64decode(payload['input'])\n leader=subprocess.Popen(command,cwd=payload['cwd'],stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,stderr=subprocess.PIPE,start_new_session=True)\n row=stat_pid(leader.pid)\n if row is None:raise RuntimeError('leader identity unavailable')\n leader_start=row[3];tracked[leader.pid]=leader_start\n pending=[raw]\n selector=selectors.DefaultSelector()\n for stream,key in [(leader.stdout,'out'),(leader.stderr,'err')]:\n os.set_blocking(stream.fileno(),False);selector.register(stream,selectors.EVENT_READ,key)\n if raw:\n os.set_blocking(leader.stdin.fileno(),False)\n selector.register(leader.stdin,selectors.EVENT_WRITE,'in')\n else:leader.stdin.close()\n deadline=start+payload['deadline']\n while True:\n leader_rc=leader.poll()\n discover()\n drain(selector)\n if leader_rc is not None and not selector.get_map():break\n if time.monotonic()>=deadline:\n timed_out=True;break\n # Handle both same-PG children and reparented, setsid descendants.\n for sig,window in [(signal.SIGTERM,0.2),(signal.SIGKILL,1.6)]:\n limit=time.monotonic()+window\n while True:\n leader_rc=leader.poll()\n rows=discover()\n for pid,row in rows.items():send(pid,row[3],sig)\n reap()\n drain(selector)\n live=discover()\n if not live and leader.poll() is not None and not selector.get_map():break\n if time.monotonic()>=limit:break\n leader_rc=leader.poll()\n if leader_rc is not None:leader.wait();reap()\n remaining=discover()\n absent=not remaining\n # PG existence alone can include foreign reused groups; record token-bound members.\n pg_absent=not any(row[2]==leader.pid for row in remaining.values())\n selector.close()\nexcept BaseException as exc:\n errors.append(['supervisor',type(exc).__name__,str(exc)])\n if leader is not None:\n for pid,row in discover().items():send(pid,row[3],signal.SIGKILL)\n until=time.monotonic()+1.8\n while leader.poll() is None and time.monotonic()<until:time.sleep(0.01)\n leader_rc=leader.poll();reap()\n remaining=discover();absent=not remaining\n pg_absent=not any(leader is not None and row[2]==leader.pid for row in remaining.values())\nmeta={'nonce':payload['nonce'],'phase':payload['phase'],'argv':payload['argv'],'cwd':payload['cwd'],\n 'deadline':payload['deadline'],'uid':uid,'gid':gid,'groups':groups,\n 'supervisor_pid':os.getpid(),'leader_pid':None if leader is None else leader.pid,\n 'leader_start':leader_start,'leader_rc':leader_rc,'leader_reaped':leader is not None and leader_rc is not None,\n 'timeout':timed_out,'controlled_rc':124 if timed_out else leader_rc,\n 'descendants_absent':absent,'pg_absent':pg_absent,'remaining':list(remaining.values()),\n 'exceptions':errors,'tracked':list(tracked.items()),'signals':signals,'adopted':adopted,\n 'elapsed':time.monotonic()-start,'stdin_len':len(raw) if 'raw' in globals() else 0,\n 'stdin_sha':hashlib.sha256(raw if 'raw' in globals() else b'').hexdigest(),\n 'stdout_len':len(out),'stdout_sha':hashlib.sha256(out).hexdigest(),\n 'stderr_len':len(err),'stderr_sha':hashlib.sha256(err).hexdigest(),\n 'stdout':base64.b64encode(out).decode(),'stderr':base64.b64encode(err).decode()}\nsys.stdout.buffer.write(json.dumps(meta,separators=(',',':')).encode())\n"
BUILD_SECONDS = 20
RUN_SECONDS = 10
CLEAN_SECONDS = 5
ABSENCE_SECONDS = 5
CLEAN = "import shutil,sys; shutil.rmtree(sys.argv[1])"
ABSENCE = "import os,sys\ntry: os.lstat(sys.argv[1])\nexcept FileNotFoundError: raise SystemExit(0)\nraise SystemExit(1)"
STAGES = []
EXCEPTIONS = []
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def positive(value):
return type(value) is int and value > 0
def phase(name, argv, cwd, raw, deadline):
nonce = uuid.uuid4().hex
payload = dict(phase=name, argv=argv, cwd=cwd, input=base64.b64encode(raw).decode(),
deadline=deadline, nonce=nonce)
record = dict(phase=name, nonce=nonce, argv=argv, cwd=cwd, deadline=deadline)
STAGES.append(record)
wrapper = subprocess.run([sys.executable, '-I', '-c', SUPERVISOR],
input=json.dumps(payload).encode(), capture_output=True,
timeout=deadline + 8)
# Record actual wrapper facts before parsing potentially missing/malformed wire.
wire = wrapper.stdout
record.update(wrapper_rc=wrapper.returncode, wire_len=len(wire), wire_sha=digest(wire),
wrapper_stderr_len=len(wrapper.stderr), wrapper_stderr_sha=digest(wrapper.stderr))
meta = json.loads(wire)
record['native'] = {key: value for key, value in meta.items() if key not in ('stdout', 'stderr')}
required = {'nonce','phase','argv','cwd','deadline','uid','gid','groups','supervisor_pid',
'leader_pid','leader_start','leader_rc','leader_reaped','timeout','controlled_rc',
'descendants_absent','pg_absent','remaining','exceptions','tracked','signals','adopted',
'elapsed','stdin_len','stdin_sha','stdout_len','stdout_sha','stderr_len','stderr_sha',
'stdout','stderr'}
if not isinstance(meta, dict) or set(meta) != required:
raise ValueError(name + ': incomplete metadata')
for key in ('nonce','phase','argv','cwd','deadline'):
if meta[key] != payload[key]:
raise ValueError(name + ': metadata binding ' + key)
uid, gid = (65534,65534) if os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534' else (os.getuid(),os.getgid())
expected_groups = [] if uid == 65534 else os.getgroups()
if type(meta['uid']) is not int or type(meta['gid']) is not int or meta['uid'] != uid or meta['gid'] != gid:
raise ValueError(name + ': identity')
if type(meta['groups']) is not list or any(type(x) is not int for x in meta['groups']) or meta['groups'] != expected_groups:
raise ValueError(name + ': groups')
if any(not positive(meta[key]) for key in ('supervisor_pid','leader_pid','leader_start')):
raise ValueError(name + ': process identity')
if meta['supervisor_pid'] == meta['leader_pid']:
raise ValueError(name + ': equal process identities')
if type(meta['elapsed']) not in (int,float) or not math.isfinite(meta['elapsed']) or not 0 <= meta['elapsed'] <= deadline + 5:
raise ValueError(name + ': elapsed')
for key in ('timeout','leader_reaped','descendants_absent','pg_absent'):
if type(meta[key]) is not bool:
raise ValueError(name + ': nonboolean ' + key)
if type(meta['leader_rc']) is not int or type(meta['controlled_rc']) is not int:
raise ValueError(name + ': returncode type')
if meta['controlled_rc'] != (124 if meta['timeout'] else meta['leader_rc']):
raise ValueError(name + ': returncode disagreement')
for key in ('remaining','exceptions','tracked','signals','adopted'):
if type(meta[key]) is not list:
raise ValueError(name + ': nonlist ' + key)
for entry in meta['tracked']:
if type(entry) is not list or len(entry) != 2 or any(not positive(x) for x in entry):
raise ValueError(name + ': tracked identity')
for entry in meta['signals']:
if type(entry) is not list or len(entry) != 3 or any(not positive(x) for x in entry):
raise ValueError(name + ': signal identity')
for entry in meta['adopted']:
if type(entry) is not list or len(entry) != 2 or not positive(entry[0]) or type(entry[1]) is not int:
raise ValueError(name + ': adopted identity')
decoded = []
for stream in ('stdout','stderr'):
if type(meta[stream + '_len']) is not int or meta[stream + '_len'] < 0 or type(meta[stream]) is not str:
raise ValueError(name + ': IO type')
value = base64.b64decode(meta[stream], validate=True)
if len(value) != meta[stream + '_len'] or digest(value) != meta[stream + '_sha']:
raise ValueError(name + ': IO binding')
decoded.append(value)
if type(meta['stdin_len']) is not int or meta['stdin_len'] != len(raw) or meta['stdin_sha'] != digest(raw):
raise ValueError(name + ': stdin binding')
if wrapper.returncode != 0 or wrapper.stderr or meta['exceptions'] or not meta['leader_reaped'] or not meta['descendants_absent'] or not meta['pg_absent'] or meta['remaining']:
raise ValueError(name + ': supervision failed')
if meta['timeout'] or meta['leader_rc'] != 0 or meta['controlled_rc'] != 0:
raise ValueError(name + ': controlled exit %d leader %d timeout %s' %
(meta['controlled_rc'],meta['leader_rc'],meta['timeout']))
return decoded[0], decoded[1]
def cases():
rng = random.Random(97052026)
items = []
def add(data, maximum=256, stop=0, divisor=7, remainder=3, seed=31,
reject=0, error=999999, chunk=11):
items.append(dict(data=data,maximum=maximum,stop=stop,divisor=divisor,
remainder=remainder,seed=seed,reject=reject,error=error,chunk=chunk))
for data in (b'',b'\n',b'\n\n',b'a',b'a\n',b'a\nb',b'\0x\n',b'\xff\x80\r\0\n',bytes(range(256))):
add(data)
for maximum in (1,2,15,16,17,37,38,127,256,1024):
add(b'x'*(maximum-1)+b'\n',maximum=maximum)
add(b'x'*maximum+b'\n',maximum=maximum)
for data in (b'',b'abc',b'abc\ndef\n',b'\n\0\n'):
add(data,reject=1)
for error in (0,1,3,4,5,7,8,9):
add(b'abc\ndef\nx',error=error,chunk=3)
for stop in (1,2,3):
add(b'a\nbb\nccc\nmore',stop=stop,chunk=37)
for maximum,stop,divisor,remainder in ((0,0,1,0),(65537,0,1,0),(10,1000001,1,0),(10,0,0,0),(10,0,257,0),(10,0,3,3)):
add(b'abc\n',maximum=maximum,stop=stop,divisor=divisor,remainder=remainder)
for _ in range(28):
data=bytes(rng.randrange(256) if rng.randrange(4) else 10 for _ in range(rng.randrange(1,160)))
divisor=rng.randrange(1,257)
add(data,maximum=rng.choice([1,7,37,256]),stop=rng.choice([0,0,1,3]),
divisor=divisor,remainder=rng.randrange(divisor),seed=rng.getrandbits(64),
error=rng.choice([999999,rng.randrange(len(data)+1)]),chunk=rng.choice([1,7,37,81]))
return items
def model(case):
data=case['data'];limit=case['maximum'];stop=case['stop'];div=case['divisor'];rem=case['remainder']
summary=[0,0,0,0,case['seed'],0]
if not 1 <= limit <= 65536 or not 0 <= stop <= 1000000 or not 1 <= div <= 256 or not 0 <= rem < div:
return [-4,-4]+summary+[0]
pos=0;current=bytearray();status=0
while True:
if stop and summary[0] >= stop:break
if pos==case['error']:
status=-1;break
if pos==len(data):
if not current:break
complete=True
else:
value=data[pos];pos+=1
if len(current)==limit:status=-3;break
if case['reject']:status=-2;break
current.append(value)
complete=value==10
if complete:
length=len(current)
summary[0]+=1;summary[1]+=length;summary[5]=max(summary[5],length)
if sum(current)%div==rem:summary[2]+=1;summary[3]+=length
for b in list(current)+[255]:
summary[4]=(summary[4]*257+b+1)&((1<<64)-1)
current.clear()
if pos==len(data) and (not data or data[-1]!=10):break
return [status,status]+summary+[pos]
def decimal(token):
if re.fullmatch(rb'[+-]?[0-9]+',token) is None:
raise ValueError('invalid decimal token')
sign=-1 if token[:1]==b'-' else 1
digits=token[1:] if token[:1] in (b'-',b'+') else token
digits=digits.lstrip(b'0') or b'0'
if len(digits)>20:raise ValueError('decimal range')
return sign*int(digits)
def compare(raw, jobs):
rows=raw.split(b'\n')
if rows[-1:]==[b'']:rows.pop()
if len(rows)!=len(jobs)+1:raise ValueError('observation row count')
probe=re.split(rb'[ \t\r]+',rows.pop(0).strip(b' \t\r'))
if not probe or probe[0]!=b'P' or [decimal(t) for t in probe[1:]] != [0,0,0,0,1,1,1,1,1,1,-4,0,0,0]:
raise ValueError('direct buffer API observation')
for number,(row,case) in enumerate(zip(rows,jobs),1):
tokens=re.split(rb'[ \t\r]+',row.strip(b' \t\r'))
if len(tokens)!=19 or tokens[0]!=b'J':raise ValueError('row %d structure'%number)
values=[decimal(t) for t in tokens[1:]]
expected=model(case)
semantic=values[1:9]+[values[17]]
if values[0]!=number or semantic!=expected:
raise ValueError('row %d semantic got %r expected %r'%(number,semantic,expected))
live,live_bytes,invalid,peak,attempts,successes,releases,reads=values[9:17]
if live or live_bytes or invalid or any(x<0 for x in (peak,attempts,successes,releases,reads)):
raise ValueError('row %d resources'%number)
if successes>attempts or releases>successes or (successes==0)!=(releases==0):
raise ValueError('row %d allocation conservation'%number)
if successes and not peak:raise ValueError('row %d peak'%number)
def main():
scratch=None
success=False
cleaned=False
absent=False
failure=''
try:
support=json.loads((Path(__file__).parent/'context.json').read_text())['readonly']
candidate=Path(sys.argv[1])
names=('source.c','line.c','summary.c','job.c')
sources={name:(candidate/name).read_bytes() for name in names}
jobs=cases()
raw=''.join('%d %d %d %d %d %d %d %d %d %s\n'%(i,c['maximum'],c['stop'],c['divisor'],c['remainder'],c['seed'],c['reject'],c['error'],c['chunk'],c['data'].hex() or '-') for i,c in enumerate(jobs,1)).encode()
scratch=tempfile.mkdtemp(prefix='resource-line-buffer-')
for name,content in support.items():
target=Path(scratch)/name
target.parent.mkdir(parents=True,exist_ok=True)
# Root directory ownership belongs to the bos mkdtemp hook.
if target.parent != Path(scratch):target.parent.chmod(0o777)
target.write_text(content);target.chmod(0o644)
for name,content in sources.items():
target=Path(scratch)/name;target.write_bytes(content);target.chmod(0o644)
argv=['gcc','-std=c11','-O2','-Wall','-Wextra','-Werror','-pedantic',*names,'observe.c','-o','observe']
phase('compiler',argv,scratch,b'',BUILD_SECONDS)
stdout,stderr=phase('candidate',[scratch+'/observe'],scratch,raw,RUN_SECONDS)
compare(stdout,jobs)
success=True
except BaseException as exc:
failure=type(exc).__name__+': '+str(exc)
EXCEPTIONS.append(['main',type(exc).__name__,str(exc)])
finally:
if scratch is not None:
try:
phase('cleanup',[sys.executable,'-I','-c',CLEAN,scratch],'/tmp',b'',CLEAN_SECONDS)
cleaned=True
except BaseException as exc:
EXCEPTIONS.append(['cleanup',type(exc).__name__,str(exc)])
try:
phase('absence',[sys.executable,'-I','-c',ABSENCE,scratch],'/tmp',b'',ABSENCE_SECONDS)
absent=True
except BaseException as exc:
EXCEPTIONS.append(['absence',type(exc).__name__,str(exc)])
facts=dict(stages=STAGES,exceptions=EXCEPTIONS,success=success,cleaned=cleaned,absent=absent,
scratch=scratch,evidence_rows=len(cases())+1)
encoded=base64.b64encode(zlib.compress(json.dumps(facts,separators=(',',':')).encode(),9)).decode()
print('NATIVE '+encoded,file=sys.stderr)
if success and cleaned and absent and not EXCEPTIONS:
print('PASS');return 0
print(failure or 'required cleanup or absence failed');return 1
if __name__=='__main__':
raise SystemExit(main())