One user-space repair task per eligible HWE debug-diag / compile-deploy base ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content filter. Each ships a public spec with a check-to-basis table, at least two defects with single-fix fixtures, a symptom-masking fixture and an alternative correct implementation. Claude re-checked all 320: structure, the HWE quality items and trace6 Docker calibration receipts against current digests (320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120 ad-hoc family names; the next batch maps them to the canonical list. The batch's 1,256-line progress log stays in its factory report and the 36-local branch batch15-hwe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
255 lines
19 KiB
Python
255 lines
19 KiB
Python
"""Independent byte-record oracle and candidate-identity process supervision."""
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import uuid
|
|
import zlib
|
|
|
|
SUPERVISOR = "import base64,ctypes,hashlib,json,os,selectors,signal,subprocess,sys,time\npayload=json.loads(sys.stdin.buffer.read())\nstart=time.monotonic()\nerrors=[]\ntracked={}\nsignals=[]\nadopted=[]\nleader=None\nleader_start=None\nleader_rc=None\ntimed_out=False\nout=bytearray();err=bytearray()\nuid=os.getuid();gid=os.getgid();groups=os.getgroups()\n\ndef stat_pid(pid):\n try:\n data=open('/proc/%d/stat'%pid,'rb').read()\n tail=data[data.rfind(b')')+2:].split()\n return (int(pid),int(tail[1]),int(tail[2]),int(tail[19]),tail[0].decode('ascii'))\n except (FileNotFoundError,ProcessLookupError):\n return None\n except BaseException as exc:\n errors.append(['stat',int(pid),type(exc).__name__,str(exc)])\n return None\n\ndef closure(table,root,known):\n roots={root}\n for pid,token in known.items():\n row=table.get(pid)\n if row and row[3]==token and row[4] not in ('Z','X'):\n roots.add(pid)\n found=set()\n changed=True\n while changed:\n changed=False\n for pid,row in table.items():\n if pid!=root and row[1] in roots and pid not in found:\n found.add(pid); roots.add(pid); changed=True\n found.update(pid for pid in roots if pid!=root)\n return found\n\ndef discover():\n table={}\n try:\n names=os.listdir('/proc')\n except BaseException as exc:\n errors.append(['proc',type(exc).__name__,str(exc)]);return {}\n for name in names:\n if name.isascii() and name.isdigit():\n row=stat_pid(int(name))\n if row:table[row[0]]=row\n found=closure(table,os.getpid(),tracked)\n for pid in found:tracked[pid]=table[pid][3]\n return {pid:table[pid] for pid in found}\n\ndef send(pid,token,sig):\n fresh=stat_pid(pid)\n if fresh is None or fresh[3]!=token or fresh[4] in ('Z','X'):\n return\n try:\n os.kill(pid,sig)\n signals.append([pid,token,sig])\n except ProcessLookupError:\n return\n except BaseException as exc:\n errors.append(['signal',pid,token,type(exc).__name__,str(exc)])\n\ndef reap():\n # Never waitpid(-1) until Popen has obtained its own leader status.\n if leader is not None and leader.poll() is None:return\n while True:\n try:\n pid,status=os.waitpid(-1,os.WNOHANG)\n if not pid:break\n adopted.append([pid,status])\n except ChildProcessError:break\n except BaseException as exc:\n errors.append(['wait',type(exc).__name__,str(exc)]);break\n\ndef drain(selector):\n for key,events in selector.select(0.02):\n file=key.fileobj\n if key.data=='in':\n try:\n sent=os.write(file.fileno(),pending[0][:65536])\n pending[0]=pending[0][sent:]\n except BrokenPipeError:pending[0]=b''\n if not pending[0]:selector.unregister(file);file.close()\n else:\n try:data=os.read(file.fileno(),65536)\n except BlockingIOError:continue\n if not data:selector.unregister(file);file.close()\n elif key.data=='out':out.extend(data)\n else:err.extend(data)\n\ntry:\n if ctypes.CDLL(None,use_errno=True).prctl(36,1,0,0,0)!=0:\n raise OSError(ctypes.get_errno(),'subreaper')\n command=payload['argv']\n raw=base64.b64decode(payload['input'])\n leader=subprocess.Popen(command,cwd=payload['cwd'],stdin=subprocess.PIPE,\n stdout=subprocess.PIPE,stderr=subprocess.PIPE,start_new_session=True)\n row=stat_pid(leader.pid)\n if row is None:raise RuntimeError('leader identity unavailable')\n leader_start=row[3];tracked[leader.pid]=leader_start\n pending=[raw]\n selector=selectors.DefaultSelector()\n for stream,key in [(leader.stdout,'out'),(leader.stderr,'err')]:\n os.set_blocking(stream.fileno(),False);selector.register(stream,selectors.EVENT_READ,key)\n if raw:\n os.set_blocking(leader.stdin.fileno(),False)\n selector.register(leader.stdin,selectors.EVENT_WRITE,'in')\n else:leader.stdin.close()\n deadline=start+payload['deadline']\n while True:\n leader_rc=leader.poll()\n discover()\n drain(selector)\n if leader_rc is not None and not selector.get_map():break\n if time.monotonic()>=deadline:\n timed_out=True;break\n # Handle both same-PG children and reparented, setsid descendants.\n for sig,window in [(signal.SIGTERM,0.2),(signal.SIGKILL,1.6)]:\n limit=time.monotonic()+window\n while True:\n leader_rc=leader.poll()\n rows=discover()\n for pid,row in rows.items():send(pid,row[3],sig)\n reap()\n drain(selector)\n live=discover()\n if not live and leader.poll() is not None and not selector.get_map():break\n if time.monotonic()>=limit:break\n leader_rc=leader.poll()\n if leader_rc is not None:leader.wait();reap()\n remaining=discover()\n absent=not remaining\n # PG existence alone can include foreign reused groups; record token-bound members.\n pg_absent=not any(row[2]==leader.pid for row in remaining.values())\n selector.close()\nexcept BaseException as exc:\n errors.append(['supervisor',type(exc).__name__,str(exc)])\n if leader is not None:\n for pid,row in discover().items():send(pid,row[3],signal.SIGKILL)\n until=time.monotonic()+1.8\n while leader.poll() is None and time.monotonic()<until:time.sleep(0.01)\n leader_rc=leader.poll();reap()\n remaining=discover();absent=not remaining\n pg_absent=not any(leader is not None and row[2]==leader.pid for row in remaining.values())\nmeta={'nonce':payload['nonce'],'phase':payload['phase'],'argv':payload['argv'],'cwd':payload['cwd'],\n 'deadline':payload['deadline'],'uid':uid,'gid':gid,'groups':groups,\n 'supervisor_pid':os.getpid(),'leader_pid':None if leader is None else leader.pid,\n 'leader_start':leader_start,'leader_rc':leader_rc,'leader_reaped':leader is not None and leader_rc is not None,\n 'timeout':timed_out,'controlled_rc':124 if timed_out else leader_rc,\n 'descendants_absent':absent,'pg_absent':pg_absent,'remaining':list(remaining.values()),\n 'exceptions':errors,'tracked':list(tracked.items()),'signals':signals,'adopted':adopted,\n 'elapsed':time.monotonic()-start,'stdin_len':len(raw) if 'raw' in globals() else 0,\n 'stdin_sha':hashlib.sha256(raw if 'raw' in globals() else b'').hexdigest(),\n 'stdout_len':len(out),'stdout_sha':hashlib.sha256(out).hexdigest(),\n 'stderr_len':len(err),'stderr_sha':hashlib.sha256(err).hexdigest(),\n 'stdout':base64.b64encode(out).decode(),'stderr':base64.b64encode(err).decode()}\nsys.stdout.buffer.write(json.dumps(meta,separators=(',',':')).encode())\n"
|
|
|
|
BUILD_SECONDS = 20
|
|
RUN_SECONDS = 10
|
|
CLEAN_SECONDS = 5
|
|
ABSENCE_SECONDS = 5
|
|
CLEAN = "import shutil,sys; shutil.rmtree(sys.argv[1])"
|
|
ABSENCE = "import os,sys\ntry: os.lstat(sys.argv[1])\nexcept FileNotFoundError: raise SystemExit(0)\nraise SystemExit(1)"
|
|
STAGES = []
|
|
EXCEPTIONS = []
|
|
|
|
|
|
def digest(raw):
|
|
return hashlib.sha256(raw).hexdigest()
|
|
|
|
|
|
def positive(value):
|
|
return type(value) is int and value > 0
|
|
|
|
|
|
def phase(name, argv, cwd, raw, deadline):
|
|
nonce = uuid.uuid4().hex
|
|
payload = dict(phase=name, argv=argv, cwd=cwd, input=base64.b64encode(raw).decode(),
|
|
deadline=deadline, nonce=nonce)
|
|
record = dict(phase=name, nonce=nonce, argv=argv, cwd=cwd, deadline=deadline)
|
|
STAGES.append(record)
|
|
wrapper = subprocess.run([sys.executable, '-I', '-c', SUPERVISOR],
|
|
input=json.dumps(payload).encode(), capture_output=True,
|
|
timeout=deadline + 8)
|
|
# Record actual wrapper facts before parsing potentially missing/malformed wire.
|
|
wire = wrapper.stdout
|
|
record.update(wrapper_rc=wrapper.returncode, wire_len=len(wire), wire_sha=digest(wire),
|
|
wrapper_stderr_len=len(wrapper.stderr), wrapper_stderr_sha=digest(wrapper.stderr))
|
|
meta = json.loads(wire)
|
|
record['native'] = {key: value for key, value in meta.items() if key not in ('stdout', 'stderr')}
|
|
required = {'nonce','phase','argv','cwd','deadline','uid','gid','groups','supervisor_pid',
|
|
'leader_pid','leader_start','leader_rc','leader_reaped','timeout','controlled_rc',
|
|
'descendants_absent','pg_absent','remaining','exceptions','tracked','signals','adopted',
|
|
'elapsed','stdin_len','stdin_sha','stdout_len','stdout_sha','stderr_len','stderr_sha',
|
|
'stdout','stderr'}
|
|
if not isinstance(meta, dict) or set(meta) != required:
|
|
raise ValueError(name + ': incomplete metadata')
|
|
for key in ('nonce','phase','argv','cwd','deadline'):
|
|
if meta[key] != payload[key]:
|
|
raise ValueError(name + ': metadata binding ' + key)
|
|
uid, gid = (65534,65534) if os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534' else (os.getuid(),os.getgid())
|
|
expected_groups = [] if uid == 65534 else os.getgroups()
|
|
if type(meta['uid']) is not int or type(meta['gid']) is not int or meta['uid'] != uid or meta['gid'] != gid:
|
|
raise ValueError(name + ': identity')
|
|
if type(meta['groups']) is not list or any(type(x) is not int for x in meta['groups']) or meta['groups'] != expected_groups:
|
|
raise ValueError(name + ': groups')
|
|
if any(not positive(meta[key]) for key in ('supervisor_pid','leader_pid','leader_start')):
|
|
raise ValueError(name + ': process identity')
|
|
if meta['supervisor_pid'] == meta['leader_pid']:
|
|
raise ValueError(name + ': equal process identities')
|
|
if type(meta['elapsed']) not in (int,float) or not math.isfinite(meta['elapsed']) or not 0 <= meta['elapsed'] <= deadline + 5:
|
|
raise ValueError(name + ': elapsed')
|
|
for key in ('timeout','leader_reaped','descendants_absent','pg_absent'):
|
|
if type(meta[key]) is not bool:
|
|
raise ValueError(name + ': nonboolean ' + key)
|
|
if type(meta['leader_rc']) is not int or type(meta['controlled_rc']) is not int:
|
|
raise ValueError(name + ': returncode type')
|
|
if meta['controlled_rc'] != (124 if meta['timeout'] else meta['leader_rc']):
|
|
raise ValueError(name + ': returncode disagreement')
|
|
for key in ('remaining','exceptions','tracked','signals','adopted'):
|
|
if type(meta[key]) is not list:
|
|
raise ValueError(name + ': nonlist ' + key)
|
|
for entry in meta['tracked']:
|
|
if type(entry) is not list or len(entry) != 2 or any(not positive(x) for x in entry):
|
|
raise ValueError(name + ': tracked identity')
|
|
for entry in meta['signals']:
|
|
if type(entry) is not list or len(entry) != 3 or any(not positive(x) for x in entry):
|
|
raise ValueError(name + ': signal identity')
|
|
for entry in meta['adopted']:
|
|
if type(entry) is not list or len(entry) != 2 or not positive(entry[0]) or type(entry[1]) is not int:
|
|
raise ValueError(name + ': adopted identity')
|
|
decoded = []
|
|
for stream in ('stdout','stderr'):
|
|
if type(meta[stream + '_len']) is not int or meta[stream + '_len'] < 0 or type(meta[stream]) is not str:
|
|
raise ValueError(name + ': IO type')
|
|
value = base64.b64decode(meta[stream], validate=True)
|
|
if len(value) != meta[stream + '_len'] or digest(value) != meta[stream + '_sha']:
|
|
raise ValueError(name + ': IO binding')
|
|
decoded.append(value)
|
|
if type(meta['stdin_len']) is not int or meta['stdin_len'] != len(raw) or meta['stdin_sha'] != digest(raw):
|
|
raise ValueError(name + ': stdin binding')
|
|
if wrapper.returncode != 0 or wrapper.stderr or meta['exceptions'] or not meta['leader_reaped'] or not meta['descendants_absent'] or not meta['pg_absent'] or meta['remaining']:
|
|
raise ValueError(name + ': supervision failed')
|
|
if meta['timeout'] or meta['leader_rc'] != 0 or meta['controlled_rc'] != 0:
|
|
raise ValueError(name + ': controlled exit %d leader %d timeout %s' %
|
|
(meta['controlled_rc'],meta['leader_rc'],meta['timeout']))
|
|
return decoded[0], decoded[1]
|
|
|
|
|
|
def cases():
|
|
rng = random.Random(97052026)
|
|
items = []
|
|
def add(data, maximum=256, stop=0, divisor=7, remainder=3, seed=31,
|
|
reject=0, error=999999, chunk=11):
|
|
items.append(dict(data=data,maximum=maximum,stop=stop,divisor=divisor,
|
|
remainder=remainder,seed=seed,reject=reject,error=error,chunk=chunk))
|
|
for data in (b'',b'\n',b'\n\n',b'a',b'a\n',b'a\nb',b'\0x\n',b'\xff\x80\r\0\n',bytes(range(256))):
|
|
add(data)
|
|
for maximum in (1,2,15,16,17,37,38,127,256,1024):
|
|
add(b'x'*(maximum-1)+b'\n',maximum=maximum)
|
|
add(b'x'*maximum+b'\n',maximum=maximum)
|
|
for data in (b'',b'abc',b'abc\ndef\n',b'\n\0\n'):
|
|
add(data,reject=1)
|
|
for error in (0,1,3,4,5,7,8,9):
|
|
add(b'abc\ndef\nx',error=error,chunk=3)
|
|
for stop in (1,2,3):
|
|
add(b'a\nbb\nccc\nmore',stop=stop,chunk=37)
|
|
for maximum,stop,divisor,remainder in ((0,0,1,0),(65537,0,1,0),(10,1000001,1,0),(10,0,0,0),(10,0,257,0),(10,0,3,3)):
|
|
add(b'abc\n',maximum=maximum,stop=stop,divisor=divisor,remainder=remainder)
|
|
for _ in range(28):
|
|
data=bytes(rng.randrange(256) if rng.randrange(4) else 10 for _ in range(rng.randrange(1,160)))
|
|
divisor=rng.randrange(1,257)
|
|
add(data,maximum=rng.choice([1,7,37,256]),stop=rng.choice([0,0,1,3]),
|
|
divisor=divisor,remainder=rng.randrange(divisor),seed=rng.getrandbits(64),
|
|
error=rng.choice([999999,rng.randrange(len(data)+1)]),chunk=rng.choice([1,7,37,81]))
|
|
return items
|
|
|
|
|
|
def model(case):
|
|
data=case['data'];limit=case['maximum'];stop=case['stop'];div=case['divisor'];rem=case['remainder']
|
|
summary=[0,0,0,0,case['seed'],0]
|
|
if not 1 <= limit <= 65536 or not 0 <= stop <= 1000000 or not 1 <= div <= 256 or not 0 <= rem < div:
|
|
return [-4,-4]+summary+[0]
|
|
pos=0;current=bytearray();status=0
|
|
while True:
|
|
if stop and summary[0] >= stop:break
|
|
if pos==case['error']:
|
|
status=-1;break
|
|
if pos==len(data):
|
|
if not current:break
|
|
complete=True
|
|
else:
|
|
value=data[pos];pos+=1
|
|
if len(current)==limit:status=-3;break
|
|
if case['reject']:status=-2;break
|
|
current.append(value)
|
|
complete=value==10
|
|
if complete:
|
|
length=len(current)
|
|
summary[0]+=1;summary[1]+=length;summary[5]=max(summary[5],length)
|
|
if sum(current)%div==rem:summary[2]+=1;summary[3]+=length
|
|
for b in list(current)+[255]:
|
|
summary[4]=(summary[4]*257+b+1)&((1<<64)-1)
|
|
current.clear()
|
|
if pos==len(data) and (not data or data[-1]!=10):break
|
|
return [status,status]+summary+[pos]
|
|
|
|
|
|
def decimal(token):
|
|
if re.fullmatch(rb'[+-]?[0-9]+',token) is None:
|
|
raise ValueError('invalid decimal token')
|
|
sign=-1 if token[:1]==b'-' else 1
|
|
digits=token[1:] if token[:1] in (b'-',b'+') else token
|
|
digits=digits.lstrip(b'0') or b'0'
|
|
if len(digits)>20:raise ValueError('decimal range')
|
|
return sign*int(digits)
|
|
|
|
|
|
def compare(raw, jobs):
|
|
rows=raw.split(b'\n')
|
|
if rows[-1:]==[b'']:rows.pop()
|
|
if len(rows)!=len(jobs)+1:raise ValueError('observation row count')
|
|
probe=re.split(rb'[ \t\r]+',rows.pop(0).strip(b' \t\r'))
|
|
if not probe or probe[0]!=b'P' or [decimal(t) for t in probe[1:]] != [0,0,0,0,1,1,1,1,1,1,-4,0,0,0]:
|
|
raise ValueError('direct buffer API observation')
|
|
for number,(row,case) in enumerate(zip(rows,jobs),1):
|
|
tokens=re.split(rb'[ \t\r]+',row.strip(b' \t\r'))
|
|
if len(tokens)!=19 or tokens[0]!=b'J':raise ValueError('row %d structure'%number)
|
|
values=[decimal(t) for t in tokens[1:]]
|
|
expected=model(case)
|
|
semantic=values[1:9]+[values[17]]
|
|
if values[0]!=number or semantic!=expected:
|
|
raise ValueError('row %d semantic got %r expected %r'%(number,semantic,expected))
|
|
live,live_bytes,invalid,peak,attempts,successes,releases,reads=values[9:17]
|
|
if live or live_bytes or invalid or any(x<0 for x in (peak,attempts,successes,releases,reads)):
|
|
raise ValueError('row %d resources'%number)
|
|
if successes>attempts or releases>successes or (successes==0)!=(releases==0):
|
|
raise ValueError('row %d allocation conservation'%number)
|
|
if successes and not peak:raise ValueError('row %d peak'%number)
|
|
|
|
|
|
def main():
|
|
scratch=None
|
|
success=False
|
|
cleaned=False
|
|
absent=False
|
|
failure=''
|
|
try:
|
|
support=json.loads((Path(__file__).parent/'context.json').read_text())['readonly']
|
|
candidate=Path(sys.argv[1])
|
|
names=('source.c','line.c','summary.c','job.c')
|
|
sources={name:(candidate/name).read_bytes() for name in names}
|
|
jobs=cases()
|
|
raw=''.join('%d %d %d %d %d %d %d %d %d %s\n'%(i,c['maximum'],c['stop'],c['divisor'],c['remainder'],c['seed'],c['reject'],c['error'],c['chunk'],c['data'].hex() or '-') for i,c in enumerate(jobs,1)).encode()
|
|
scratch=tempfile.mkdtemp(prefix='resource-line-buffer-')
|
|
for name,content in support.items():
|
|
target=Path(scratch)/name
|
|
target.parent.mkdir(parents=True,exist_ok=True)
|
|
# Root directory ownership belongs to the bos mkdtemp hook.
|
|
if target.parent != Path(scratch):target.parent.chmod(0o777)
|
|
target.write_text(content);target.chmod(0o644)
|
|
for name,content in sources.items():
|
|
target=Path(scratch)/name;target.write_bytes(content);target.chmod(0o644)
|
|
argv=['gcc','-std=c11','-O2','-Wall','-Wextra','-Werror','-pedantic',*names,'observe.c','-o','observe']
|
|
phase('compiler',argv,scratch,b'',BUILD_SECONDS)
|
|
stdout,stderr=phase('candidate',[scratch+'/observe'],scratch,raw,RUN_SECONDS)
|
|
compare(stdout,jobs)
|
|
success=True
|
|
except BaseException as exc:
|
|
failure=type(exc).__name__+': '+str(exc)
|
|
EXCEPTIONS.append(['main',type(exc).__name__,str(exc)])
|
|
finally:
|
|
if scratch is not None:
|
|
try:
|
|
phase('cleanup',[sys.executable,'-I','-c',CLEAN,scratch],'/tmp',b'',CLEAN_SECONDS)
|
|
cleaned=True
|
|
except BaseException as exc:
|
|
EXCEPTIONS.append(['cleanup',type(exc).__name__,str(exc)])
|
|
try:
|
|
phase('absence',[sys.executable,'-I','-c',ABSENCE,scratch],'/tmp',b'',ABSENCE_SECONDS)
|
|
absent=True
|
|
except BaseException as exc:
|
|
EXCEPTIONS.append(['absence',type(exc).__name__,str(exc)])
|
|
facts=dict(stages=STAGES,exceptions=EXCEPTIONS,success=success,cleaned=cleaned,absent=absent,
|
|
scratch=scratch,evidence_rows=len(cases())+1)
|
|
encoded=base64.b64encode(zlib.compress(json.dumps(facts,separators=(',',':')).encode(),9)).decode()
|
|
print('NATIVE '+encoded,file=sys.stderr)
|
|
if success and cleaned and absent and not EXCEPTIONS:
|
|
print('PASS');return 0
|
|
print(failure or 'required cleanup or absence failed');return 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
raise SystemExit(main())
|