One user-space repair task per eligible HWE debug-diag / compile-deploy base ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content filter. Each ships a public spec with a check-to-basis table, at least two defects with single-fix fixtures, a symptom-masking fixture and an alternative correct implementation. Claude re-checked all 320: structure, the HWE quality items and trace6 Docker calibration receipts against current digests (320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120 ad-hoc family names; the next batch maps them to the canonical list. The batch's 1,256-line progress log stays in its factory report and the 36-local branch batch15-hwe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
456 lines
27 KiB
Python
456 lines
27 KiB
Python
import base64
|
|
|
|
import bz2
|
|
|
|
import hashlib
|
|
|
|
import json
|
|
|
|
import math
|
|
|
|
import os
|
|
|
|
from pathlib import Path
|
|
|
|
import random
|
|
|
|
import subprocess
|
|
|
|
import sys
|
|
|
|
import tempfile
|
|
|
|
import shutil
|
|
|
|
import time
|
|
|
|
RECOVERY_SECONDS = 3
|
|
|
|
RECORD_WIRE = False
|
|
|
|
SUPERVISOR = r'''
|
|
import base64,bz2,ctypes,hashlib,json,os,signal,subprocess,sys,time
|
|
cfg=json.loads(sys.argv[1]); start=time.monotonic(); errors=[]; tracked={}; signals={}; waits=[]
|
|
leader=None; leader_start=None; output=b''; diagnostic=b''; timed_out=False; drained=False
|
|
reaped=False; leader_rc=None; remaining=[]; pg_absent=False
|
|
|
|
def fault(op, error):
|
|
errors.append([op,type(error).__name__,str(error)])
|
|
|
|
def exact(pid):
|
|
try:
|
|
with open('/proc/%d/stat'%pid,'rb') as stream: data=stream.read()
|
|
words=data[data.rfind(b')')+2:].split()
|
|
return {'pid':pid,'state':words[0].decode('ascii'),'ppid':int(words[1]),'pg':int(words[2]),'start':int(words[19])}
|
|
except (FileNotFoundError,ProcessLookupError): return None
|
|
|
|
def table_now():
|
|
result={}
|
|
for name in os.listdir('/proc'):
|
|
if name.isascii() and name.isdigit():
|
|
value=exact(int(name))
|
|
if value is not None: result[value['pid']]=value
|
|
return result
|
|
|
|
def closure(table, supervisor, remembered):
|
|
roots={supervisor}
|
|
for pid,old in remembered.items():
|
|
current=table.get(pid)
|
|
if current and current['start']==old['start'] and current['state'] not in ('Z','X'):
|
|
roots.add(pid)
|
|
found={}
|
|
changed=True
|
|
while changed:
|
|
changed=False
|
|
for pid,current in table.items():
|
|
if pid!=supervisor and pid not in found and (current['ppid'] in roots or pid in roots):
|
|
found[pid]=current; roots.add(pid); changed=True
|
|
return found
|
|
|
|
def discover():
|
|
table=table_now(); found=closure(table,os.getpid(),tracked)
|
|
for pid,current in found.items():
|
|
if pid not in tracked or tracked[pid]['start']!=current['start']:
|
|
tracked[pid]=dict(current)
|
|
return table,found
|
|
|
|
def send_fresh(pid, token):
|
|
current=exact(pid)
|
|
if current is None or current['start']!=token['start'] or current['state'] in ('Z','X'): return
|
|
os.kill(pid,signal.SIGKILL)
|
|
signals[(pid,token['start'])]=signals.get((pid,token['start']),0)+1
|
|
|
|
def own_wait():
|
|
global reaped,leader_rc
|
|
if leader is not None:
|
|
value=leader.poll()
|
|
if value is not None:
|
|
leader_rc=leader.wait(timeout=0)
|
|
reaped=True
|
|
|
|
def adopted_wait():
|
|
if leader is not None and not reaped: return
|
|
while True:
|
|
try: pid,status=os.waitpid(-1,os.WNOHANG)
|
|
except ChildProcessError: break
|
|
if pid==0: break
|
|
waits.append([pid,None,os.waitstatus_to_exitcode(status)])
|
|
|
|
supervisor_start=exact(os.getpid())['start']
|
|
try:
|
|
libc=ctypes.CDLL(None,use_errno=True)
|
|
if libc.prctl(36,1,0,0,0)!=0: raise OSError(ctypes.get_errno(),'subreaper')
|
|
leader=subprocess.Popen(cfg['argv'],cwd=cfg['cwd'],stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,stderr=subprocess.PIPE,start_new_session=True)
|
|
info=exact(leader.pid)
|
|
if info is None: raise RuntimeError('leader identity disappeared before capture')
|
|
leader_start=info['start']; tracked[leader.pid]=info
|
|
payload=sys.stdin.buffer.read()
|
|
first=True
|
|
while True:
|
|
try:
|
|
output,diagnostic=leader.communicate(input=payload if first else None,timeout=0.04)
|
|
drained=True; own_wait(); break
|
|
except subprocess.TimeoutExpired:
|
|
first=False
|
|
own_wait()
|
|
discover()
|
|
if time.monotonic()-start>=cfg['deadline']:
|
|
timed_out=True; break
|
|
except BaseException as error:
|
|
fault('execute',error)
|
|
finally:
|
|
recovery_end=time.monotonic()+cfg['recovery']
|
|
while True:
|
|
try: own_wait()
|
|
except BaseException as error: fault('own_wait',error)
|
|
found={}
|
|
try: _,found=discover()
|
|
except BaseException as error: fault('discover',error)
|
|
for pid,token in list(found.items()):
|
|
try: send_fresh(pid,token)
|
|
except (FileNotFoundError,ProcessLookupError): pass
|
|
except BaseException as error: fault('signal',error)
|
|
try: own_wait()
|
|
except BaseException as error: fault('own_wait',error)
|
|
try: adopted_wait()
|
|
except BaseException as error: fault('adopted_wait',error)
|
|
if leader is not None and not drained:
|
|
try:
|
|
output,diagnostic=leader.communicate(timeout=0.03)
|
|
drained=True; own_wait()
|
|
except subprocess.TimeoutExpired: pass
|
|
except BaseException as error: fault('drain',error)
|
|
try:
|
|
table,found=discover()
|
|
remaining=[[p,v['start'],v['state']] for p,v in found.items()]
|
|
pg_absent=leader is not None and not any(v['pg']==leader.pid for v in table.values())
|
|
except BaseException as error:
|
|
fault('absence',error); remaining=[['unknown']]; pg_absent=False
|
|
if reaped and drained and not remaining and pg_absent: break
|
|
if time.monotonic()>=recovery_end:
|
|
fault('recovery',RuntimeError('deadline with unconfirmed closure')); break
|
|
time.sleep(0.01)
|
|
|
|
meta={'phase':cfg['phase'],'nonce':cfg['nonce'],'argv':cfg['argv'],
|
|
'uid':os.getuid(),'gid':os.getgid(),'groups':os.getgroups(),
|
|
'supervisor_pid':os.getpid(),'supervisor_start':supervisor_start,
|
|
'leader_pid':None if leader is None else leader.pid,'leader_start':leader_start,
|
|
'leader_rc':leader_rc,'leader_reaped':reaped,'timeout':timed_out,
|
|
'controlled_rc':124 if timed_out else leader_rc,'elapsed':time.monotonic()-start,
|
|
'deadline':cfg['deadline'],'recovery':cfg['recovery'],
|
|
'pg_absent':pg_absent,'remaining':remaining,'exceptions':errors,
|
|
'adopted':waits,
|
|
'descendants':[[p,t['start'],signals.get((p,t['start']),0)]
|
|
for p,t in tracked.items() if leader is None or p!=leader.pid],
|
|
'stdout_len':len(output),'stdout_sha256':hashlib.sha256(output).hexdigest(),
|
|
'stderr_len':len(diagnostic),'stderr_sha256':hashlib.sha256(diagnostic).hexdigest()}
|
|
wire={'meta':meta,'stdout':base64.b64encode(bz2.compress(output)).decode('ascii'),
|
|
'stderr':base64.b64encode(bz2.compress(diagnostic)).decode('ascii')}
|
|
sys.stdout.write(json.dumps(wire,separators=(',',':'))+'\n')
|
|
'''
|
|
|
|
CLEANUP = "import shutil,sys; shutil.rmtree(sys.argv[1])"
|
|
|
|
ABSENCE = "import os,sys\ntry: os.lstat(sys.argv[1])\nexcept FileNotFoundError: sys.exit(0)\nsys.exit(1)"
|
|
|
|
def identity_int(value, minimum=0):
|
|
return type(value) is int and value >= minimum
|
|
|
|
def validate_meta(meta, config, out, err):
|
|
required = {'phase','nonce','argv','uid','gid','groups','supervisor_pid','supervisor_start',
|
|
'leader_pid','leader_start','leader_rc','leader_reaped','timeout','controlled_rc',
|
|
'elapsed','deadline','recovery','pg_absent','remaining','exceptions','descendants',
|
|
'stdout_len','stdout_sha256','stderr_len','stderr_sha256','adopted'}
|
|
if type(meta) is not dict or not required <= meta.keys():
|
|
raise ValueError('missing stage metadata')
|
|
if meta['phase'] != config['phase'] or meta['nonce'] != config['nonce'] or meta['argv'] != config['argv']:
|
|
raise ValueError('stage binding conflict')
|
|
for name in ('uid','gid','supervisor_pid','supervisor_start','leader_pid','leader_start'):
|
|
if not identity_int(meta[name], 1 if name not in ('uid','gid') else 0):
|
|
raise ValueError('invalid identity '+name)
|
|
if meta['supervisor_pid'] == meta['leader_pid']:
|
|
raise ValueError('supervisor and leader identical')
|
|
isolated = os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534'
|
|
expected_identity = (65534,65534,[]) if isolated else (os.getuid(),os.getgid(),os.getgroups())
|
|
if type(meta['groups']) is not list or any(type(x) is not int or x < 0 for x in meta['groups']):
|
|
raise ValueError('invalid groups')
|
|
if (meta['uid'],meta['gid'],meta['groups']) != expected_identity:
|
|
raise ValueError('execution identity conflict')
|
|
if type(meta['deadline']) is not int or meta['deadline'] != config['deadline']:
|
|
raise ValueError('deadline conflict')
|
|
if type(meta['recovery']) is not int or meta['recovery'] != RECOVERY_SECONDS:
|
|
raise ValueError('recovery conflict')
|
|
elapsed = meta['elapsed']
|
|
if type(elapsed) not in (int,float) or not math.isfinite(elapsed) or not 0 <= elapsed <= config['deadline']+RECOVERY_SECONDS+2:
|
|
raise ValueError('invalid elapsed')
|
|
for name in ('timeout','leader_reaped','pg_absent'):
|
|
if type(meta[name]) is not bool:
|
|
raise ValueError('invalid boolean '+name)
|
|
for name in ('leader_rc','controlled_rc'):
|
|
if type(meta[name]) is not int:
|
|
raise ValueError('invalid return code '+name)
|
|
if meta['controlled_rc'] != (124 if meta['timeout'] else meta['leader_rc']):
|
|
raise ValueError('return code conflict')
|
|
for name in ('remaining','exceptions','descendants','adopted'):
|
|
if type(meta[name]) is not list:
|
|
raise ValueError('invalid list '+name)
|
|
for row in meta['descendants']:
|
|
if type(row) is not list or len(row) != 3 or not all(identity_int(x,1) for x in row[:2]) or not identity_int(row[2]):
|
|
raise ValueError('invalid descendant record')
|
|
for row in meta['adopted']:
|
|
if type(row) is not list or len(row) != 3 or not identity_int(row[0],1) or row[1] is not None or type(row[2]) is not int:
|
|
raise ValueError('invalid adopted wait record')
|
|
for name,data in (('stdout',out),('stderr',err)):
|
|
if not identity_int(meta[name+'_len']) or meta[name+'_len'] != len(data):
|
|
raise ValueError('invalid IO length')
|
|
if type(meta[name+'_sha256']) is not str or meta[name+'_sha256'] != hashlib.sha256(data).hexdigest():
|
|
raise ValueError('invalid IO digest')
|
|
if meta['exceptions'] or meta['remaining'] or not meta['leader_reaped'] or not meta['pg_absent']:
|
|
raise ValueError('stage recovery or exception failure')
|
|
if meta['timeout'] or meta['leader_rc'] != 0 or meta['controlled_rc'] != 0:
|
|
raise ValueError('controlled timeout' if meta['timeout'] else 'leader nonzero exit')
|
|
|
|
def execute_stage(phase, argv, cwd, deadline, payload, evidence):
|
|
nonce = os.urandom(8).hex()
|
|
config = dict(phase=phase,argv=argv,cwd=str(cwd),deadline=deadline,recovery=RECOVERY_SECONDS,nonce=nonce)
|
|
wrapper_argv = [sys.executable,'-I','-c',SUPERVISOR,json.dumps(config,separators=(',',':'))]
|
|
raw = subprocess.run(wrapper_argv,input=payload,capture_output=True,timeout=deadline+RECOVERY_SECONDS+5)
|
|
# Preserve actual wrapper facts before touching or trusting its wire protocol.
|
|
entry = dict(phase=phase,nonce=nonce,wrapper_rc=raw.returncode,
|
|
wire_len=len(raw.stdout),wire_sha256=hashlib.sha256(raw.stdout).hexdigest(),
|
|
wrapper_stderr_len=len(raw.stderr),wrapper_stderr_sha256=hashlib.sha256(raw.stderr).hexdigest())
|
|
evidence.append(entry)
|
|
if raw.returncode != 0:
|
|
raise ValueError(phase+' supervisor nonzero')
|
|
wire = json.loads(raw.stdout)
|
|
out = bz2.decompress(base64.b64decode(wire['stdout'],validate=True))
|
|
err = bz2.decompress(base64.b64decode(wire['stderr'],validate=True))
|
|
meta = wire['meta']
|
|
entry['meta'] = meta
|
|
entry['actual_stdout_len']=len(out)
|
|
entry['actual_stdout_sha256']=hashlib.sha256(out).hexdigest()
|
|
entry['actual_stderr_len']=len(err)
|
|
entry['actual_stderr_sha256']=hashlib.sha256(err).hexdigest()
|
|
validate_meta(meta,config,out,err)
|
|
return out,err
|
|
|
|
|
|
COMPILE_SECONDS = 12
|
|
CANDIDATE_SECONDS = 10
|
|
CLEANUP_SECONDS = 6
|
|
ABSENCE_SECONDS = 6
|
|
FILES = ('rr.h','catalog.c','buffer.c','reader.c')
|
|
SUPPORT = {'observer.c': '#include "rr.h"\n#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\n#include <fcntl.h>\n#include <unistd.h>\n\ntypedef struct {\n unsigned char data[RR_DATA_SIZE];\n size_t length;\n size_t offset;\n int available;\n int failure;\n size_t chunk;\n int handle;\n} source;\ntypedef struct {\n const rr_plan *plan;\n source items[RR_MAX_RESOURCES];\n int events[12000][4];\n size_t count;\n} backend;\nstatic void event(backend *b, int op, int first, int second, int third)\n{\n if (b->count < 12000) {\n int *row = b->events[b->count++];\n row[0] = op; row[1] = first; row[2] = second; row[3] = third;\n }\n}\nstatic int acquire(void *context, const char *name)\n{\n backend *b = context;\n int index = rr_plan_find(b->plan, name);\n int handle = -1;\n if (index >= 0 && b->items[index].available) {\n handle = open("/dev/null", O_RDONLY);\n b->items[index].handle = handle;\n }\n event(b, 1, index, handle, 0);\n return handle;\n}\nstatic source *find_handle(backend *b, int handle)\n{\n size_t index;\n for (index = 0; index < b->plan->count; ++index) {\n if (b->items[index].handle == handle) {\n return &b->items[index];\n }\n }\n return NULL;\n}\nstatic ssize_t consume(void *context, int handle, unsigned char *data, size_t wanted)\n{\n backend *b = context;\n source *s = find_handle(b, handle);\n ssize_t answer = -5;\n if (s != NULL && fcntl(handle, F_GETFD) >= 0) {\n if (s->failure < 0 || s->offset < (size_t)s->failure) {\n size_t count = s->length - s->offset;\n if (count > wanted) count = wanted;\n if (count > s->chunk) count = s->chunk;\n if (s->failure >= 0 && count > (size_t)s->failure - s->offset)\n count = (size_t)s->failure - s->offset;\n memcpy(data, s->data + s->offset, count);\n s->offset += count;\n answer = (ssize_t)count;\n }\n }\n event(b, 2, handle, (int)wanted, (int)answer);\n return answer;\n}\nstatic int relinquish(void *context, int handle)\n{\n backend *b = context;\n source *s = find_handle(b, handle);\n int answer = close(handle);\n if (s != NULL) s->handle = -1;\n event(b, 3, handle, answer, 0);\n return answer;\n}\nint main(void)\n{\n unsigned jobs, job;\n if (scanf("%u", &jobs) != 1) return 71;\n for (job = 0; job < jobs; ++job) {\n char text[512], canonical[256] = "", hex[RR_DATA_SIZE * 2 + 1];\n size_t limit, block, count, index;\n rr_plan plan;\n rr_result result;\n rr_buffer copied, slice;\n backend b;\n rr_io io = { &b, acquire, consume, relinquish };\n int parsed, rc, live = 0, copy_ok, slice_ok;\n memset(&b, 0, sizeof(b));\n if (scanf("%511s %zu %zu %zu", text, &limit, &block, &count) != 4) return 71;\n parsed = rr_plan_parse(&plan, text);\n b.plan = &plan;\n if (count > RR_MAX_RESOURCES) return 71;\n for (index = 0; index < count; ++index) {\n char encoded[RR_DATA_SIZE * 2 + 1];\n rr_buffer input;\n source *s = &b.items[index];\n if (scanf("%d %d %zu %2048s", &s->available, &s->failure,\n &s->chunk, encoded) != 4) return 71;\n if (!strcmp(encoded,"-")) encoded[0] = \'\\0\';\n if (rr_buffer_from_hex(&input, encoded, RR_DATA_SIZE) != 0) return 71;\n s->length = input.size;\n memcpy(s->data, input.data, input.size);\n s->handle = -1;\n }\n if (parsed == 0) rr_plan_format(&plan, canonical, sizeof(canonical));\n rc = rr_read_plan(&plan, &io, limit, block, &result);\n if (rr_buffer_hex(&result.content, hex, sizeof(hex)) != 0) return 71;\n copy_ok = rr_buffer_copy(&copied, &result.content) == 0 &&\n rr_buffer_equal(&copied, &result.content);\n slice_ok = rr_buffer_slice(&result.content, result.content.size / 2,\n result.content.size - result.content.size / 2, &slice) == 0;\n for (index = 0; index < count; ++index) {\n if (b.items[index].handle >= 0 && fcntl(b.items[index].handle,F_GETFD) >= 0)\n ++live;\n }\n printf("{\\"parsed\\":%d,\\"canonical\\":\\"%s\\",\\"rc\\":%d,\\"status\\":%d,"\n "\\"status_name\\":\\"%s\\",\\"selected\\":%d,\\"attempts\\":%zu,"\n "\\"opened\\":%zu,\\"reads\\":%zu,\\"closed\\":%zu,\\"failures\\":%zu,"\n "\\"size\\":%zu,\\"checksum\\":%u,\\"hex\\":\\"%s\\",\\"live\\":%d,"\n "\\"copy_ok\\":%d,\\"slice_ok\\":%d,\\"slice_checksum\\":%u,\\"events\\":[",\n parsed,canonical,rc,(int)result.status,rr_status_name(result.status),\n result.selected,result.attempts,result.opened,result.reads,result.closed,\n result.failures,result.content.size,result.content.checksum,hex,live,\n copy_ok,slice_ok,slice.checksum);\n for (index = 0; index < b.count; ++index) {\n int *r = b.events[index];\n printf("%s[%d,%d,%d,%d]",index ? "," : "",r[0],r[1],r[2],r[3]);\n }\n puts("]}");\n for (index = 0; index < count; ++index) {\n if (b.items[index].handle >= 0) close(b.items[index].handle);\n }\n }\n return 0;\n}\n', 'Makefile': 'CC = gcc\nCFLAGS = -std=c11 -O2 -Wall -Wextra -Werror\nSOURCES = catalog.c buffer.c reader.c\nHEADERS = rr.h\n.PHONY: all clean demo demo-failure\nall: reader-observer\nreader-observer: $(SOURCES) $(HEADERS) observer.c\n\t$(CC) $(CFLAGS) -I. $(SOURCES) observer.c -o $@\ndemo: reader-observer\n\t./reader-observer < examples/default.txt\ndemo-failure: reader-observer\n\t./reader-observer < examples/read-failure.txt\nclean:\n\trm -f reader-observer\n'}
|
|
|
|
def checksum(data):
|
|
value=2166136261
|
|
for byte in data:
|
|
value=((value^byte)*16777619)&0xffffffff
|
|
return value
|
|
|
|
def cases():
|
|
rng=random.Random(440735)
|
|
def src(data=b'', available=1, failure=-1, chunk=7):
|
|
return dict(data=data,available=available,failure=failure,chunk=chunk)
|
|
rows=[
|
|
dict(plan='primary,backup',limit=64,block=7,sources=[src(b'Read the config',chunk=3),src(b'backup')]),
|
|
dict(plan='empty',limit=64,block=8,sources=[src()]),
|
|
dict(plan='primary,backup',limit=64,block=7,sources=[src(b'bad-primary',failure=3,chunk=3),src(b'good-backup',chunk=4)]),
|
|
dict(plan='failed',limit=64,block=8,sources=[src(failure=0)]),
|
|
dict(plan='gone,empty',limit=0,block=1,sources=[src(available=0),src()]),
|
|
dict(plan='p,q',limit=0,block=1,sources=[src(b'x'),src(b'ignored')]),
|
|
dict(plan='p,q',limit=8,block=2,sources=[src(b'partial',failure=3,chunk=2),src(available=0)]),
|
|
dict(plan='p,q',limit=8,block=2,sources=[src(available=0),src(available=0)]),
|
|
]
|
|
for n in [1,2,63,64,65,1023,1024]:
|
|
data=bytes(rng.randrange(256) for _ in range(n))
|
|
for limit in sorted({n,max(0,n-1)}):
|
|
rows.append(dict(plan='single',limit=limit,block=64,sources=[src(data,chunk=13)]))
|
|
for i in range(42):
|
|
count=rng.randrange(1,9); sources=[]
|
|
for j in range(count):
|
|
n=rng.randrange(0,90)
|
|
sources.append(src(bytes(rng.randrange(256) for _ in range(n)),
|
|
available=int(rng.random()>.22), failure=rng.choice([-1,-1,0,n//2,n]),chunk=rng.randrange(1,65)))
|
|
rows.append(dict(plan=','.join('R%d_%d'%(i,j) for j in range(count)),
|
|
limit=rng.choice([0,1,17,64,128,1024]),block=rng.randrange(1,65),sources=sources))
|
|
for name in ['bad,,gap','duplicate,duplicate','1bad','too_long_'+('a'*24),'trailing,','a,b,c,d,e,f,g,h,i','bad/name','a b']:
|
|
# Observer's whitespace-delimited CSV token cannot carry spaces.
|
|
if ' ' not in name:
|
|
rows.append(dict(plan=name,limit=64,block=8,sources=[]))
|
|
for limit,block in [(1025,8),(64,0),(64,65)]:
|
|
rows.append(dict(plan='valid',limit=limit,block=block,sources=[src(b'x')]))
|
|
return rows
|
|
|
|
def encode(rows):
|
|
lines=[str(len(rows))]
|
|
for row in rows:
|
|
lines.append('%s %d %d %d'%(row['plan'],row['limit'],row['block'],len(row['sources'])))
|
|
for source in row['sources']:
|
|
lines.append('%d %d %d %s'%(source['available'],source['failure'],source['chunk'],source['data'].hex() or '-'))
|
|
return ('\n'.join(lines)+'\n').encode('ascii')
|
|
|
|
def model(row):
|
|
names=row['plan'].split(',')
|
|
import re
|
|
valid=1<=len(names)<=8 and len(set(names))==len(names) and all(re.fullmatch('[A-Za-z][A-Za-z0-9_-]{0,23}',name) for name in names)
|
|
parsed=0 if valid else -1
|
|
answer=dict(parsed=parsed,canonical=row['plan'] if valid else '',rc=0,status=1,
|
|
status_name='unavailable',selected=-1,attempts=0,opened=0,reads=0,closed=0,
|
|
failures=0,size=0,checksum=2166136261,hex='',live=0,copy_ok=1,slice_ok=1,
|
|
slice_checksum=2166136261,events=[])
|
|
events=answer['events']; chosen=b''
|
|
if not valid or not 0<=row['limit']<=1024 or not 1<=row['block']<=64:
|
|
answer.update(status=3,status_name='bad-plan');return answer
|
|
for index,source in enumerate(row['sources']):
|
|
answer['attempts']+=1
|
|
token='handle%d'%index
|
|
events.append([1,index,token if source['available'] else -1,0])
|
|
if not source['available']:
|
|
answer['failures']+=1;continue
|
|
answer['opened']+=1
|
|
offset=0;content=bytearray();failed=False;overflow=False
|
|
while True:
|
|
failure=source['failure']
|
|
if failure>=0 and offset>=failure:
|
|
returned=-5
|
|
else:
|
|
returned=min(row['block'],source['chunk'],len(source['data'])-offset)
|
|
if failure>=0: returned=min(returned,failure-offset)
|
|
events.append([2,token,row['block'],returned]);answer['reads']+=1
|
|
if returned<0:failed=True;break
|
|
if returned==0:break
|
|
if len(content)+returned>row['limit']:overflow=True;break
|
|
content.extend(source['data'][offset:offset+returned]);offset+=returned
|
|
events.append([3,token,0,0]);answer['closed']+=1
|
|
if failed:answer['failures']+=1;continue
|
|
if overflow:
|
|
answer.update(status=2,status_name='too-large');break
|
|
chosen=bytes(content)
|
|
answer.update(status=0,status_name='ok',selected=index)
|
|
break
|
|
answer.update(size=len(chosen),checksum=checksum(chosen),hex=chosen.hex(),
|
|
slice_checksum=checksum(chosen[len(chosen)//2:]))
|
|
return answer
|
|
|
|
def equivalent(actual,expected):
|
|
if type(actual) is not type(expected):return False
|
|
if type(expected) is dict:
|
|
return actual.keys()==expected.keys() and all(equivalent(actual[k],v) for k,v in expected.items())
|
|
if type(expected) is list:
|
|
return len(actual)==len(expected) and all(equivalent(a,b) for a,b in zip(actual,expected))
|
|
return actual==expected
|
|
|
|
def compare(actual,expected):
|
|
if type(actual) is not dict or type(actual.get('events')) is not list:return False
|
|
observed=actual['events']; wanted=expected['events']
|
|
if len(observed)!=len(wanted):return False
|
|
handles={}
|
|
normalized=[]
|
|
for got,want in zip(observed,wanted):
|
|
if type(got) is not list or len(got)!=4 or any(type(n) is not int for n in got):return False
|
|
row=list(want)
|
|
if want[0]==1 and type(want[2]) is str:
|
|
if got[2]<0:return False
|
|
handles[want[2]]=got[2]
|
|
for i,v in enumerate(row):
|
|
if type(v) is str:
|
|
if v not in handles:return False
|
|
row[i]=handles[v]
|
|
normalized.append(row)
|
|
expected=dict(expected,events=normalized)
|
|
return equivalent(actual,expected)
|
|
|
|
def parse_observations(raw, count):
|
|
text=raw.decode('utf-8')
|
|
whitespace=' \t\r\n'
|
|
def reject_constant(token):
|
|
raise ValueError('nonfinite JSON number')
|
|
def finite_float(token):
|
|
value=float(token)
|
|
if not math.isfinite(value):raise ValueError('nonfinite JSON number')
|
|
return value
|
|
decoder=json.JSONDecoder(parse_constant=reject_constant,parse_float=finite_float)
|
|
observations=[]; position=0
|
|
for _ in range(count):
|
|
while position<len(text) and text[position] in whitespace:position+=1
|
|
if position==len(text):raise ValueError('raw observation object count')
|
|
value,position=decoder.raw_decode(text,position)
|
|
if type(value) is not dict:raise ValueError('raw observation must be an object')
|
|
if position<len(text) and text[position] not in whitespace:
|
|
raise ValueError('raw observations require JSON whitespace separation')
|
|
observations.append(value)
|
|
while position<len(text) and text[position] in whitespace:position+=1
|
|
if position!=len(text):raise ValueError('extra raw observation content')
|
|
return observations
|
|
|
|
def main():
|
|
evidence=[]; failures=[]; scratch=None; business_ok=False; cleanup_ok=False; absence_ok=False
|
|
try:
|
|
scratch=Path(tempfile.mkdtemp(prefix='rrf-'))
|
|
build=scratch/'build'
|
|
build.mkdir(mode=0o755)
|
|
build.chmod(0o777)
|
|
candidate=Path(sys.argv[1])
|
|
for name in FILES:
|
|
(build/name).write_bytes((candidate/name).read_bytes())
|
|
for name,body in SUPPORT.items():
|
|
(build/name).write_text(body)
|
|
for item in build.iterdir():item.chmod(0o644)
|
|
rows=cases(); payload=encode(rows)
|
|
compiler=['gcc','-std=c11','-O2','-Wall','-Wextra','-Werror','-I.',
|
|
'catalog.c','buffer.c','reader.c','observer.c','-o','reader']
|
|
execute_stage('compiler',compiler,build,COMPILE_SECONDS,b'',evidence)
|
|
out,err=execute_stage('candidate',[str(build/'reader')],build,CANDIDATE_SECONDS,payload,evidence)
|
|
observations=parse_observations(out,len(rows))
|
|
for index,(actual,row) in enumerate(zip(observations,rows)):
|
|
if not compare(actual,model(row)):
|
|
raise ValueError('first raw observation differs at row %d'%index)
|
|
business_ok=True
|
|
except BaseException as error:
|
|
failures.append(['business',type(error).__name__,str(error)])
|
|
finally:
|
|
if scratch is not None:
|
|
try:
|
|
execute_stage('cleanup',[sys.executable,'-I','-c',CLEANUP,str(scratch)],Path('/tmp'),CLEANUP_SECONDS,b'',evidence)
|
|
cleanup_ok=True
|
|
except BaseException as error:
|
|
failures.append(['cleanup',type(error).__name__,str(error)])
|
|
try:
|
|
execute_stage('absence',[sys.executable,'-I','-c',ABSENCE,str(scratch)],Path('/tmp'),ABSENCE_SECONDS,b'',evidence)
|
|
absence_ok=True
|
|
except BaseException as error:
|
|
failures.append(['absence',type(error).__name__,str(error)])
|
|
# Compact complete metadata, actual wire/IO length+SHA only. Original wire and
|
|
# original IO bodies are not persisted by this checker; no claim otherwise.
|
|
record=dict(stages=evidence,failures=failures,business_ok=business_ok,
|
|
cleanup_ok=cleanup_ok,absence_ok=absence_ok,evidence_rows=len(cases()))
|
|
marker=base64.b64encode(bz2.compress(json.dumps(record,separators=(',',':')).encode())).decode()
|
|
sys.stderr.write('RRF_EVIDENCE '+marker+'\n')
|
|
ok=business_ok and cleanup_ok and absence_ok and not failures
|
|
print('PASS' if ok else 'FAIL '+repr(failures)[:220])
|
|
return 0 if ok else 1
|
|
|
|
if __name__=='__main__':sys.exit(main())
|