import base64 import bz2 import hashlib import json import math import re import os from pathlib import Path import random import subprocess import sys import tempfile import time RECOVERY_SECONDS = 3 RECORD_WIRE = False SUPERVISOR = r''' import base64,bz2,ctypes,hashlib,json,os,selectors,signal,subprocess,sys,time cfg=json.loads(sys.argv[1]); start=time.monotonic(); errors=[]; tracked={}; signals={}; waits=[] leader=None; leader_start=None; output=bytearray(); diagnostic=bytearray(); timed_out=False; drained=False reaped=False; leader_rc=None; remaining=[]; pg_absent=False selector=selectors.DefaultSelector(); input_offset=0; payload=b'' def fault(op, error): errors.append([op,type(error).__name__,str(error)]) def exact(pid): try: with open('/proc/%d/stat'%pid,'rb') as stream: data=stream.read() words=data[data.rfind(b')')+2:].split() return {'pid':pid,'state':words[0].decode('ascii'),'ppid':int(words[1]),'pg':int(words[2]),'start':int(words[19])} except (FileNotFoundError,ProcessLookupError): return None def table_now(): result={} for name in os.listdir('/proc'): if name.isascii() and name.isdigit(): value=exact(int(name)) if value is not None: result[value['pid']]=value return result def closure(table, supervisor, remembered): roots={supervisor} for pid,old in remembered.items(): current=table.get(pid) if current and current['start']==old['start'] and current['state'] not in ('Z','X'): roots.add(pid) found={} changed=True while changed: changed=False for pid,current in table.items(): if pid!=supervisor and pid not in found and (current['ppid'] in roots or pid in roots): found[pid]=current; roots.add(pid); changed=True return found def discover(): table=table_now(); found=closure(table,os.getpid(),tracked) for pid,current in found.items(): if pid not in tracked or tracked[pid]['start']!=current['start']: tracked[pid]=dict(current) return table,found def send_fresh(pid, token): current=exact(pid) if current is None or current['start']!=token['start'] or current['state'] in ('Z','X'): return os.kill(pid,signal.SIGKILL) signals[(pid,token['start'])]=signals.get((pid,token['start']),0)+1 def own_wait(): global reaped,leader_rc if leader is not None: value=leader.poll() if value is not None: leader_rc=leader.wait(timeout=0) reaped=True def adopted_wait(): if leader is not None and not reaped: return while True: try: pid,status=os.waitpid(-1,os.WNOHANG) except ChildProcessError: break if pid==0: break waits.append([pid,None,os.waitstatus_to_exitcode(status)]) def pump(timeout): global input_offset,drained for key,events in selector.select(timeout): stream=key.fileobj if key.data=='stdin': try: if input_offset=cfg['deadline']: timed_out=True; break except BaseException as error: fault('execute',error) finally: recovery_end=time.monotonic()+cfg['recovery'] while True: try: own_wait() except BaseException as error: fault('own_wait',error) found={} try: _,found=discover() except BaseException as error: fault('discover',error) for pid,token in list(found.items()): try: send_fresh(pid,token) except (FileNotFoundError,ProcessLookupError): pass except BaseException as error: fault('signal',error) try: own_wait() except BaseException as error: fault('own_wait',error) try: adopted_wait() except BaseException as error: fault('adopted_wait',error) if leader is not None and not drained: try: pump(0.01) except BaseException as error: fault('drain',error) try: table,found=discover() remaining=[[p,v['start'],v['state']] for p,v in found.items()] pg_absent=leader is not None and not any(v['pg']==leader.pid for v in table.values()) except BaseException as error: fault('absence',error); remaining=[['unknown']]; pg_absent=False if reaped and drained and not remaining and pg_absent: break if time.monotonic()>=recovery_end: fault('recovery',RuntimeError('deadline with unconfirmed closure')); break time.sleep(0.01) output=bytes(output); diagnostic=bytes(diagnostic) meta={'phase':cfg['phase'],'nonce':cfg['nonce'],'argv':cfg['argv'], 'uid':os.getuid(),'gid':os.getgid(),'groups':os.getgroups(), 'supervisor_pid':os.getpid(),'supervisor_start':supervisor_start, 'leader_pid':None if leader is None else leader.pid,'leader_start':leader_start, 'leader_rc':leader_rc,'leader_reaped':reaped,'timeout':timed_out, 'controlled_rc':124 if timed_out else leader_rc,'elapsed':time.monotonic()-start, 'deadline':cfg['deadline'],'recovery':cfg['recovery'], 'pg_absent':pg_absent,'remaining':remaining,'exceptions':errors, 'descendants':[[p,t['start'],signals.get((p,t['start']),0),None] for p,t in tracked.items() if leader is None or p!=leader.pid], 'adopted':waits, 'stdout_len':len(output),'stdout_sha256':hashlib.sha256(output).hexdigest(), 'stderr_len':len(diagnostic),'stderr_sha256':hashlib.sha256(diagnostic).hexdigest()} wire={'meta':meta,'stdout':base64.b64encode(bz2.compress(output)).decode('ascii'), 'stderr':base64.b64encode(bz2.compress(diagnostic)).decode('ascii')} sys.stdout.write(json.dumps(wire,separators=(',',':'))+'\n') ''' CLEANUP = "import shutil,sys; shutil.rmtree(sys.argv[1])" ABSENCE = "import os,sys\ntry: os.lstat(sys.argv[1])\nexcept FileNotFoundError: sys.exit(0)\nsys.exit(1)" def identity_int(value, minimum=0): return type(value) is int and value >= minimum def validate_meta(meta, config, out, err): required = {'phase','nonce','argv','uid','gid','groups','supervisor_pid','supervisor_start', 'leader_pid','leader_start','leader_rc','leader_reaped','timeout','controlled_rc', 'elapsed','deadline','recovery','pg_absent','remaining','exceptions','descendants', 'stdout_len','stdout_sha256','stderr_len','stderr_sha256','adopted'} if type(meta) is not dict or not required <= meta.keys(): raise ValueError('missing stage metadata') if meta['phase'] != config['phase'] or meta['nonce'] != config['nonce'] or meta['argv'] != config['argv']: raise ValueError('stage binding conflict') for name in ('uid','gid','supervisor_pid','supervisor_start','leader_pid','leader_start'): if not identity_int(meta[name], 1 if name not in ('uid','gid') else 0): raise ValueError('invalid identity '+name) if meta['supervisor_pid'] == meta['leader_pid']: raise ValueError('supervisor and leader identical') isolated = os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534' expected_identity = (65534,65534,[]) if isolated else (os.getuid(),os.getgid(),os.getgroups()) if type(meta['groups']) is not list or any(type(x) is not int or x < 0 for x in meta['groups']): raise ValueError('invalid groups') if (meta['uid'],meta['gid'],meta['groups']) != expected_identity: raise ValueError('execution identity conflict') if type(meta['deadline']) is not int or meta['deadline'] != config['deadline']: raise ValueError('deadline conflict') if type(meta['recovery']) is not int or meta['recovery'] != RECOVERY_SECONDS: raise ValueError('recovery conflict') elapsed = meta['elapsed'] if type(elapsed) not in (int,float) or not math.isfinite(elapsed) or not 0 <= elapsed <= config['deadline']+RECOVERY_SECONDS+2: raise ValueError('invalid elapsed') for name in ('timeout','leader_reaped','pg_absent'): if type(meta[name]) is not bool: raise ValueError('invalid boolean '+name) for name in ('leader_rc','controlled_rc'): if type(meta[name]) is not int: raise ValueError('invalid return code '+name) if meta['controlled_rc'] != (124 if meta['timeout'] else meta['leader_rc']): raise ValueError('return code conflict') for name in ('remaining','exceptions','descendants','adopted'): if type(meta[name]) is not list: raise ValueError('invalid list '+name) for row in meta['descendants']: if type(row) is not list or len(row) != 4 or not all(identity_int(x,1) for x in row[:2]) or not identity_int(row[2]) or (row[3] is not None and type(row[3]) is not int): raise ValueError('invalid descendant record') for row in meta['adopted']: if type(row) is not list or len(row) != 3 or not identity_int(row[0],1) or row[1] is not None or type(row[2]) is not int: raise ValueError('invalid adopted wait record') for name,data in (('stdout',out),('stderr',err)): if not identity_int(meta[name+'_len']) or meta[name+'_len'] != len(data): raise ValueError('invalid IO length') if type(meta[name+'_sha256']) is not str or meta[name+'_sha256'] != hashlib.sha256(data).hexdigest(): raise ValueError('invalid IO digest') if meta['exceptions'] or meta['remaining'] or not meta['leader_reaped'] or not meta['pg_absent']: raise ValueError('stage recovery or exception failure') if meta['timeout'] or meta['leader_rc'] != 0 or meta['controlled_rc'] != 0: raise ValueError('controlled timeout' if meta['timeout'] else 'leader nonzero exit') def execute_stage(phase, argv, cwd, deadline, payload, evidence): nonce = os.urandom(8).hex() config = dict(phase=phase,argv=argv,cwd=str(cwd),deadline=deadline,recovery=RECOVERY_SECONDS,nonce=nonce) wrapper_argv = [sys.executable,'-I','-c',SUPERVISOR,json.dumps(config,separators=(',',':'))] raw = subprocess.run(wrapper_argv,input=payload,capture_output=True,timeout=deadline+RECOVERY_SECONDS+5) # Preserve actual wrapper facts before touching or trusting its wire protocol. entry = dict(phase=phase,nonce=nonce,wrapper_rc=raw.returncode, wire_len=len(raw.stdout),wire_sha256=hashlib.sha256(raw.stdout).hexdigest(), wrapper_stderr_len=len(raw.stderr),wrapper_stderr_sha256=hashlib.sha256(raw.stderr).hexdigest()) if RECORD_WIRE: entry['wire_ascii'] = raw.stdout.decode('ascii') entry['wrapper_stderr_bz2'] = base64.b64encode(bz2.compress(raw.stderr)).decode('ascii') evidence.append(entry) if raw.returncode != 0: raise ValueError(phase+' supervisor nonzero') wire = json.loads(raw.stdout) out = bz2.decompress(base64.b64decode(wire['stdout'],validate=True)) err = bz2.decompress(base64.b64decode(wire['stderr'],validate=True)) meta = wire['meta'] if not RECORD_WIRE: entry['meta'] = meta validate_meta(meta,config,out,err) return out,err OBSERVER = '#include "fields.h"\n#include \n#include \n#include \n#include \n\nstatic int next_command(void)\n{\n int character;\n do {\n character = getchar();\n } while (character == \' \' || character == \'\\t\' || character == \'\\r\' || character == \'\\n\');\n return character;\n}\n\nstatic int number(uint32_t *result)\n{\n int character = next_command();\n uint32_t value = 0u;\n int digits = 0;\n int bad = 0;\n if (character == \'+\') {\n character = getchar();\n }\n while (character != EOF && character != \' \' && character != \'\\t\' && character != \'\\r\' && character != \'\\n\') {\n if (character < \'0\' || character > \'9\') {\n bad = 1;\n } else {\n uint32_t digit = (uint32_t)(character - \'0\');\n digits = 1;\n if (value > (UINT32_MAX - digit) / 10u) {\n bad = 1;\n } else {\n value = value * 10u + digit;\n }\n }\n character = getchar();\n }\n *result = value;\n return digits && !bad;\n}\n\nstatic void emit(const struct pf_bank *bank, int status, const struct pf_info *info)\n{\n printf("%d %" PRIu32 " %" PRIu32 " %" PRIu32 " %" PRIu32 " %" PRIu32 " %" PRIu32,\n status, bank->count, bank->word, bank->epoch,\n bank->writes, bank->imports, bank->rejects);\n printf(" %" PRIu32 " %" PRIu32 " %" PRIu32 " %" PRIu32,\n pf_occupied(bank), pf_fingerprint(bank),\n pf_minimum_word(bank), pf_maximum_word(bank));\n printf(" %" PRIu32 " %" PRIu32 " %" PRIu32 " %" PRIu32,\n pf_value_sum(bank), pf_value_weight(bank),\n pf_layout_span(bank), pf_free_bits(bank));\n printf(" %" PRIu32 " %" PRIu32 " %" PRIu32 " %" PRIu32,\n info->present, info->width, info->offset, info->scale);\n printf(" %" PRIu32 " %" PRIu32 " %" PRIu32 " %" PRIu32 "\\n",\n info->bias, info->value, info->limit, info->token);\n}\n\nint main(void)\n{\n struct pf_bank bank;\n int command;\n pf_init(&bank);\n while ((command = next_command()) != EOF) {\n uint32_t args[5] = {0u, 0u, 0u, 0u, 0u};\n struct pf_info info;\n int count;\n int index;\n int status = PF_OK;\n memset(&info, 0, sizeof(info));\n switch (command) {\n case \'A\':\n case \'R\':\n count = 5;\n break;\n case \'W\':\n count = 2;\n break;\n case \'D\':\n case \'I\':\n case \'Q\':\n count = 1;\n break;\n case \'Z\':\n case \'C\':\n case \'S\':\n count = 0;\n break;\n default:\n return 64;\n }\n for (index = 0; index < count; ++index) {\n if (!number(&args[index])) {\n return 64;\n }\n }\n switch (command) {\n case \'A\':\n status = pf_add(&bank, args[0], args[1], args[2], args[3], args[4]);\n break;\n case \'R\':\n status = pf_replace(&bank, args[0], args[1], args[2], args[3], args[4]);\n break;\n case \'W\':\n status = pf_write(&bank, args[0], args[1]);\n break;\n case \'D\':\n status = pf_drop(&bank, args[0]);\n break;\n case \'I\':\n status = pf_import(&bank, args[0]);\n break;\n case \'Z\':\n status = pf_zero(&bank);\n break;\n case \'C\':\n status = pf_clear(&bank);\n break;\n case \'Q\':\n pf_info(&bank, args[0], &info);\n status = info.present ? PF_OK : PF_NOT_FOUND;\n break;\n case \'S\':\n break;\n }\n emit(&bank, status, &info);\n }\n return 0;\n}\n' CANDIDATES = ['include/fields.h', 'src/layout.c', 'src/codec.c', 'src/bank.c', 'src/metrics.c', 'Makefile'] import random U32=(1<<32)-1 def shape(f): ident,width,offset,scale,bias,value=f if not 1<=ident<=255: return 1 if not 1<=width<=16 or not 0<=offset<=31 or width+offset>32: return 2 if not 0<=scale<=5 or not 0<=bias<=5 or bias>2**width-1: return 2 return 0 def ceiling(f): return ((2**f[1]-1)-f[4])//2**f[3] def occupied(fields): return sum((2**f[1]-1)*2**f[2] for f in fields) def word_for(fields): return sum((f[5]*2**f[3]+f[4])*2**f[2] for f in fields) def oracle(commands): fields=[]; word=epoch=writes=imports=rejects=0; result=[] for command in commands: op,*args=command; status=0; info=[0]*8 trial=[f[:] for f in fields] index=next((i for i,f in enumerate(fields) if args and f[0]==args[0]),None) if op in ('A','R'): fresh=args+[0] if op=='R' and index is None: status=4 else: status=shape(fresh) if not status and op=='A': if index is not None: status=1 elif len(fields)==8: status=7 else: trial.append(fresh) elif not status: fresh[5]=fields[index][5]; trial[index]=fresh if not status: seen=0 for f in trial: bits=(2**f[1]-1)*2**f[2] if seen&bits: status=3; break seen|=bits if not status and any(f[5]>ceiling(f) for f in trial): status=5 if not status: fields=trial; word=word_for(fields); epoch=(epoch+1)&U32 elif op=='D': if index is None: status=4 else: fields=trial[:index]+trial[index+1:]; word=word_for(fields); epoch=(epoch+1)&U32 elif op=='W': if index is None: status=4 elif args[1]>ceiling(fields[index]): status=5 else: trial[index][5]=args[1]; fields=trial; word=word_for(fields); writes=(writes+1)&U32 elif op=='I': imported=args[0] if imported&~occupied(fields): status=6 else: for f in trial: token=(imported//2**f[2])%(2**f[1]) difference=token-f[4] if difference<0 or difference%2**f[3]: status=6; break f[5]=difference//2**f[3] if not status: fields=trial; word=imported; imports=(imports+1)&U32 elif op=='Z': for f in trial: f[5]=0 fields=trial; word=word_for(fields); writes=(writes+1)&U32 elif op=='C': fields=[]; word=0; epoch=(epoch+1)&U32 elif op=='Q': if index is None: status=4 else: f=fields[index] info=[1,*f[1:5],f[5],ceiling(f),(word//2**f[2])%(2**f[1])] elif op!='S': raise ValueError(op) if status and op not in ('Q','S'): rejects=(rejects+1)&U32 fingerprint=5381 for number in [len(fields)]+[n for f in fields for n in f[:5]]: fingerprint=(33*fingerprint+number)&U32 minimum=sum(f[4]*2**f[2] for f in fields) maximum=sum((ceiling(f)*2**f[3]+f[4])*2**f[2] for f in fields) result.append([status,len(fields),word,epoch,writes,imports,rejects,occupied(fields),fingerprint,minimum,maximum, sum(f[5] for f in fields)&U32,sum(f[0]*f[5] for f in fields)&U32, max((f[1]+f[2] for f in fields),default=0),32-sum(f[1] for f in fields),*info]) return result def cases(): commands=[('S',),('I',0),('I',1),('Q',1),('Z',),('D',0),('W',0,1)] commands += [('A',7,8,0,2,1),('W',7,9),('Q',7),('A',9,8,16,1,0),('W',9,17),('S',),('C',)] commands += [('A',7,8,0,2,0),('W',7,9),('A',9,8,16,1,0),('W',9,17),('R',7,8,16,2,0),('Q',7),('S',),('R',7,7,1,1,0),('Q',7),('C',)] commands += [('A',255,1,31,0,0),('W',255,1),('I',2147483648),('Q',255),('R',255,2,31,0,0),('A',1,1,0,0,1),('Q',1),('I',2147483649),('D',255),('S',),('C',)] for ident in range(1,9): commands += [('A',ident,1,ident-1,0,0),('W',ident,1)] commands += [('A',9,1,20,0,0),('A',1,1,20,0,0),('A',0,1,20,0,0),('R',8,1,0,0,0),('S',),('C',)] commands += [('A',1,16,0,0,0),('A',2,16,16,0,0),('W',1,65535),('W',2,65535),('I',4294967295),('Q',1),('Q',2),('W',1,65536),('R',1,15,0,0,0),('S',),('C',)] for scale in range(6): for bias in range(6): limit=(255-bias)//2**scale commands += [('A',41,8,8,scale,bias),('W',41,limit),('Q',41),('W',41,limit+1),('I',(limit*2**scale+bias)*256),('I',0),('I',(bias+1)*256),('R',41,1,8,scale,bias),('S',),('Z',),('D',41),('Q',41)] rng=random.Random(0x51F1E1D) for batch in range(8): commands.append(('C',)) for ident in range(1,5): commands += [('A',ident,8,(ident-1)*8,rng.randrange(4),rng.randrange(6)),('W',ident,rng.randrange(10))] for _ in range(56): op=rng.choice(('A','R','W','I','Q','D','S','Z')) if op in ('A','R'): command=(op,rng.randrange(0,7),rng.randrange(0,18),rng.randrange(0,33),rng.randrange(0,7),rng.randrange(0,7)) elif op=='W': command=(op,rng.randrange(0,7),rng.choice((0,1,7,31,255,65535,4294967295))) elif op=='I': command=(op,rng.randrange(2**32)) elif op in ('Q','D'): command=(op,rng.randrange(0,7)) else: command=(op,) commands.append(command) commands += [('C',),('A',1,1,0,0,0),('W',1,0),('Q',1),('S',)] return commands def payload_for(commands): rows=[] for index,command in enumerate(commands): parts=[command[0]] for number in command[1:]: token=str(number) if index%17==0: token='+'+'0'*3+token parts.append(token) rows.append('\t'.join(parts)) rows[-3]='W 1 +'+'0'*5000 return ('\n'.join(rows)+'\n').encode('ascii') def parse_observations(data, expected): if not data.endswith(b'\n'): raise ValueError('stdout does not end in LF') lines=data[:-1].split(b'\n') if len(lines)!=len(expected): raise ValueError('stdout row count %d expected %d'%(len(lines),len(expected))) for index,(line,wanted) in enumerate(zip(lines,expected)): tokens=re.split(rb'[ \t\r]+',line.strip(b' \t\r')) if len(tokens)!=23: raise ValueError('row %d has %d columns'%(index,len(tokens))) observed=[] for token in tokens: if re.fullmatch(rb'[+-]?[0-9]+',token) is None: raise ValueError('invalid decimal output at row %d'%index) negative=token.startswith(b'-') digits=token.lstrip(b'+-').lstrip(b'0') or b'0' if len(digits)>10: raise ValueError('output magnitude exceeds public uint32 at row %d'%index) value=int(digits) observed.append(-value if negative else value) if observed!=wanted: column=next(i for i,(a,b) in enumerate(zip(observed,wanted)) if a!=b) raise ValueError('row %d column %d actual %d expected %d'%(index,column,observed[column],wanted[column])) def main(): evidence=[]; errors=[]; scratch=None; success=False; cleanup_ok=False; absence_ok=False try: commands=cases(); expected=oracle(commands); payload=payload_for(commands) source=Path(sys.argv[1]) material={name:(source/name).read_bytes() for name in CANDIDATES} scratch=Path(tempfile.mkdtemp(prefix='shift-fields-')) work=scratch/'work'; work.mkdir(mode=0o777); os.chmod(work,0o777) for name,data in {**material,'tools/observe.c':OBSERVER.encode('utf-8')}.items(): target=work/name; target.parent.mkdir(parents=True,exist_ok=True); target.parent.chmod(0o777) target.write_bytes(data); target.chmod(0o644) # All candidate tree access ends here, before the first child. compiler=['gcc','-std=c11','-O2','-Wall','-Wextra','-Werror','-Iinclude', 'src/layout.c','src/codec.c','src/bank.c','src/metrics.c','tools/observe.c','-o','field-bank'] out,err=execute_stage('compiler',compiler,work,20,b'',evidence) out,err=execute_stage('candidate',[str(work/'field-bank')],work,10,payload,evidence) parse_observations(out,expected) success=True except BaseException as error: errors.append(['main',type(error).__name__,str(error)]) finally: if scratch is not None: try: execute_stage('cleanup',[sys.executable,'-I','-c',CLEANUP,str(scratch)],Path('/tmp'),5,b'',evidence) cleanup_ok=True except BaseException as error: errors.append(['cleanup',type(error).__name__,str(error)]) try: execute_stage('absence',[sys.executable,'-I','-c',ABSENCE,str(scratch)],Path('/tmp'),5,b'',evidence) absence_ok=True except BaseException as error: errors.append(['absence',type(error).__name__,str(error)]) details=dict(evidence=evidence,errors=errors,scratch=str(scratch),success=success, cleanup_ok=cleanup_ok,absence_ok=absence_ok, expected_rows=len(expected) if 'expected' in locals() else None) compressed=base64.b64encode(bz2.compress(json.dumps(details,separators=(',',':')).encode())).decode('ascii') sys.stderr.write('SHIFT_EVIDENCE '+compressed+'\n') passed=success and cleanup_ok and absence_ok and not errors print('PASS' if passed else 'FAIL') return 0 if passed else 1 if __name__=='__main__': raise SystemExit(main())