One user-space repair task per eligible HWE debug-diag / compile-deploy base ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content filter. Each ships a public spec with a check-to-basis table, at least two defects with single-fix fixtures, a symptom-masking fixture and an alternative correct implementation. Claude re-checked all 320: structure, the HWE quality items and trace6 Docker calibration receipts against current digests (320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120 ad-hoc family names; the next batch maps them to the canonical list. The batch's 1,256-line progress log stays in its factory report and the 36-local branch batch15-hwe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
506 lines
27 KiB
Python
506 lines
27 KiB
Python
import base64
|
|
|
|
import bz2
|
|
|
|
import hashlib
|
|
|
|
import json
|
|
|
|
import math
|
|
|
|
import os
|
|
|
|
from pathlib import Path
|
|
|
|
import random
|
|
|
|
import subprocess
|
|
|
|
import sys
|
|
|
|
import tempfile
|
|
|
|
import time
|
|
|
|
RECOVERY_SECONDS = 3
|
|
|
|
RECORD_WIRE = False
|
|
|
|
SUPERVISOR = r'''
|
|
import base64,bz2,ctypes,hashlib,json,os,signal,subprocess,sys,time
|
|
cfg=json.loads(sys.argv[1]); start=time.monotonic(); errors=[]; tracked={}; signals={}; waits={}
|
|
leader=None; leader_start=None; output=b''; diagnostic=b''; timed_out=False; drained=False
|
|
reaped=False; leader_rc=None; remaining=[]; pg_absent=False
|
|
|
|
def fault(op, error):
|
|
errors.append([op,type(error).__name__,str(error)])
|
|
|
|
def exact(pid):
|
|
try:
|
|
with open('/proc/%d/stat'%pid,'rb') as stream: data=stream.read()
|
|
words=data[data.rfind(b')')+2:].split()
|
|
return {'pid':pid,'state':words[0].decode('ascii'),'ppid':int(words[1]),'pg':int(words[2]),'start':int(words[19])}
|
|
except (FileNotFoundError,ProcessLookupError): return None
|
|
|
|
def table_now():
|
|
result={}
|
|
for name in os.listdir('/proc'):
|
|
if name.isascii() and name.isdigit():
|
|
value=exact(int(name))
|
|
if value is not None: result[value['pid']]=value
|
|
return result
|
|
|
|
def closure(table, supervisor, remembered):
|
|
roots={supervisor}
|
|
for pid,old in remembered.items():
|
|
current=table.get(pid)
|
|
if current and current['start']==old['start'] and current['state'] not in ('Z','X'):
|
|
roots.add(pid)
|
|
found={}
|
|
changed=True
|
|
while changed:
|
|
changed=False
|
|
for pid,current in table.items():
|
|
if pid!=supervisor and pid not in found and (current['ppid'] in roots or pid in roots):
|
|
found[pid]=current; roots.add(pid); changed=True
|
|
return found
|
|
|
|
def discover():
|
|
table=table_now(); found=closure(table,os.getpid(),tracked)
|
|
for pid,current in found.items():
|
|
if pid not in tracked or tracked[pid]['start']!=current['start']:
|
|
tracked[pid]=dict(current)
|
|
return table,found
|
|
|
|
def send_fresh(pid, token):
|
|
current=exact(pid)
|
|
if current is None or current['start']!=token['start'] or current['state'] in ('Z','X'): return
|
|
os.kill(pid,signal.SIGKILL)
|
|
signals[(pid,token['start'])]=signals.get((pid,token['start']),0)+1
|
|
|
|
def own_wait():
|
|
global reaped,leader_rc
|
|
if leader is not None:
|
|
value=leader.poll()
|
|
if value is not None:
|
|
leader_rc=leader.wait(timeout=0)
|
|
reaped=True
|
|
|
|
def adopted_wait():
|
|
if leader is not None and not reaped: return
|
|
while True:
|
|
try: pid,status=os.waitpid(-1,os.WNOHANG)
|
|
except ChildProcessError: break
|
|
if pid==0: break
|
|
waits[pid]=os.waitstatus_to_exitcode(status)
|
|
|
|
supervisor_start=exact(os.getpid())['start']
|
|
try:
|
|
libc=ctypes.CDLL(None,use_errno=True)
|
|
if libc.prctl(36,1,0,0,0)!=0: raise OSError(ctypes.get_errno(),'subreaper')
|
|
leader=subprocess.Popen(cfg['argv'],cwd=cfg['cwd'],stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,stderr=subprocess.PIPE,start_new_session=True)
|
|
info=exact(leader.pid)
|
|
if info is None: raise RuntimeError('leader identity disappeared before capture')
|
|
leader_start=info['start']; tracked[leader.pid]=info
|
|
payload=sys.stdin.buffer.read()
|
|
first=True
|
|
while True:
|
|
try:
|
|
output,diagnostic=leader.communicate(input=payload if first else None,timeout=0.04)
|
|
drained=True; own_wait(); break
|
|
except subprocess.TimeoutExpired:
|
|
first=False
|
|
own_wait()
|
|
discover()
|
|
if time.monotonic()-start>=cfg['deadline']:
|
|
timed_out=True; break
|
|
except BaseException as error:
|
|
fault('execute',error)
|
|
finally:
|
|
recovery_end=time.monotonic()+cfg['recovery']
|
|
while True:
|
|
try: own_wait()
|
|
except BaseException as error: fault('own_wait',error)
|
|
found={}
|
|
try: _,found=discover()
|
|
except BaseException as error: fault('discover',error)
|
|
for pid,token in list(found.items()):
|
|
try: send_fresh(pid,token)
|
|
except (FileNotFoundError,ProcessLookupError): pass
|
|
except BaseException as error: fault('signal',error)
|
|
try: own_wait()
|
|
except BaseException as error: fault('own_wait',error)
|
|
try: adopted_wait()
|
|
except BaseException as error: fault('adopted_wait',error)
|
|
if leader is not None and not drained:
|
|
try:
|
|
output,diagnostic=leader.communicate(timeout=0.03)
|
|
drained=True; own_wait()
|
|
except subprocess.TimeoutExpired: pass
|
|
except BaseException as error: fault('drain',error)
|
|
try:
|
|
table,found=discover()
|
|
remaining=[[p,v['start'],v['state']] for p,v in found.items()]
|
|
pg_absent=leader is not None and not any(v['pg']==leader.pid for v in table.values())
|
|
except BaseException as error:
|
|
fault('absence',error); remaining=[['unknown']]; pg_absent=False
|
|
if reaped and drained and not remaining and pg_absent: break
|
|
if time.monotonic()>=recovery_end:
|
|
fault('recovery',RuntimeError('deadline with unconfirmed closure')); break
|
|
time.sleep(0.01)
|
|
|
|
meta={'phase':cfg['phase'],'nonce':cfg['nonce'],'argv':cfg['argv'],
|
|
'uid':os.getuid(),'gid':os.getgid(),'groups':os.getgroups(),
|
|
'supervisor_pid':os.getpid(),'supervisor_start':supervisor_start,
|
|
'leader_pid':None if leader is None else leader.pid,'leader_start':leader_start,
|
|
'leader_rc':leader_rc,'leader_reaped':reaped,'timeout':timed_out,
|
|
'controlled_rc':124 if timed_out else leader_rc,'elapsed':time.monotonic()-start,
|
|
'deadline':cfg['deadline'],'recovery':cfg['recovery'],
|
|
'pg_absent':pg_absent,'remaining':remaining,'exceptions':errors,
|
|
'descendants':[[p,t['start'],signals.get((p,t['start']),0)]
|
|
for p,t in tracked.items() if leader is None or p!=leader.pid],
|
|
'adopted_waits':[[p,None,rc] for p,rc in waits.items()],
|
|
'stdout_len':len(output),'stdout_sha256':hashlib.sha256(output).hexdigest(),
|
|
'stderr_len':len(diagnostic),'stderr_sha256':hashlib.sha256(diagnostic).hexdigest()}
|
|
wire={'meta':meta,'stdout':base64.b64encode(bz2.compress(output)).decode('ascii'),
|
|
'stderr':base64.b64encode(bz2.compress(diagnostic)).decode('ascii')}
|
|
sys.stdout.write(json.dumps(wire,separators=(',',':'))+'\n')
|
|
'''
|
|
|
|
CLEANUP = "import shutil,sys; shutil.rmtree(sys.argv[1])"
|
|
|
|
ABSENCE = "import os,sys\ntry: os.lstat(sys.argv[1])\nexcept FileNotFoundError: sys.exit(0)\nsys.exit(1)"
|
|
|
|
def identity_int(value, minimum=0):
|
|
return type(value) is int and value >= minimum
|
|
|
|
def validate_meta(meta, config, out, err):
|
|
required = {'phase','nonce','argv','uid','gid','groups','supervisor_pid','supervisor_start',
|
|
'leader_pid','leader_start','leader_rc','leader_reaped','timeout','controlled_rc',
|
|
'elapsed','deadline','recovery','pg_absent','remaining','exceptions','descendants',
|
|
'adopted_waits','stdout_len','stdout_sha256','stderr_len','stderr_sha256'}
|
|
if type(meta) is not dict or not required <= meta.keys():
|
|
raise ValueError('missing stage metadata')
|
|
if meta['phase'] != config['phase'] or meta['nonce'] != config['nonce'] or meta['argv'] != config['argv']:
|
|
raise ValueError('stage binding conflict')
|
|
for name in ('uid','gid','supervisor_pid','supervisor_start','leader_pid','leader_start'):
|
|
if not identity_int(meta[name], 1 if name not in ('uid','gid') else 0):
|
|
raise ValueError('invalid identity '+name)
|
|
if meta['supervisor_pid'] == meta['leader_pid']:
|
|
raise ValueError('supervisor and leader identical')
|
|
isolated = os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534'
|
|
expected_identity = (65534,65534,[]) if isolated else (os.getuid(),os.getgid(),os.getgroups())
|
|
if type(meta['groups']) is not list or any(type(x) is not int or x < 0 for x in meta['groups']):
|
|
raise ValueError('invalid groups')
|
|
if (meta['uid'],meta['gid'],meta['groups']) != expected_identity:
|
|
raise ValueError('execution identity conflict')
|
|
if type(meta['deadline']) is not int or meta['deadline'] != config['deadline']:
|
|
raise ValueError('deadline conflict')
|
|
if type(meta['recovery']) is not int or meta['recovery'] != RECOVERY_SECONDS:
|
|
raise ValueError('recovery conflict')
|
|
elapsed = meta['elapsed']
|
|
if type(elapsed) not in (int,float) or not math.isfinite(elapsed) or not 0 <= elapsed <= config['deadline']+RECOVERY_SECONDS+2:
|
|
raise ValueError('invalid elapsed')
|
|
for name in ('timeout','leader_reaped','pg_absent'):
|
|
if type(meta[name]) is not bool:
|
|
raise ValueError('invalid boolean '+name)
|
|
for name in ('leader_rc','controlled_rc'):
|
|
if type(meta[name]) is not int:
|
|
raise ValueError('invalid return code '+name)
|
|
if meta['controlled_rc'] != (124 if meta['timeout'] else meta['leader_rc']):
|
|
raise ValueError('return code conflict')
|
|
for name in ('remaining','exceptions','descendants'):
|
|
if type(meta[name]) is not list:
|
|
raise ValueError('invalid list '+name)
|
|
for row in meta['descendants']:
|
|
if type(row) is not list or len(row) != 3 or not all(identity_int(x,1) for x in row[:2]) or not identity_int(row[2]):
|
|
raise ValueError('invalid descendant record')
|
|
if type(meta['adopted_waits']) is not list:
|
|
raise ValueError('invalid adopted waits')
|
|
for row in meta['adopted_waits']:
|
|
if type(row) is not list or len(row) != 3 or not identity_int(row[0],1) or row[1] is not None or type(row[2]) is not int:
|
|
raise ValueError('invalid adopted wait identity')
|
|
for name,data in (('stdout',out),('stderr',err)):
|
|
if not identity_int(meta[name+'_len']) or meta[name+'_len'] != len(data):
|
|
raise ValueError('invalid IO length')
|
|
if type(meta[name+'_sha256']) is not str or meta[name+'_sha256'] != hashlib.sha256(data).hexdigest():
|
|
raise ValueError('invalid IO digest')
|
|
if meta['exceptions'] or meta['remaining'] or not meta['leader_reaped'] or not meta['pg_absent']:
|
|
raise ValueError('stage recovery or exception failure')
|
|
if meta['timeout'] or meta['leader_rc'] != 0 or meta['controlled_rc'] != 0:
|
|
raise ValueError('controlled timeout' if meta['timeout'] else 'leader nonzero exit')
|
|
|
|
def execute_stage(phase, argv, cwd, deadline, payload, evidence):
|
|
nonce = os.urandom(8).hex()
|
|
config = dict(phase=phase,argv=argv,cwd=str(cwd),deadline=deadline,recovery=RECOVERY_SECONDS,nonce=nonce)
|
|
wrapper_argv = [sys.executable,'-I','-c',SUPERVISOR,json.dumps(config,separators=(',',':'))]
|
|
raw = subprocess.run(wrapper_argv,input=payload,capture_output=True,timeout=deadline+RECOVERY_SECONDS+5)
|
|
# Preserve actual wrapper facts before touching or trusting its wire protocol.
|
|
entry = dict(phase=phase,nonce=nonce,wrapper_rc=raw.returncode,
|
|
wire_len=len(raw.stdout),wire_sha256=hashlib.sha256(raw.stdout).hexdigest(),
|
|
wrapper_stderr_len=len(raw.stderr),wrapper_stderr_sha256=hashlib.sha256(raw.stderr).hexdigest())
|
|
if RECORD_WIRE:
|
|
entry['wire_ascii'] = raw.stdout.decode('ascii')
|
|
entry['wrapper_stderr_bz2'] = base64.b64encode(bz2.compress(raw.stderr)).decode('ascii')
|
|
evidence.append(entry)
|
|
if raw.returncode != 0:
|
|
raise ValueError(phase+' supervisor nonzero')
|
|
wire = json.loads(raw.stdout)
|
|
out = bz2.decompress(base64.b64decode(wire['stdout'],validate=True))
|
|
err = bz2.decompress(base64.b64decode(wire['stderr'],validate=True))
|
|
meta = wire['meta']
|
|
if not RECORD_WIRE: entry['meta'] = meta
|
|
validate_meta(meta,config,out,err)
|
|
return out,err
|
|
|
|
HEADER = '#ifndef RESOURCE_PARSER_H\n#define RESOURCE_PARSER_H\n#include <stddef.h>\n#define RP_NAME 24\n#define RP_SENSORS 8\n#define RP_LINE 160\n#define RP_DOCUMENT 8192\n#define RP_OK 0\n#define RP_OPEN -1\n#define RP_FORMAT -2\n#define RP_READ -3\n#define RP_CAPACITY -4\n#define RP_CLOSE -5\nstruct rp_sensor {\n int id;\n char kind;\n int gain;\n int bias;\n int has_limit;\n int low;\n int high;\n};\nstruct rp_model {\n char site[RP_NAME + 1];\n int rate;\n size_t count;\n struct rp_sensor sensors[RP_SENSORS];\n};\nstruct rp_tokens {\n size_t count;\n char words[6][RP_LINE + 1];\n};\nstruct rp_builder {\n struct rp_model model;\n int saw_site;\n int saw_rate;\n int ended;\n};\nstruct rp_stream;\nstruct rp_stream *rp_open(const char *name);\nint rp_get(struct rp_stream *stream);\nint rp_error(struct rp_stream *stream);\nint rp_close(struct rp_stream *stream);\nint rp_tokenize(const char *line, struct rp_tokens *tokens);\nint rp_number(const char *text, int low, int high, int *value);\nint rp_name(const char *text);\nint rp_keyword(const struct rp_tokens *tokens, const char *word, size_t count);\nvoid rp_builder_init(struct rp_builder *builder);\nint rp_builder_line(struct rp_builder *builder, const struct rp_tokens *tokens);\nint rp_builder_done(const struct rp_builder *builder);\nint rp_load(const char *name, struct rp_model *result);\n#endif\n'
|
|
OBSERVER = '#include "parser.h"\n#include <stdio.h>\n#include <stdlib.h>\n#include <string.h>\nstruct rp_stream {\n int active;\n size_t position;\n int failed;\n};\nstatic struct rp_stream slots[4];\nstatic unsigned char bytes[RP_DOCUMENT + 512];\nstatic size_t byte_count;\nstatic long error_at;\nstatic int deny_open;\nstatic int deny_close;\nstatic int opens;\nstatic int closes;\n\nstruct rp_stream *rp_open(const char *name)\n{\n size_t i;\n if (strcmp(name, "ledger") != 0 || deny_open) {\n return NULL;\n }\n for (i = 0; i < 4; ++i) {\n if (!slots[i].active) {\n slots[i].active = 1;\n slots[i].position = 0;\n slots[i].failed = 0;\n ++opens;\n return &slots[i];\n }\n }\n return NULL;\n}\n\nint rp_get(struct rp_stream *stream)\n{\n if (error_at >= 0 && stream->position >= (size_t)error_at) {\n stream->failed = 1;\n return -1;\n }\n if (stream->position == byte_count) {\n return -1;\n }\n return bytes[stream->position++];\n}\n\nint rp_error(struct rp_stream *stream)\n{\n return stream->failed;\n}\n\nint rp_close(struct rp_stream *stream)\n{\n stream->active = 0;\n ++closes;\n return deny_close ? -1 : 0;\n}\n\nstatic int nibble(int c)\n{\n if (c >= \'0\' && c <= \'9\') return c - \'0\';\n if (c >= \'a\' && c <= \'f\') return c - \'a\' + 10;\n return -1;\n}\n\nstatic int read_case(void)\n{\n char hex[(RP_DOCUMENT + 512) * 2 + 1];\n size_t i;\n size_t length;\n if (scanf("%ld %d %d %17408s", &error_at, &deny_open, &deny_close, hex) != 4) {\n return 0;\n }\n if (strcmp(hex, "-") == 0) {\n byte_count = 0;\n return 1;\n }\n length = strlen(hex);\n if (length % 2 != 0) return 0;\n byte_count = length / 2;\n for (i = 0; i < byte_count; ++i) {\n int a = nibble(hex[i * 2]);\n int b = nibble(hex[i * 2 + 1]);\n if (a < 0 || b < 0) return 0;\n bytes[i] = (unsigned char)(a * 16 + b);\n }\n return 1;\n}\n\nstatic void print_row(int status, const struct rp_model *model, int unchanged)\n{\n size_t i;\n int live = 0;\n for (i = 0; i < 4; ++i) live += slots[i].active;\n printf("{\\"rc\\":%d,\\"live\\":%d,\\"opens\\":%d,\\"closes\\":%d,\\"unchanged\\":%d",\n status, live, opens, closes, unchanged);\n if (status == RP_OK) {\n printf(",\\"site\\":\\"%s\\",\\"rate\\":%d,\\"sensors\\":[", model->site, model->rate);\n for (i = 0; i < model->count && i < RP_SENSORS; ++i) {\n const struct rp_sensor *s = &model->sensors[i];\n if (i) printf(",");\n printf("[%d,%d,%d,%d,%d", s->id, (unsigned char)s->kind,\n s->gain, s->bias, s->has_limit);\n if (s->has_limit) printf(",%d,%d", s->low, s->high);\n printf("]");\n }\n printf("]");\n }\n printf("}");\n}\n\nint main(void)\n{\n unsigned int count;\n unsigned int i;\n struct rp_model model;\n memset(&model, 0x5a, sizeof(model));\n if (scanf("%u", &count) != 1) return 70;\n printf("{\\"rows\\":[");\n for (i = 0; i < count; ++i) {\n struct rp_model before;\n int status;\n if (!read_case()) return 70;\n before = model;\n status = rp_load("ledger", &model);\n if (i) printf(",");\n print_row(status, &model, memcmp(&before, &model, sizeof(model)) == 0);\n }\n printf("]}\\n");\n return 0;\n}\n'
|
|
|
|
import re
|
|
|
|
COMPILER_DEADLINE = 20
|
|
CANDIDATE_DEADLINE = 8
|
|
CLEANUP_DEADLINE = 5
|
|
ABSENCE_DEADLINE = 5
|
|
|
|
|
|
def integer(token, low, high):
|
|
if re.fullmatch(rb'[+-]?[0-9]+',token) is None:
|
|
raise ValueError('number')
|
|
negative = token.startswith(b'-')
|
|
digits = token.lstrip(b'+-').lstrip(b'0') or b'0'
|
|
if len(digits) > 6:
|
|
raise ValueError('range')
|
|
value = int(digits) * (-1 if negative else 1)
|
|
if not low <= value <= high:
|
|
raise ValueError('range')
|
|
return value
|
|
|
|
|
|
def document(data, error_at):
|
|
# Injected faults only accompany valid documents or valid prefixes. Format
|
|
# examples are fault-free, preserving the public first-detected-error rule.
|
|
if error_at >= 0 and error_at <= len(data):
|
|
return -3,None
|
|
if len(data)>8192:
|
|
return -2,None
|
|
state={'site':None,'rate':1,'sensors':[]}
|
|
rate_seen=False
|
|
ended=False
|
|
for line in data.split(b'\n'):
|
|
if len(line)>160 or b'\0' in line:
|
|
return -2,None
|
|
line=line.strip(b' \t\r')
|
|
if not line or line.startswith(b'#'):
|
|
continue
|
|
words=re.split(rb'[ \t\r]+',line)
|
|
if ended or any(any(c<33 or c>126 for c in w) for w in words):
|
|
return -2,None
|
|
key=words[0]
|
|
try:
|
|
if key==b'SITE' and len(words)==2:
|
|
if state['site'] is not None or not re.fullmatch(rb'[a-z0-9-]{1,24}',words[1]):
|
|
raise ValueError('site')
|
|
state['site']=words[1].decode('ascii')
|
|
elif key==b'RATE' and len(words)==2:
|
|
if state['site'] is None or rate_seen or state['sensors']:
|
|
raise ValueError('rate placement')
|
|
state['rate']=integer(words[1],1,1000)
|
|
rate_seen=True
|
|
elif key==b'SENSOR' and len(words)==5:
|
|
if state['site'] is None or words[2] not in (b'T',b'V',b'I'):
|
|
raise ValueError('sensor')
|
|
sensor=[integer(words[1],0,31),words[2][0],integer(words[3],1,1000),integer(words[4],-100000,100000),0]
|
|
if any(s[0]==sensor[0] for s in state['sensors']):
|
|
raise ValueError('duplicate')
|
|
if len(state['sensors'])==8:
|
|
return -4,None
|
|
state['sensors'].append(sensor)
|
|
elif key==b'LIMIT' and len(words)==4:
|
|
ident=integer(words[1],0,31)
|
|
low=integer(words[2],-100000,100000)
|
|
high=integer(words[3],-100000,100000)
|
|
sensor=next((s for s in state['sensors'] if s[0]==ident),None)
|
|
if sensor is None or sensor[4] or low>high:
|
|
raise ValueError('limit')
|
|
sensor[4]=1
|
|
sensor.extend([low,high])
|
|
elif key==b'END' and len(words)==1:
|
|
if state['site'] is None or not state['sensors']:
|
|
raise ValueError('incomplete')
|
|
ended=True
|
|
else:
|
|
raise ValueError('record')
|
|
except ValueError:
|
|
return -2,None
|
|
if not ended:
|
|
return -2,None
|
|
return 0,state
|
|
|
|
|
|
def cases():
|
|
valid=b'SITE north-2\nRATE 50\nSENSOR 3 T 2 -4\nLIMIT 3 -8 8\nEND\n'
|
|
result=[(valid,-1,0,0)]
|
|
bad=b'SITE north-2\nSENSOR broken\n'
|
|
result.extend([(bad,-1,0,0)]*10)
|
|
result.append((valid,-1,0,0))
|
|
for offset in [0,1,5,len(valid)-1,len(valid),len(valid)+1]:
|
|
result.append((valid,offset,0,0))
|
|
result.extend([(valid,-1,1,0),(valid,-1,0,1),(bad,-1,0,1),(valid,-1,0,0)])
|
|
forms=[b'',b'END\n',b'SITE x\nEND\n',b'SITE X\nSENSOR 0 T 1 0\nEND',
|
|
b'SITE x\nSITE y\n',b'SITE x\nRATE 0\n',b'SITE x\nRATE 1001\n',
|
|
b'SITE x\nRATE 1_0\n',b'SITE x\nRATE 1\nRATE 2\n',
|
|
b'SITE x\nSENSOR 0 T 1 0\nRATE 2\nEND',
|
|
b'SITE x\nSENSOR 0 Q 1 0\nEND',b'SITE x\nSENSOR 32 T 1 0\nEND',
|
|
b'SITE x\nSENSOR -1 T 1 0\nEND',b'SITE x\nSENSOR 0 T 0 0\nEND',
|
|
b'SITE x\nSENSOR 0 T 1 -100001\nEND',b'SITE x\nSENSOR 0 T 1 100001\nEND',
|
|
b'SITE x\nSENSOR 0 T 1 0\nSENSOR 0 V 1 0\nEND',
|
|
b'SITE x\nLIMIT 0 1 2\nEND',
|
|
b'SITE x\nSENSOR 0 T 1 0\nLIMIT 0 2 1\nEND',
|
|
b'SITE x\nSENSOR 0 T 1 0\nLIMIT 0 1 1\nLIMIT 0 1 1\nEND',
|
|
valid+b'END\n',valid+b'SENSOR 1 T 1 0\n',
|
|
valid.replace(b'RATE 50',b'RATE\v50'),valid.replace(b'RATE 50',b'RATE\f50'),
|
|
valid.replace(b'RATE 50',b'RATE 5\x0050'),b'#'+b'x'*160+b'\n'+valid,
|
|
b'#'+b'x'*159+b'\n'+valid,b'#\xff\t\n'+valid,
|
|
valid+b'# trailing comment',valid.rstrip(b'\n'),valid+b' \t\r\n',
|
|
b'SITE '+b'z'*24+b'\nSENSOR +00031 I +1000 -100000\nLIMIT 31 -100000 +100000\nEND',
|
|
b'SITE '+b'z'*25+b'\nSENSOR 0 T 1 0\nEND',
|
|
b'SITE a\nSENSOR -000 V 001 -000\nEND',
|
|
b'SITE a\nSENSOR 0 T 1 0\nEND # no-inline\n',
|
|
b'\tSITE\tx\r\nSENSOR\t0\tT\t1\t0\r\nEND\r\n']
|
|
forms += [b'SITE x\n'+b''.join(f'SENSOR {i} T 1 0\n'.encode() for i in range(n))+b'END\n' for n in [8,9]]
|
|
for data in forms:
|
|
result.append((data,-1,0,0))
|
|
# Exact document and line boundaries, followed by recovery.
|
|
prefix=b'#'+b'p'*158+b'\n'
|
|
extra=8192-len(valid)
|
|
fill=prefix*(extra//160)
|
|
rest=extra%160
|
|
if rest: fill+=b'#'+b'p'*(rest-1)
|
|
boundary=fill+b'\n'+valid if rest else fill+valid
|
|
# A separate all-comment sequence avoids concatenating records at 8192.
|
|
boundary=b'#\n'*((8192-len(valid))//2)+valid
|
|
boundary+=b' '*(8192-len(boundary))
|
|
result.extend([(boundary,-1,0,0),(boundary+b' ',-1,0,0),(valid,-1,0,0)])
|
|
rng=random.Random(19920261025)
|
|
for iteration in range(112):
|
|
ids=rng.sample(range(32),rng.randint(1,8))
|
|
records=[f'SITE field-{iteration}',f'RATE {rng.randint(1,1000)}']
|
|
for ident in ids:
|
|
records.append(f'SENSOR {ident} {rng.choice("TVI")} {rng.randint(1,1000)} {rng.randint(-100000,100000)}')
|
|
for ident in rng.sample(ids,rng.randrange(len(ids)+1)):
|
|
low=rng.randint(-100000,100000)
|
|
high=rng.randint(low,100000)
|
|
records.append(f'LIMIT {ident} {low} {high}')
|
|
records.append('END')
|
|
data=('\n'.join(records)+ ('\n' if iteration%2 else '')).encode()
|
|
result.append((data,-1,0,0))
|
|
if iteration%4==0:
|
|
result.extend([(data,len(data),0,0),(data,-1,0,1),(data,-1,1,0)])
|
|
if iteration%3==0:
|
|
result.append((data+b'\nBAD\n',-1,0,0))
|
|
return result
|
|
|
|
|
|
def expected_rows(entries):
|
|
opened=closed=0
|
|
rows=[]
|
|
for data,at,op,cl in entries:
|
|
if op:
|
|
status,model=-1,None
|
|
else:
|
|
opened+=1
|
|
closed+=1
|
|
status,model=document(data,at)
|
|
if status==0 and cl:
|
|
status,model=-5,None
|
|
row=dict(rc=status,live=0,opens=opened,closes=closed)
|
|
if status:
|
|
row['unchanged']=1
|
|
else:
|
|
row.update(model)
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def compare_output(actual, wanted):
|
|
# JSON's own whitespace domain; full consumption without an invented line rule.
|
|
value=json.loads(actual)
|
|
if type(value) is not dict or set(value)!={'rows'} or type(value['rows']) is not list:
|
|
raise ValueError('observation shape')
|
|
rows=value['rows']
|
|
if len(rows)!=len(wanted):
|
|
raise ValueError('observation row count')
|
|
for index,(row,expected) in enumerate(zip(rows,wanted)):
|
|
if type(row) is not dict:
|
|
raise ValueError('row not object '+str(index))
|
|
if expected['rc']==0:
|
|
if type(row.get('unchanged')) is not int or row['unchanged'] not in (0,1):
|
|
raise ValueError('success unchanged observation '+str(index))
|
|
row=dict(row)
|
|
row.pop('unchanged')
|
|
# Recursively reject bool standing in for integral observations.
|
|
def typed_equal(left,right):
|
|
if type(left) is not type(right): return False
|
|
if type(right) is list: return len(left)==len(right) and all(typed_equal(a,b) for a,b in zip(left,right))
|
|
if type(right) is dict: return set(left)==set(right) and all(typed_equal(left[k],right[k]) for k in right)
|
|
return left==right
|
|
if not typed_equal(row,expected):
|
|
raise ValueError('first raw observation difference row '+str(index)+' actual='+repr(row)+' expected='+repr(expected))
|
|
|
|
|
|
def main():
|
|
evidence=[]
|
|
problems=[]
|
|
business_ok=False
|
|
cleanup_ok=False
|
|
absence_ok=False
|
|
scratch=None
|
|
try:
|
|
scratch=Path(tempfile.mkdtemp(prefix='resource-parser-'))
|
|
work=scratch/'build'
|
|
work.mkdir(mode=0o777)
|
|
os.chmod(work,0o777)
|
|
for name in ('lexer.c','model.c','parser.c'):
|
|
data=(Path(sys.argv[1])/name).read_bytes()
|
|
(work/name).write_bytes(data)
|
|
(work/'parser.h').write_text(HEADER)
|
|
(work/'observer.c').write_text(OBSERVER)
|
|
entries=cases()
|
|
payload=(str(len(entries))+'\n'+''.join(str(at)+' '+str(op)+' '+str(cl)+' '+(data.hex() or '-')+'\n' for data,at,op,cl in entries)).encode('ascii')
|
|
wanted=expected_rows(entries)
|
|
evidence.append(dict(phase='input',rows=len(entries),stdin_len=len(payload),stdin_sha256=hashlib.sha256(payload).hexdigest()))
|
|
compiler=['gcc','-std=c11','-O2','-Wall','-Wextra','-Werror','lexer.c','model.c','parser.c','observer.c','-o','ledger']
|
|
execute_stage('compiler',compiler,work,COMPILER_DEADLINE,b'',evidence)
|
|
output,_=execute_stage('candidate',[str(work/'ledger')],work,CANDIDATE_DEADLINE,payload,evidence)
|
|
compare_output(output,wanted)
|
|
business_ok=True
|
|
except BaseException as error:
|
|
problems.append(['business',type(error).__name__,str(error)])
|
|
finally:
|
|
if scratch is not None:
|
|
try:
|
|
execute_stage('cleanup',[sys.executable,'-I','-c',CLEANUP,str(scratch)],'/',CLEANUP_DEADLINE,b'',evidence)
|
|
cleanup_ok=True
|
|
except BaseException as error:
|
|
problems.append(['cleanup',type(error).__name__,str(error)])
|
|
try:
|
|
execute_stage('absence',[sys.executable,'-I','-c',ABSENCE,str(scratch)],'/',ABSENCE_DEADLINE,b'',evidence)
|
|
absence_ok=True
|
|
except BaseException as error:
|
|
problems.append(['absence',type(error).__name__,str(error)])
|
|
marker=dict(stages=evidence,flags=[business_ok,cleanup_ok,absence_ok],exceptions=problems,
|
|
raw_storage='not retained; actual captured bytes lengths and SHA256 only')
|
|
packed=base64.b64encode(bz2.compress(json.dumps(marker,separators=(',',':')).encode())).decode('ascii')
|
|
sys.stderr.write('RESOURCE_META_BZ2 '+packed+'\n')
|
|
if business_ok and cleanup_ok and absence_ok and not problems:
|
|
print('PASS')
|
|
return 0
|
|
print('FAIL '+repr(problems)[:240])
|
|
return 1
|
|
|
|
|
|
if __name__=='__main__':
|
|
sys.exit(main())
|