One user-space repair task per eligible HWE debug-diag / compile-deploy base ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content filter. Each ships a public spec with a check-to-basis table, at least two defects with single-fix fixtures, a symptom-masking fixture and an alternative correct implementation. Claude re-checked all 320: structure, the HWE quality items and trace6 Docker calibration receipts against current digests (320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120 ad-hoc family names; the next batch maps them to the canonical list. The batch's 1,256-line progress log stays in its factory report and the 36-local branch batch15-hwe. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
460 lines
30 KiB
Python
460 lines
30 KiB
Python
import base64
|
|
|
|
import bz2
|
|
|
|
import hashlib
|
|
|
|
import json
|
|
|
|
import math
|
|
|
|
import os
|
|
|
|
from pathlib import Path
|
|
|
|
import random
|
|
|
|
import re
|
|
|
|
import subprocess
|
|
|
|
import sys
|
|
|
|
import tempfile
|
|
|
|
import time
|
|
|
|
RECOVERY_SECONDS = 3
|
|
|
|
RECORD_WIRE = False
|
|
|
|
SUPERVISOR = r'''
|
|
import base64,bz2,ctypes,hashlib,json,os,signal,subprocess,sys,time
|
|
cfg=json.loads(sys.argv[1]); start=time.monotonic(); errors=[]; tracked={}; signals={}; waits=[]
|
|
leader=None; leader_start=None; output=b''; diagnostic=b''; timed_out=False; drained=False
|
|
reaped=False; leader_rc=None; remaining=[]; pg_absent=False
|
|
|
|
def fault(op, error):
|
|
errors.append([op,type(error).__name__,str(error)])
|
|
|
|
def exact(pid):
|
|
try:
|
|
with open('/proc/%d/stat'%pid,'rb') as stream: data=stream.read()
|
|
words=data[data.rfind(b')')+2:].split()
|
|
return {'pid':pid,'state':words[0].decode('ascii'),'ppid':int(words[1]),'pg':int(words[2]),'start':int(words[19])}
|
|
except (FileNotFoundError,ProcessLookupError): return None
|
|
|
|
def table_now():
|
|
result={}
|
|
for name in os.listdir('/proc'):
|
|
if name.isascii() and name.isdigit():
|
|
value=exact(int(name))
|
|
if value is not None: result[value['pid']]=value
|
|
return result
|
|
|
|
def closure(table, supervisor, remembered):
|
|
roots={supervisor}
|
|
for pid,old in remembered.items():
|
|
current=table.get(pid)
|
|
if current and current['start']==old['start'] and current['state'] not in ('Z','X'):
|
|
roots.add(pid)
|
|
found={}
|
|
changed=True
|
|
while changed:
|
|
changed=False
|
|
for pid,current in table.items():
|
|
if pid!=supervisor and pid not in found and (current['ppid'] in roots or pid in roots):
|
|
found[pid]=current; roots.add(pid); changed=True
|
|
return found
|
|
|
|
def discover():
|
|
table=table_now(); found=closure(table,os.getpid(),tracked)
|
|
for pid,current in found.items():
|
|
if pid not in tracked or tracked[pid]['start']!=current['start']:
|
|
tracked[pid]=dict(current)
|
|
return table,found
|
|
|
|
def send_fresh(pid, token):
|
|
current=exact(pid)
|
|
if current is None or current['start']!=token['start'] or current['state'] in ('Z','X'): return
|
|
os.kill(pid,signal.SIGKILL)
|
|
signals[(pid,token['start'])]=signals.get((pid,token['start']),0)+1
|
|
|
|
def own_wait():
|
|
global reaped,leader_rc
|
|
if leader is not None:
|
|
value=leader.poll()
|
|
if value is not None:
|
|
leader_rc=leader.wait(timeout=0)
|
|
reaped=True
|
|
|
|
def adopted_wait():
|
|
if leader is not None and not reaped: return
|
|
while True:
|
|
try: pid,status=os.waitpid(-1,os.WNOHANG)
|
|
except ChildProcessError: break
|
|
if pid==0: break
|
|
waits.append([pid,None,os.waitstatus_to_exitcode(status)])
|
|
|
|
supervisor_start=exact(os.getpid())['start']
|
|
try:
|
|
libc=ctypes.CDLL(None,use_errno=True)
|
|
if libc.prctl(36,1,0,0,0)!=0: raise OSError(ctypes.get_errno(),'subreaper')
|
|
leader=subprocess.Popen(cfg['argv'],cwd=cfg['cwd'],stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,stderr=subprocess.PIPE,start_new_session=True)
|
|
info=exact(leader.pid)
|
|
if info is None: raise RuntimeError('leader identity disappeared before capture')
|
|
leader_start=info['start']; tracked[leader.pid]=info
|
|
payload=sys.stdin.buffer.read()
|
|
first=True
|
|
while True:
|
|
try:
|
|
output,diagnostic=leader.communicate(input=payload if first else None,timeout=0.04)
|
|
drained=True; own_wait(); break
|
|
except subprocess.TimeoutExpired:
|
|
first=False
|
|
own_wait()
|
|
discover()
|
|
if time.monotonic()-start>=cfg['deadline']:
|
|
timed_out=True; break
|
|
except BaseException as error:
|
|
fault('execute',error)
|
|
finally:
|
|
recovery_end=time.monotonic()+cfg['recovery']
|
|
while True:
|
|
try: own_wait()
|
|
except BaseException as error: fault('own_wait',error)
|
|
found={}
|
|
try: _,found=discover()
|
|
except BaseException as error: fault('discover',error)
|
|
for pid,token in list(found.items()):
|
|
try: send_fresh(pid,token)
|
|
except (FileNotFoundError,ProcessLookupError): pass
|
|
except BaseException as error: fault('signal',error)
|
|
try: own_wait()
|
|
except BaseException as error: fault('own_wait',error)
|
|
try: adopted_wait()
|
|
except BaseException as error: fault('adopted_wait',error)
|
|
if leader is not None and not drained:
|
|
try:
|
|
output,diagnostic=leader.communicate(timeout=0.03)
|
|
drained=True; own_wait()
|
|
except subprocess.TimeoutExpired: pass
|
|
except BaseException as error: fault('drain',error)
|
|
try:
|
|
table,found=discover()
|
|
remaining=[[p,v['start'],v['state']] for p,v in found.items()]
|
|
pg_absent=leader is not None and not any(v['pg']==leader.pid for v in table.values())
|
|
except BaseException as error:
|
|
fault('absence',error); remaining=[['unknown']]; pg_absent=False
|
|
if reaped and drained and not remaining and pg_absent: break
|
|
if time.monotonic()>=recovery_end:
|
|
fault('recovery',RuntimeError('deadline with unconfirmed closure')); break
|
|
time.sleep(0.01)
|
|
|
|
meta={'phase':cfg['phase'],'nonce':cfg['nonce'],'argv':cfg['argv'],
|
|
'uid':os.getuid(),'gid':os.getgid(),'groups':os.getgroups(),
|
|
'supervisor_pid':os.getpid(),'supervisor_start':supervisor_start,
|
|
'leader_pid':None if leader is None else leader.pid,'leader_start':leader_start,
|
|
'leader_rc':leader_rc,'leader_reaped':reaped,'timeout':timed_out,
|
|
'controlled_rc':124 if timed_out else leader_rc,'elapsed':time.monotonic()-start,
|
|
'deadline':cfg['deadline'],'recovery':cfg['recovery'],
|
|
'pg_absent':pg_absent,'remaining':remaining,'exceptions':errors,
|
|
'adopted_waits':waits,
|
|
'descendants':[[p,t['start'],signals.get((p,t['start']),0)]
|
|
for p,t in tracked.items() if leader is None or p!=leader.pid],
|
|
'stdout_len':len(output),'stdout_sha256':hashlib.sha256(output).hexdigest(),
|
|
'stderr_len':len(diagnostic),'stderr_sha256':hashlib.sha256(diagnostic).hexdigest()}
|
|
wire={'meta':meta,'stdout':base64.b64encode(bz2.compress(output)).decode('ascii'),
|
|
'stderr':base64.b64encode(bz2.compress(diagnostic)).decode('ascii')}
|
|
sys.stdout.write(json.dumps(wire,separators=(',',':'))+'\n')
|
|
'''
|
|
|
|
CLEANUP = "import shutil,sys; shutil.rmtree(sys.argv[1])"
|
|
|
|
ABSENCE = "import os,sys\ntry: os.lstat(sys.argv[1])\nexcept FileNotFoundError: sys.exit(0)\nsys.exit(1)"
|
|
|
|
def identity_int(value, minimum=0):
|
|
return type(value) is int and value >= minimum
|
|
|
|
def validate_meta(meta, config, out, err):
|
|
required = {'phase','nonce','argv','uid','gid','groups','supervisor_pid','supervisor_start',
|
|
'leader_pid','leader_start','leader_rc','leader_reaped','timeout','controlled_rc',
|
|
'elapsed','deadline','recovery','pg_absent','remaining','exceptions','descendants',
|
|
'stdout_len','stdout_sha256','stderr_len','stderr_sha256','adopted_waits'}
|
|
if type(meta) is not dict or not required <= meta.keys():
|
|
raise ValueError('missing stage metadata')
|
|
if meta['phase'] != config['phase'] or meta['nonce'] != config['nonce'] or meta['argv'] != config['argv']:
|
|
raise ValueError('stage binding conflict')
|
|
for name in ('uid','gid','supervisor_pid','supervisor_start','leader_pid','leader_start'):
|
|
if not identity_int(meta[name], 1 if name not in ('uid','gid') else 0):
|
|
raise ValueError('invalid identity '+name)
|
|
if meta['supervisor_pid'] == meta['leader_pid']:
|
|
raise ValueError('supervisor and leader identical')
|
|
isolated = os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534'
|
|
expected_identity = (65534,65534,[]) if isolated else (os.getuid(),os.getgid(),os.getgroups())
|
|
if type(meta['groups']) is not list or any(type(x) is not int or x < 0 for x in meta['groups']):
|
|
raise ValueError('invalid groups')
|
|
if (meta['uid'],meta['gid'],meta['groups']) != expected_identity:
|
|
raise ValueError('execution identity conflict')
|
|
if type(meta['deadline']) is not int or meta['deadline'] != config['deadline']:
|
|
raise ValueError('deadline conflict')
|
|
if type(meta['recovery']) is not int or meta['recovery'] != RECOVERY_SECONDS:
|
|
raise ValueError('recovery conflict')
|
|
elapsed = meta['elapsed']
|
|
if type(elapsed) not in (int,float) or not math.isfinite(elapsed) or not 0 <= elapsed <= config['deadline']+RECOVERY_SECONDS+2:
|
|
raise ValueError('invalid elapsed')
|
|
for name in ('timeout','leader_reaped','pg_absent'):
|
|
if type(meta[name]) is not bool:
|
|
raise ValueError('invalid boolean '+name)
|
|
for name in ('leader_rc','controlled_rc'):
|
|
if type(meta[name]) is not int:
|
|
raise ValueError('invalid return code '+name)
|
|
if meta['controlled_rc'] != (124 if meta['timeout'] else meta['leader_rc']):
|
|
raise ValueError('return code conflict')
|
|
for name in ('remaining','exceptions','descendants','adopted_waits'):
|
|
if type(meta[name]) is not list:
|
|
raise ValueError('invalid list '+name)
|
|
for row in meta['descendants']:
|
|
if type(row) is not list or len(row) != 3 or not all(identity_int(x,1) for x in row[:2]) or not identity_int(row[2]):
|
|
raise ValueError('invalid descendant record')
|
|
for row in meta['adopted_waits']:
|
|
if type(row) is not list or len(row) != 3 or not identity_int(row[0],1) or row[1] is not None or type(row[2]) is not int:
|
|
raise ValueError('invalid adopted wait fact')
|
|
for name,data in (('stdout',out),('stderr',err)):
|
|
if not identity_int(meta[name+'_len']) or meta[name+'_len'] != len(data):
|
|
raise ValueError('invalid IO length')
|
|
if type(meta[name+'_sha256']) is not str or meta[name+'_sha256'] != hashlib.sha256(data).hexdigest():
|
|
raise ValueError('invalid IO digest')
|
|
if meta['exceptions'] or meta['remaining'] or not meta['leader_reaped'] or not meta['pg_absent']:
|
|
raise ValueError('stage recovery or exception failure')
|
|
if meta['timeout'] or meta['leader_rc'] != 0 or meta['controlled_rc'] != 0:
|
|
raise ValueError('controlled timeout' if meta['timeout'] else 'leader nonzero exit')
|
|
|
|
def execute_stage(phase, argv, cwd, deadline, payload, evidence):
|
|
nonce = os.urandom(8).hex()
|
|
config = dict(phase=phase,argv=argv,cwd=str(cwd),deadline=deadline,recovery=RECOVERY_SECONDS,nonce=nonce)
|
|
wrapper_argv = [sys.executable,'-I','-c',SUPERVISOR,json.dumps(config,separators=(',',':'))]
|
|
raw = subprocess.run(wrapper_argv,input=payload,capture_output=True,timeout=deadline+RECOVERY_SECONDS+5)
|
|
# Preserve actual wrapper facts before touching or trusting its wire protocol.
|
|
entry = dict(phase=phase,nonce=nonce,wrapper_rc=raw.returncode,
|
|
wire_len=len(raw.stdout),wire_sha256=hashlib.sha256(raw.stdout).hexdigest(),
|
|
wrapper_stderr_len=len(raw.stderr),wrapper_stderr_sha256=hashlib.sha256(raw.stderr).hexdigest())
|
|
if RECORD_WIRE:
|
|
entry['wire_ascii'] = raw.stdout.decode('ascii')
|
|
entry['wrapper_stderr_bz2'] = base64.b64encode(bz2.compress(raw.stderr)).decode('ascii')
|
|
evidence.append(entry)
|
|
if raw.returncode != 0:
|
|
raise ValueError(phase+' supervisor nonzero')
|
|
wire = json.loads(raw.stdout)
|
|
out = bz2.decompress(base64.b64decode(wire['stdout'],validate=True))
|
|
err = bz2.decompress(base64.b64decode(wire['stderr'],validate=True))
|
|
meta = wire['meta']
|
|
if not RECORD_WIRE: entry['meta'] = meta
|
|
validate_meta(meta,config,out,err)
|
|
return out,err
|
|
|
|
SUPPORT = {'include/batch.h': '#ifndef BATCH_H\n#define BATCH_H\n#include <stddef.h>\n#include <stdint.h>\n#include <stdio.h>\n#define BF_CHANNELS 8u\n#define BF_CAPACITY 64u\n#define BF_MAX_BATCH 1000000u\n#define BF_MIN_VALUE (-1000000)\n#define BF_MAX_VALUE 1000000\n#define BF_MAX_WEIGHT 1000u\n#define BF_LINE_SIZE 1024u\ntypedef enum {\n BF_OK = 0,\n BF_STATE = 1,\n BF_RANGE = 2,\n BF_FULL = 3\n} bf_result;\ntypedef struct {\n uint32_t channel;\n int32_t value;\n uint32_t weight;\n uint32_t filtered;\n} bf_row;\ntypedef struct {\n uint64_t rows;\n uint64_t units;\n int64_t weighted_total;\n int32_t minimum;\n int32_t maximum;\n} bf_bucket;\ntypedef struct {\n bf_bucket channels[BF_CHANNELS];\n uint64_t accepted;\n uint64_t ignored;\n uint32_t batches;\n uint32_t last_batch;\n} bf_totals;\ntypedef struct {\n bf_totals committed;\n bf_row staged[BF_CAPACITY];\n size_t staged_count;\n uint32_t active_batch;\n int active;\n} bf_ledger;\ntypedef struct {\n bf_totals totals;\n size_t staged_count;\n uint32_t active_batch;\n int active;\n} bf_view;\ntypedef enum {\n BF_CMD_BEGIN,\n BF_CMD_ADD,\n BF_CMD_COMMIT,\n BF_CMD_CANCEL,\n BF_CMD_SNAPSHOT,\n BF_CMD_INVALID\n} bf_opcode;\ntypedef struct {\n bf_opcode opcode;\n uint32_t batch;\n bf_row row;\n} bf_command;\ntypedef enum {\n BF_PARSE_OK,\n BF_PARSE_EMPTY,\n BF_PARSE_INVALID\n} bf_parse_result;\nvoid bf_init(bf_ledger *ledger);\nbf_result bf_begin(bf_ledger *ledger, uint32_t batch);\nbf_result bf_add(bf_ledger *ledger, bf_row row);\nbf_result bf_commit(bf_ledger *ledger);\nbf_result bf_cancel(bf_ledger *ledger);\nvoid bf_snapshot(const bf_ledger *ledger, bf_view *view);\nbf_parse_result bf_parse_line(const char *line, bf_command *command);\nint bf_read_line(FILE *stream, char *line, size_t capacity);\nconst char *bf_operation_name(bf_opcode opcode);\nvoid bf_print_result(FILE *stream, size_t ordinal, bf_opcode opcode, bf_result result);\nvoid bf_print_view(FILE *stream, size_t ordinal, const bf_view *view);\nint bf_execute(FILE *input, FILE *output, FILE *diagnostic);\n#endif\n', 'src/text.c': '#include "batch.h"\n#include <limits.h>\n#include <string.h>\n\ntypedef struct {\n const char *first;\n size_t length;\n} token;\n\nstatic int horizontal(unsigned char character)\n{\n return character == \' \' || character == \'\\t\' || character == \'\\r\';\n}\n\nstatic size_t tokenize(const char *line, token *tokens, size_t capacity)\n{\n size_t count = 0;\n const char *cursor = line;\n while (*cursor != \'\\0\') {\n const char *begin;\n while (horizontal((unsigned char)*cursor)) {\n ++cursor;\n }\n if (*cursor == \'\\0\') {\n break;\n }\n if (count == capacity) {\n return capacity + 1;\n }\n begin = cursor;\n while (*cursor != \'\\0\' && !horizontal((unsigned char)*cursor)) {\n ++cursor;\n }\n tokens[count].first = begin;\n tokens[count].length = (size_t)(cursor - begin);\n ++count;\n }\n return count;\n}\n\nstatic int literal(token value, const char *word)\n{\n size_t length = strlen(word);\n return value.length == length && memcmp(value.first, word, length) == 0;\n}\n\nstatic int number(token value, int64_t minimum, int64_t maximum, int64_t *result)\n{\n size_t offset = 0;\n uint64_t magnitude = 0;\n int negative = 0;\n int64_t signed_value;\n if (value.length == 0) {\n return 0;\n }\n if (value.first[offset] == \'-\' || value.first[offset] == \'+\') {\n negative = value.first[offset] == \'-\';\n ++offset;\n }\n if (offset == value.length) {\n return 0;\n }\n for (; offset < value.length; ++offset) {\n unsigned char character = (unsigned char)value.first[offset];\n unsigned digit;\n if (character < \'0\' || character > \'9\') {\n return 0;\n }\n digit = (unsigned)(character - \'0\');\n if (magnitude > (UINT64_MAX - digit) / 10u) {\n return 0;\n }\n magnitude = magnitude * 10u + digit;\n }\n if (magnitude > INT64_MAX) {\n return 0;\n }\n signed_value = (int64_t)magnitude;\n if (negative) {\n signed_value = -signed_value;\n }\n if (signed_value < minimum || signed_value > maximum) {\n return 0;\n }\n *result = signed_value;\n return 1;\n}\n\nstatic int parse_add(const token *tokens, bf_command *command)\n{\n int64_t channel;\n int64_t value;\n int64_t weight;\n int64_t filtered;\n if (!number(tokens[1], 0, UINT32_MAX, &channel)) {\n return 0;\n }\n if (!number(tokens[2], INT32_MIN, INT32_MAX, &value)) {\n return 0;\n }\n if (!number(tokens[3], 0, UINT32_MAX, &weight)) {\n return 0;\n }\n if (!number(tokens[4], 0, UINT32_MAX, &filtered)) {\n return 0;\n }\n command->row.channel = (uint32_t)channel;\n command->row.value = (int32_t)value;\n command->row.weight = (uint32_t)weight;\n command->row.filtered = (uint32_t)filtered;\n command->opcode = BF_CMD_ADD;\n return 1;\n}\n\nbf_parse_result bf_parse_line(const char *line, bf_command *command)\n{\n token tokens[5];\n size_t count = tokenize(line, tokens, 5);\n int64_t batch;\n memset(command, 0, sizeof(*command));\n command->opcode = BF_CMD_INVALID;\n if (count == 0) {\n return BF_PARSE_EMPTY;\n }\n if (count == 2 && literal(tokens[0], "BEGIN")) {\n if (!number(tokens[1], 0, UINT32_MAX, &batch)) {\n return BF_PARSE_INVALID;\n }\n command->opcode = BF_CMD_BEGIN;\n command->batch = (uint32_t)batch;\n return BF_PARSE_OK;\n }\n if (count == 5 && literal(tokens[0], "ADD")) {\n return parse_add(tokens, command) ? BF_PARSE_OK : BF_PARSE_INVALID;\n }\n if (count != 1) {\n return BF_PARSE_INVALID;\n }\n if (literal(tokens[0], "COMMIT")) {\n command->opcode = BF_CMD_COMMIT;\n } else if (literal(tokens[0], "CANCEL")) {\n command->opcode = BF_CMD_CANCEL;\n } else if (literal(tokens[0], "SNAPSHOT")) {\n command->opcode = BF_CMD_SNAPSHOT;\n } else {\n return BF_PARSE_INVALID;\n }\n return BF_PARSE_OK;\n}\n\nint bf_read_line(FILE *stream, char *line, size_t capacity)\n{\n size_t length = 0;\n int character;\n int too_long = 0;\n while ((character = fgetc(stream)) != EOF) {\n if (character == \'\\n\') {\n break;\n }\n if (length + 1 < capacity) {\n line[length++] = (char)character;\n } else {\n too_long = 1;\n }\n if (character == 0) {\n too_long = 1;\n }\n }\n if (ferror(stream)) {\n return -1;\n }\n if (character == EOF && length == 0 && !too_long) {\n return 0;\n }\n line[length] = \'\\0\';\n return too_long ? -1 : 1;\n}\n', 'src/report.c': '#include "batch.h"\n#include <inttypes.h>\n\nconst char *bf_operation_name(bf_opcode opcode)\n{\n switch (opcode) {\n case BF_CMD_BEGIN:\n return "BEGIN";\n case BF_CMD_ADD:\n return "ADD";\n case BF_CMD_COMMIT:\n return "COMMIT";\n case BF_CMD_CANCEL:\n return "CANCEL";\n case BF_CMD_SNAPSHOT:\n return "SNAPSHOT";\n default:\n return "INVALID";\n }\n}\n\nvoid bf_print_result(FILE *stream, size_t ordinal, bf_opcode opcode, bf_result result)\n{\n fprintf(stream, "R %zu %s %d\\n", ordinal, bf_operation_name(opcode), (int)result);\n}\n\nstatic void print_bucket(FILE *stream, size_t ordinal, size_t channel, const bf_bucket *bucket)\n{\n fprintf(stream,\n "B %zu %zu %" PRIu64 " %" PRIu64 " %" PRId64 " %" PRId32 " %" PRId32 "\\n",\n ordinal,\n channel,\n bucket->rows,\n bucket->units,\n bucket->weighted_total,\n bucket->minimum,\n bucket->maximum);\n}\n\nvoid bf_print_view(FILE *stream, size_t ordinal, const bf_view *view)\n{\n size_t channel;\n fprintf(stream,\n "H %zu %" PRIu64 " %" PRIu64 " %" PRIu32 " %" PRIu32 " %d %" PRIu32 " %zu\\n",\n ordinal,\n view->totals.accepted,\n view->totals.ignored,\n view->totals.batches,\n view->totals.last_batch,\n view->active,\n view->active_batch,\n view->staged_count);\n for (channel = 0; channel < BF_CHANNELS; ++channel) {\n print_bucket(stream, ordinal, channel, &view->totals.channels[channel]);\n }\n}\n', 'src/main.c': '#include "batch.h"\n#include <string.h>\n\nstatic bf_result dispatch(bf_ledger *ledger, const bf_command *command)\n{\n switch (command->opcode) {\n case BF_CMD_BEGIN:\n return bf_begin(ledger, command->batch);\n case BF_CMD_ADD:\n return bf_add(ledger, command->row);\n case BF_CMD_COMMIT:\n return bf_commit(ledger);\n case BF_CMD_CANCEL:\n return bf_cancel(ledger);\n case BF_CMD_SNAPSHOT:\n return BF_OK;\n default:\n return BF_RANGE;\n }\n}\n\nint bf_execute(FILE *input, FILE *output, FILE *diagnostic)\n{\n bf_ledger ledger;\n char line[BF_LINE_SIZE];\n size_t ordinal = 0;\n int read_status;\n bf_init(&ledger);\n while ((read_status = bf_read_line(input, line, sizeof(line))) != 0) {\n bf_command command;\n bf_parse_result parsed;\n bf_result result;\n if (read_status < 0) {\n fprintf(diagnostic, "input line cannot be read\\n");\n return 2;\n }\n parsed = bf_parse_line(line, &command);\n if (parsed == BF_PARSE_EMPTY) {\n continue;\n }\n if (parsed != BF_PARSE_OK) {\n fprintf(diagnostic, "invalid command after %zu operations\\n", ordinal);\n return 2;\n }\n ++ordinal;\n result = dispatch(&ledger, &command);\n bf_print_result(output, ordinal, command.opcode, result);\n if (command.opcode == BF_CMD_SNAPSHOT) {\n bf_view view;\n bf_snapshot(&ledger, &view);\n bf_print_view(output, ordinal, &view);\n }\n if (ferror(output)) {\n fprintf(diagnostic, "output failure\\n");\n return 2;\n }\n }\n if (fflush(output) != 0) {\n fprintf(diagnostic, "output flush failure\\n");\n return 2;\n }\n return 0;\n}\n\nint main(int argc, char **argv)\n{\n FILE *input = stdin;\n int status;\n if (argc > 2) {\n fprintf(stderr, "usage: batch-fold [script|-]\\n");\n return 2;\n }\n if (argc == 2 && strcmp(argv[1], "-") != 0) {\n input = fopen(argv[1], "rb");\n if (input == NULL) {\n fprintf(stderr, "cannot open batch script\\n");\n return 2;\n }\n }\n status = bf_execute(input, stdout, stderr);\n if (input != stdin && fclose(input) != 0) {\n fprintf(stderr, "input close failure\\n");\n status = 2;\n }\n return status;\n}\n'}
|
|
|
|
COMPILER_SECONDS = 15
|
|
CANDIDATE_SECONDS = 10
|
|
CLEANUP_SECONDS = 6
|
|
ABSENCE_SECONDS = 6
|
|
|
|
|
|
def script_operations():
|
|
operations = []
|
|
def add(*parts):
|
|
operations.append(tuple(parts))
|
|
add('SNAPSHOT')
|
|
add('ADD',0,1,1,0)
|
|
add('COMMIT')
|
|
add('CANCEL')
|
|
add('BEGIN',0)
|
|
add('BEGIN',1000001)
|
|
add('BEGIN',1)
|
|
add('BEGIN',2)
|
|
add('ADD',8,0,1,0)
|
|
add('ADD',0,-1000001,1,0)
|
|
add('ADD',0,1000001,1,0)
|
|
add('ADD',0,0,0,0)
|
|
add('ADD',0,0,1001,0)
|
|
add('ADD',0,0,1,2)
|
|
add('SNAPSHOT')
|
|
add('COMMIT')
|
|
add('SNAPSHOT')
|
|
add('BEGIN',1)
|
|
add('BEGIN',2)
|
|
add('ADD',0,12,2,0)
|
|
add('ADD',1,-5,3,0)
|
|
add('ADD',0,8,1,0)
|
|
add('SNAPSHOT')
|
|
add('COMMIT')
|
|
add('SNAPSHOT')
|
|
# A canceled nonempty batch and re-use of its ID exercise D2 independently.
|
|
add('BEGIN',3)
|
|
add('ADD',4,-93,11,0)
|
|
add('ADD',7,22,7,1)
|
|
add('SNAPSHOT')
|
|
add('CANCEL')
|
|
add('SNAPSHOT')
|
|
add('BEGIN',3)
|
|
add('CANCEL')
|
|
add('SNAPSHOT')
|
|
rng = random.Random(0xBA7C4)
|
|
for batch in range(4,22):
|
|
add('BEGIN',batch)
|
|
size = [1,2,7,64,0,13][(batch-4)%6]
|
|
for index in range(size):
|
|
channel = (index+batch)%8
|
|
value = rng.choice([-1000000,1000000,0,rng.randrange(-999999,1000000)])
|
|
weight = rng.choice([1,1000,rng.randrange(1,1001)])
|
|
flag = int((index+batch)%5 == 0)
|
|
add('ADD',channel,value,weight,flag)
|
|
if index in (0,31):
|
|
add('SNAPSHOT')
|
|
if size == 64:
|
|
add('ADD',8,0,0,2) # invalid outranks full
|
|
add('ADD',0,1,1,0)
|
|
add('SNAPSHOT')
|
|
add('CANCEL' if batch in (9,15) else 'COMMIT')
|
|
add('SNAPSHOT')
|
|
add('BEGIN',1000000)
|
|
for channel in range(8):
|
|
add('ADD',channel,-1000000,1000,0)
|
|
add('ADD',channel,1000000,1000,0)
|
|
add('ADD',channel,0,1,1)
|
|
add('COMMIT')
|
|
add('SNAPSHOT')
|
|
add('BEGIN',1000000)
|
|
add('SNAPSHOT')
|
|
return operations
|
|
|
|
|
|
def model(operations):
|
|
# Independent list-of-committed-records model; no candidate/gold execution.
|
|
committed = []
|
|
staged = None
|
|
active_id = 0
|
|
batches = 0
|
|
last = 0
|
|
result = []
|
|
for ordinal, operation in enumerate(operations,1):
|
|
opcode, *args = operation
|
|
rc = 0
|
|
if opcode == 'BEGIN':
|
|
batch = args[0]
|
|
if staged is not None: rc = 1
|
|
elif not 1 <= batch <= 1000000 or batch <= last: rc = 2
|
|
else: staged,active_id = [],batch
|
|
elif opcode == 'ADD':
|
|
channel,value,weight,flag = args
|
|
if staged is None: rc = 1
|
|
elif not (0<=channel<8 and -1000000<=value<=1000000 and 1<=weight<=1000 and flag in (0,1)): rc = 2
|
|
elif len(staged)==64: rc = 3
|
|
else: staged.append(tuple(args))
|
|
elif opcode in ('COMMIT','CANCEL'):
|
|
if staged is None: rc = 1
|
|
else:
|
|
if opcode == 'COMMIT':
|
|
committed.extend(staged)
|
|
batches += 1
|
|
last = active_id
|
|
staged,active_id = None,0
|
|
result.append(('R',ordinal,opcode,rc))
|
|
if opcode == 'SNAPSHOT':
|
|
accepted = [r for r in committed if r[3]==0]
|
|
ignored = sum(r[3] for r in committed)
|
|
result.append(('H',ordinal,len(accepted),ignored,batches,last,int(staged is not None),active_id,0 if staged is None else len(staged)))
|
|
for channel in range(8):
|
|
rows = [r for r in accepted if r[0]==channel]
|
|
values = [r[1] for r in rows]
|
|
result.append(('B',ordinal,channel,len(rows),sum(r[2] for r in rows),sum(r[1]*r[2] for r in rows),min(values) if values else 0,max(values) if values else 0))
|
|
return result
|
|
|
|
|
|
def normalize_integer(token):
|
|
if re.fullmatch(rb'[+-]?[0-9]+',token) is None:
|
|
raise ValueError('non-decimal stdout token')
|
|
negative = token.startswith(b'-')
|
|
magnitude = token.lstrip(b'+-').lstrip(b'0') or b'0'
|
|
# All observable values in this <=100000-operation contract fit 20 effective digits.
|
|
if len(magnitude)>20:
|
|
raise ValueError('stdout integer out of observable range')
|
|
value = int(magnitude)
|
|
return -value if negative else value
|
|
|
|
|
|
def compare(raw, expected):
|
|
lines = [line.strip(b' \t\r') for line in raw.split(b'\n') if line.strip(b' \t\r')]
|
|
if len(lines)!=len(expected):
|
|
raise ValueError('observation record count %d != %d'%(len(lines),len(expected)))
|
|
for index,(line,wanted) in enumerate(zip(lines,expected)):
|
|
tokens = re.split(rb'[ \t\r]+',line)
|
|
if len(tokens)!=len(wanted):
|
|
raise ValueError('observation width at row %d'%index)
|
|
observed = tuple(normalize_integer(token) if type(value) is int else token.decode('ascii') for token,value in zip(tokens,wanted))
|
|
if observed!=wanted:
|
|
raise ValueError('observation difference at row %d: %r != %r'%(index,observed,wanted))
|
|
|
|
|
|
def emit_evidence(evidence, scratch, failures, expected_rows, success):
|
|
document = {'scratch':str(scratch),'stages':evidence,'exceptions':failures,
|
|
'expected_rows':expected_rows,'behavior_success':success}
|
|
packed = base64.b64encode(bz2.compress(json.dumps(document,separators=(',',':')).encode())).decode('ascii')
|
|
sys.stderr.write('BF_EVIDENCE '+packed+'\n')
|
|
|
|
|
|
def main():
|
|
evidence = []
|
|
failures = []
|
|
success = False
|
|
root = None
|
|
expected = []
|
|
try:
|
|
operations = script_operations()
|
|
expected = model(operations)
|
|
payload = ('\n'.join(' '.join(map(str,op)) for op in operations)+'\n').encode('ascii')
|
|
candidate_bytes = (Path(sys.argv[1])/'src/ledger.c').read_bytes()
|
|
root = Path(tempfile.mkdtemp(prefix='batch-fold-'))
|
|
# Prepare everything before the first identity child. Never chmod the hook-owned root.
|
|
for name, content in SUPPORT.items():
|
|
target = root/name
|
|
target.parent.mkdir(parents=True,exist_ok=True)
|
|
target.write_text(content)
|
|
(root/'src/ledger.c').write_bytes(candidate_bytes)
|
|
for directory in (root/'src',root/'include'):
|
|
os.chmod(directory,0o755)
|
|
if os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534':
|
|
for name in [*SUPPORT,'src/ledger.c']:
|
|
os.chown(root/name,65534,65534)
|
|
for directory in (root/'src',root/'include'):
|
|
os.chown(directory,65534,65534)
|
|
compiler = ['gcc','-std=c11','-O2','-Wall','-Wextra','-Wpedantic','-Iinclude',
|
|
'src/main.c','src/text.c','src/report.c','src/ledger.c','-o','batch-fold']
|
|
execute_stage('compiler',compiler,root,COMPILER_SECONDS,b'',evidence)
|
|
actual,_ = execute_stage('candidate',[str(root/'batch-fold'),'-'],root,CANDIDATE_SECONDS,payload,evidence)
|
|
compare(actual,expected)
|
|
success = True
|
|
except BaseException as error:
|
|
failures.append(['main',type(error).__name__,str(error)])
|
|
finally:
|
|
if root is not None:
|
|
try:
|
|
execute_stage('cleanup',[sys.executable,'-I','-c',CLEANUP,str(root)],'/',CLEANUP_SECONDS,b'',evidence)
|
|
except BaseException as error:
|
|
failures.append(['cleanup',type(error).__name__,str(error)])
|
|
try:
|
|
execute_stage('absence',[sys.executable,'-I','-c',ABSENCE,str(root)],'/',ABSENCE_SECONDS,b'',evidence)
|
|
except BaseException as error:
|
|
failures.append(['absence',type(error).__name__,str(error)])
|
|
emit_evidence(evidence,root,failures,len(expected),success)
|
|
if failures or not success:
|
|
print('FAIL: '+repr(failures))
|
|
return 1
|
|
print('PASS')
|
|
return 0
|
|
|
|
if __name__=='__main__':
|
|
sys.exit(main())
|