amos 9d633e9e67 Batch 15 (20261023-C-hwe-scale-15): 320 HWE-derived repair tasks
One user-space repair task per eligible HWE debug-diag / compile-deploy base
ID: 320 admitted (C 218, C++ 97, build scripts 5), 1 skipped by the content
filter. Each ships a public spec with a check-to-basis table, at least two
defects with single-fix fixtures, a symptom-masking fixture and an alternative
correct implementation. Claude re-checked all 320: structure, the HWE quality
items and trace6 Docker calibration receipts against current digests
(320/320 USABLE). Difficulty is unmeasured. Known follow-up: 212 tasks use 120
ad-hoc family names; the next batch maps them to the canonical list.
The batch's 1,256-line progress log stays in its factory report and the
36-local branch batch15-hwe.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 01:55:56 +08:00

460 lines
30 KiB
Python

import base64
import bz2
import hashlib
import json
import math
import os
from pathlib import Path
import random
import re
import subprocess
import sys
import tempfile
import time
RECOVERY_SECONDS = 3
RECORD_WIRE = False
SUPERVISOR = r'''
import base64,bz2,ctypes,hashlib,json,os,signal,subprocess,sys,time
cfg=json.loads(sys.argv[1]); start=time.monotonic(); errors=[]; tracked={}; signals={}; waits=[]
leader=None; leader_start=None; output=b''; diagnostic=b''; timed_out=False; drained=False
reaped=False; leader_rc=None; remaining=[]; pg_absent=False
def fault(op, error):
errors.append([op,type(error).__name__,str(error)])
def exact(pid):
try:
with open('/proc/%d/stat'%pid,'rb') as stream: data=stream.read()
words=data[data.rfind(b')')+2:].split()
return {'pid':pid,'state':words[0].decode('ascii'),'ppid':int(words[1]),'pg':int(words[2]),'start':int(words[19])}
except (FileNotFoundError,ProcessLookupError): return None
def table_now():
result={}
for name in os.listdir('/proc'):
if name.isascii() and name.isdigit():
value=exact(int(name))
if value is not None: result[value['pid']]=value
return result
def closure(table, supervisor, remembered):
roots={supervisor}
for pid,old in remembered.items():
current=table.get(pid)
if current and current['start']==old['start'] and current['state'] not in ('Z','X'):
roots.add(pid)
found={}
changed=True
while changed:
changed=False
for pid,current in table.items():
if pid!=supervisor and pid not in found and (current['ppid'] in roots or pid in roots):
found[pid]=current; roots.add(pid); changed=True
return found
def discover():
table=table_now(); found=closure(table,os.getpid(),tracked)
for pid,current in found.items():
if pid not in tracked or tracked[pid]['start']!=current['start']:
tracked[pid]=dict(current)
return table,found
def send_fresh(pid, token):
current=exact(pid)
if current is None or current['start']!=token['start'] or current['state'] in ('Z','X'): return
os.kill(pid,signal.SIGKILL)
signals[(pid,token['start'])]=signals.get((pid,token['start']),0)+1
def own_wait():
global reaped,leader_rc
if leader is not None:
value=leader.poll()
if value is not None:
leader_rc=leader.wait(timeout=0)
reaped=True
def adopted_wait():
if leader is not None and not reaped: return
while True:
try: pid,status=os.waitpid(-1,os.WNOHANG)
except ChildProcessError: break
if pid==0: break
waits.append([pid,None,os.waitstatus_to_exitcode(status)])
supervisor_start=exact(os.getpid())['start']
try:
libc=ctypes.CDLL(None,use_errno=True)
if libc.prctl(36,1,0,0,0)!=0: raise OSError(ctypes.get_errno(),'subreaper')
leader=subprocess.Popen(cfg['argv'],cwd=cfg['cwd'],stdin=subprocess.PIPE,
stdout=subprocess.PIPE,stderr=subprocess.PIPE,start_new_session=True)
info=exact(leader.pid)
if info is None: raise RuntimeError('leader identity disappeared before capture')
leader_start=info['start']; tracked[leader.pid]=info
payload=sys.stdin.buffer.read()
first=True
while True:
try:
output,diagnostic=leader.communicate(input=payload if first else None,timeout=0.04)
drained=True; own_wait(); break
except subprocess.TimeoutExpired:
first=False
own_wait()
discover()
if time.monotonic()-start>=cfg['deadline']:
timed_out=True; break
except BaseException as error:
fault('execute',error)
finally:
recovery_end=time.monotonic()+cfg['recovery']
while True:
try: own_wait()
except BaseException as error: fault('own_wait',error)
found={}
try: _,found=discover()
except BaseException as error: fault('discover',error)
for pid,token in list(found.items()):
try: send_fresh(pid,token)
except (FileNotFoundError,ProcessLookupError): pass
except BaseException as error: fault('signal',error)
try: own_wait()
except BaseException as error: fault('own_wait',error)
try: adopted_wait()
except BaseException as error: fault('adopted_wait',error)
if leader is not None and not drained:
try:
output,diagnostic=leader.communicate(timeout=0.03)
drained=True; own_wait()
except subprocess.TimeoutExpired: pass
except BaseException as error: fault('drain',error)
try:
table,found=discover()
remaining=[[p,v['start'],v['state']] for p,v in found.items()]
pg_absent=leader is not None and not any(v['pg']==leader.pid for v in table.values())
except BaseException as error:
fault('absence',error); remaining=[['unknown']]; pg_absent=False
if reaped and drained and not remaining and pg_absent: break
if time.monotonic()>=recovery_end:
fault('recovery',RuntimeError('deadline with unconfirmed closure')); break
time.sleep(0.01)
meta={'phase':cfg['phase'],'nonce':cfg['nonce'],'argv':cfg['argv'],
'uid':os.getuid(),'gid':os.getgid(),'groups':os.getgroups(),
'supervisor_pid':os.getpid(),'supervisor_start':supervisor_start,
'leader_pid':None if leader is None else leader.pid,'leader_start':leader_start,
'leader_rc':leader_rc,'leader_reaped':reaped,'timeout':timed_out,
'controlled_rc':124 if timed_out else leader_rc,'elapsed':time.monotonic()-start,
'deadline':cfg['deadline'],'recovery':cfg['recovery'],
'pg_absent':pg_absent,'remaining':remaining,'exceptions':errors,
'adopted_waits':waits,
'descendants':[[p,t['start'],signals.get((p,t['start']),0)]
for p,t in tracked.items() if leader is None or p!=leader.pid],
'stdout_len':len(output),'stdout_sha256':hashlib.sha256(output).hexdigest(),
'stderr_len':len(diagnostic),'stderr_sha256':hashlib.sha256(diagnostic).hexdigest()}
wire={'meta':meta,'stdout':base64.b64encode(bz2.compress(output)).decode('ascii'),
'stderr':base64.b64encode(bz2.compress(diagnostic)).decode('ascii')}
sys.stdout.write(json.dumps(wire,separators=(',',':'))+'\n')
'''
CLEANUP = "import shutil,sys; shutil.rmtree(sys.argv[1])"
ABSENCE = "import os,sys\ntry: os.lstat(sys.argv[1])\nexcept FileNotFoundError: sys.exit(0)\nsys.exit(1)"
def identity_int(value, minimum=0):
return type(value) is int and value >= minimum
def validate_meta(meta, config, out, err):
required = {'phase','nonce','argv','uid','gid','groups','supervisor_pid','supervisor_start',
'leader_pid','leader_start','leader_rc','leader_reaped','timeout','controlled_rc',
'elapsed','deadline','recovery','pg_absent','remaining','exceptions','descendants',
'stdout_len','stdout_sha256','stderr_len','stderr_sha256','adopted_waits'}
if type(meta) is not dict or not required <= meta.keys():
raise ValueError('missing stage metadata')
if meta['phase'] != config['phase'] or meta['nonce'] != config['nonce'] or meta['argv'] != config['argv']:
raise ValueError('stage binding conflict')
for name in ('uid','gid','supervisor_pid','supervisor_start','leader_pid','leader_start'):
if not identity_int(meta[name], 1 if name not in ('uid','gid') else 0):
raise ValueError('invalid identity '+name)
if meta['supervisor_pid'] == meta['leader_pid']:
raise ValueError('supervisor and leader identical')
isolated = os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534'
expected_identity = (65534,65534,[]) if isolated else (os.getuid(),os.getgid(),os.getgroups())
if type(meta['groups']) is not list or any(type(x) is not int or x < 0 for x in meta['groups']):
raise ValueError('invalid groups')
if (meta['uid'],meta['gid'],meta['groups']) != expected_identity:
raise ValueError('execution identity conflict')
if type(meta['deadline']) is not int or meta['deadline'] != config['deadline']:
raise ValueError('deadline conflict')
if type(meta['recovery']) is not int or meta['recovery'] != RECOVERY_SECONDS:
raise ValueError('recovery conflict')
elapsed = meta['elapsed']
if type(elapsed) not in (int,float) or not math.isfinite(elapsed) or not 0 <= elapsed <= config['deadline']+RECOVERY_SECONDS+2:
raise ValueError('invalid elapsed')
for name in ('timeout','leader_reaped','pg_absent'):
if type(meta[name]) is not bool:
raise ValueError('invalid boolean '+name)
for name in ('leader_rc','controlled_rc'):
if type(meta[name]) is not int:
raise ValueError('invalid return code '+name)
if meta['controlled_rc'] != (124 if meta['timeout'] else meta['leader_rc']):
raise ValueError('return code conflict')
for name in ('remaining','exceptions','descendants','adopted_waits'):
if type(meta[name]) is not list:
raise ValueError('invalid list '+name)
for row in meta['descendants']:
if type(row) is not list or len(row) != 3 or not all(identity_int(x,1) for x in row[:2]) or not identity_int(row[2]):
raise ValueError('invalid descendant record')
for row in meta['adopted_waits']:
if type(row) is not list or len(row) != 3 or not identity_int(row[0],1) or row[1] is not None or type(row[2]) is not int:
raise ValueError('invalid adopted wait fact')
for name,data in (('stdout',out),('stderr',err)):
if not identity_int(meta[name+'_len']) or meta[name+'_len'] != len(data):
raise ValueError('invalid IO length')
if type(meta[name+'_sha256']) is not str or meta[name+'_sha256'] != hashlib.sha256(data).hexdigest():
raise ValueError('invalid IO digest')
if meta['exceptions'] or meta['remaining'] or not meta['leader_reaped'] or not meta['pg_absent']:
raise ValueError('stage recovery or exception failure')
if meta['timeout'] or meta['leader_rc'] != 0 or meta['controlled_rc'] != 0:
raise ValueError('controlled timeout' if meta['timeout'] else 'leader nonzero exit')
def execute_stage(phase, argv, cwd, deadline, payload, evidence):
nonce = os.urandom(8).hex()
config = dict(phase=phase,argv=argv,cwd=str(cwd),deadline=deadline,recovery=RECOVERY_SECONDS,nonce=nonce)
wrapper_argv = [sys.executable,'-I','-c',SUPERVISOR,json.dumps(config,separators=(',',':'))]
raw = subprocess.run(wrapper_argv,input=payload,capture_output=True,timeout=deadline+RECOVERY_SECONDS+5)
# Preserve actual wrapper facts before touching or trusting its wire protocol.
entry = dict(phase=phase,nonce=nonce,wrapper_rc=raw.returncode,
wire_len=len(raw.stdout),wire_sha256=hashlib.sha256(raw.stdout).hexdigest(),
wrapper_stderr_len=len(raw.stderr),wrapper_stderr_sha256=hashlib.sha256(raw.stderr).hexdigest())
if RECORD_WIRE:
entry['wire_ascii'] = raw.stdout.decode('ascii')
entry['wrapper_stderr_bz2'] = base64.b64encode(bz2.compress(raw.stderr)).decode('ascii')
evidence.append(entry)
if raw.returncode != 0:
raise ValueError(phase+' supervisor nonzero')
wire = json.loads(raw.stdout)
out = bz2.decompress(base64.b64decode(wire['stdout'],validate=True))
err = bz2.decompress(base64.b64decode(wire['stderr'],validate=True))
meta = wire['meta']
if not RECORD_WIRE: entry['meta'] = meta
validate_meta(meta,config,out,err)
return out,err
SUPPORT = {'include/batch.h': '#ifndef BATCH_H\n#define BATCH_H\n#include <stddef.h>\n#include <stdint.h>\n#include <stdio.h>\n#define BF_CHANNELS 8u\n#define BF_CAPACITY 64u\n#define BF_MAX_BATCH 1000000u\n#define BF_MIN_VALUE (-1000000)\n#define BF_MAX_VALUE 1000000\n#define BF_MAX_WEIGHT 1000u\n#define BF_LINE_SIZE 1024u\ntypedef enum {\n BF_OK = 0,\n BF_STATE = 1,\n BF_RANGE = 2,\n BF_FULL = 3\n} bf_result;\ntypedef struct {\n uint32_t channel;\n int32_t value;\n uint32_t weight;\n uint32_t filtered;\n} bf_row;\ntypedef struct {\n uint64_t rows;\n uint64_t units;\n int64_t weighted_total;\n int32_t minimum;\n int32_t maximum;\n} bf_bucket;\ntypedef struct {\n bf_bucket channels[BF_CHANNELS];\n uint64_t accepted;\n uint64_t ignored;\n uint32_t batches;\n uint32_t last_batch;\n} bf_totals;\ntypedef struct {\n bf_totals committed;\n bf_row staged[BF_CAPACITY];\n size_t staged_count;\n uint32_t active_batch;\n int active;\n} bf_ledger;\ntypedef struct {\n bf_totals totals;\n size_t staged_count;\n uint32_t active_batch;\n int active;\n} bf_view;\ntypedef enum {\n BF_CMD_BEGIN,\n BF_CMD_ADD,\n BF_CMD_COMMIT,\n BF_CMD_CANCEL,\n BF_CMD_SNAPSHOT,\n BF_CMD_INVALID\n} bf_opcode;\ntypedef struct {\n bf_opcode opcode;\n uint32_t batch;\n bf_row row;\n} bf_command;\ntypedef enum {\n BF_PARSE_OK,\n BF_PARSE_EMPTY,\n BF_PARSE_INVALID\n} bf_parse_result;\nvoid bf_init(bf_ledger *ledger);\nbf_result bf_begin(bf_ledger *ledger, uint32_t batch);\nbf_result bf_add(bf_ledger *ledger, bf_row row);\nbf_result bf_commit(bf_ledger *ledger);\nbf_result bf_cancel(bf_ledger *ledger);\nvoid bf_snapshot(const bf_ledger *ledger, bf_view *view);\nbf_parse_result bf_parse_line(const char *line, bf_command *command);\nint bf_read_line(FILE *stream, char *line, size_t capacity);\nconst char *bf_operation_name(bf_opcode opcode);\nvoid bf_print_result(FILE *stream, size_t ordinal, bf_opcode opcode, bf_result result);\nvoid bf_print_view(FILE *stream, size_t ordinal, const bf_view *view);\nint bf_execute(FILE *input, FILE *output, FILE *diagnostic);\n#endif\n', 'src/text.c': '#include "batch.h"\n#include <limits.h>\n#include <string.h>\n\ntypedef struct {\n const char *first;\n size_t length;\n} token;\n\nstatic int horizontal(unsigned char character)\n{\n return character == \' \' || character == \'\\t\' || character == \'\\r\';\n}\n\nstatic size_t tokenize(const char *line, token *tokens, size_t capacity)\n{\n size_t count = 0;\n const char *cursor = line;\n while (*cursor != \'\\0\') {\n const char *begin;\n while (horizontal((unsigned char)*cursor)) {\n ++cursor;\n }\n if (*cursor == \'\\0\') {\n break;\n }\n if (count == capacity) {\n return capacity + 1;\n }\n begin = cursor;\n while (*cursor != \'\\0\' && !horizontal((unsigned char)*cursor)) {\n ++cursor;\n }\n tokens[count].first = begin;\n tokens[count].length = (size_t)(cursor - begin);\n ++count;\n }\n return count;\n}\n\nstatic int literal(token value, const char *word)\n{\n size_t length = strlen(word);\n return value.length == length && memcmp(value.first, word, length) == 0;\n}\n\nstatic int number(token value, int64_t minimum, int64_t maximum, int64_t *result)\n{\n size_t offset = 0;\n uint64_t magnitude = 0;\n int negative = 0;\n int64_t signed_value;\n if (value.length == 0) {\n return 0;\n }\n if (value.first[offset] == \'-\' || value.first[offset] == \'+\') {\n negative = value.first[offset] == \'-\';\n ++offset;\n }\n if (offset == value.length) {\n return 0;\n }\n for (; offset < value.length; ++offset) {\n unsigned char character = (unsigned char)value.first[offset];\n unsigned digit;\n if (character < \'0\' || character > \'9\') {\n return 0;\n }\n digit = (unsigned)(character - \'0\');\n if (magnitude > (UINT64_MAX - digit) / 10u) {\n return 0;\n }\n magnitude = magnitude * 10u + digit;\n }\n if (magnitude > INT64_MAX) {\n return 0;\n }\n signed_value = (int64_t)magnitude;\n if (negative) {\n signed_value = -signed_value;\n }\n if (signed_value < minimum || signed_value > maximum) {\n return 0;\n }\n *result = signed_value;\n return 1;\n}\n\nstatic int parse_add(const token *tokens, bf_command *command)\n{\n int64_t channel;\n int64_t value;\n int64_t weight;\n int64_t filtered;\n if (!number(tokens[1], 0, UINT32_MAX, &channel)) {\n return 0;\n }\n if (!number(tokens[2], INT32_MIN, INT32_MAX, &value)) {\n return 0;\n }\n if (!number(tokens[3], 0, UINT32_MAX, &weight)) {\n return 0;\n }\n if (!number(tokens[4], 0, UINT32_MAX, &filtered)) {\n return 0;\n }\n command->row.channel = (uint32_t)channel;\n command->row.value = (int32_t)value;\n command->row.weight = (uint32_t)weight;\n command->row.filtered = (uint32_t)filtered;\n command->opcode = BF_CMD_ADD;\n return 1;\n}\n\nbf_parse_result bf_parse_line(const char *line, bf_command *command)\n{\n token tokens[5];\n size_t count = tokenize(line, tokens, 5);\n int64_t batch;\n memset(command, 0, sizeof(*command));\n command->opcode = BF_CMD_INVALID;\n if (count == 0) {\n return BF_PARSE_EMPTY;\n }\n if (count == 2 && literal(tokens[0], "BEGIN")) {\n if (!number(tokens[1], 0, UINT32_MAX, &batch)) {\n return BF_PARSE_INVALID;\n }\n command->opcode = BF_CMD_BEGIN;\n command->batch = (uint32_t)batch;\n return BF_PARSE_OK;\n }\n if (count == 5 && literal(tokens[0], "ADD")) {\n return parse_add(tokens, command) ? BF_PARSE_OK : BF_PARSE_INVALID;\n }\n if (count != 1) {\n return BF_PARSE_INVALID;\n }\n if (literal(tokens[0], "COMMIT")) {\n command->opcode = BF_CMD_COMMIT;\n } else if (literal(tokens[0], "CANCEL")) {\n command->opcode = BF_CMD_CANCEL;\n } else if (literal(tokens[0], "SNAPSHOT")) {\n command->opcode = BF_CMD_SNAPSHOT;\n } else {\n return BF_PARSE_INVALID;\n }\n return BF_PARSE_OK;\n}\n\nint bf_read_line(FILE *stream, char *line, size_t capacity)\n{\n size_t length = 0;\n int character;\n int too_long = 0;\n while ((character = fgetc(stream)) != EOF) {\n if (character == \'\\n\') {\n break;\n }\n if (length + 1 < capacity) {\n line[length++] = (char)character;\n } else {\n too_long = 1;\n }\n if (character == 0) {\n too_long = 1;\n }\n }\n if (ferror(stream)) {\n return -1;\n }\n if (character == EOF && length == 0 && !too_long) {\n return 0;\n }\n line[length] = \'\\0\';\n return too_long ? -1 : 1;\n}\n', 'src/report.c': '#include "batch.h"\n#include <inttypes.h>\n\nconst char *bf_operation_name(bf_opcode opcode)\n{\n switch (opcode) {\n case BF_CMD_BEGIN:\n return "BEGIN";\n case BF_CMD_ADD:\n return "ADD";\n case BF_CMD_COMMIT:\n return "COMMIT";\n case BF_CMD_CANCEL:\n return "CANCEL";\n case BF_CMD_SNAPSHOT:\n return "SNAPSHOT";\n default:\n return "INVALID";\n }\n}\n\nvoid bf_print_result(FILE *stream, size_t ordinal, bf_opcode opcode, bf_result result)\n{\n fprintf(stream, "R %zu %s %d\\n", ordinal, bf_operation_name(opcode), (int)result);\n}\n\nstatic void print_bucket(FILE *stream, size_t ordinal, size_t channel, const bf_bucket *bucket)\n{\n fprintf(stream,\n "B %zu %zu %" PRIu64 " %" PRIu64 " %" PRId64 " %" PRId32 " %" PRId32 "\\n",\n ordinal,\n channel,\n bucket->rows,\n bucket->units,\n bucket->weighted_total,\n bucket->minimum,\n bucket->maximum);\n}\n\nvoid bf_print_view(FILE *stream, size_t ordinal, const bf_view *view)\n{\n size_t channel;\n fprintf(stream,\n "H %zu %" PRIu64 " %" PRIu64 " %" PRIu32 " %" PRIu32 " %d %" PRIu32 " %zu\\n",\n ordinal,\n view->totals.accepted,\n view->totals.ignored,\n view->totals.batches,\n view->totals.last_batch,\n view->active,\n view->active_batch,\n view->staged_count);\n for (channel = 0; channel < BF_CHANNELS; ++channel) {\n print_bucket(stream, ordinal, channel, &view->totals.channels[channel]);\n }\n}\n', 'src/main.c': '#include "batch.h"\n#include <string.h>\n\nstatic bf_result dispatch(bf_ledger *ledger, const bf_command *command)\n{\n switch (command->opcode) {\n case BF_CMD_BEGIN:\n return bf_begin(ledger, command->batch);\n case BF_CMD_ADD:\n return bf_add(ledger, command->row);\n case BF_CMD_COMMIT:\n return bf_commit(ledger);\n case BF_CMD_CANCEL:\n return bf_cancel(ledger);\n case BF_CMD_SNAPSHOT:\n return BF_OK;\n default:\n return BF_RANGE;\n }\n}\n\nint bf_execute(FILE *input, FILE *output, FILE *diagnostic)\n{\n bf_ledger ledger;\n char line[BF_LINE_SIZE];\n size_t ordinal = 0;\n int read_status;\n bf_init(&ledger);\n while ((read_status = bf_read_line(input, line, sizeof(line))) != 0) {\n bf_command command;\n bf_parse_result parsed;\n bf_result result;\n if (read_status < 0) {\n fprintf(diagnostic, "input line cannot be read\\n");\n return 2;\n }\n parsed = bf_parse_line(line, &command);\n if (parsed == BF_PARSE_EMPTY) {\n continue;\n }\n if (parsed != BF_PARSE_OK) {\n fprintf(diagnostic, "invalid command after %zu operations\\n", ordinal);\n return 2;\n }\n ++ordinal;\n result = dispatch(&ledger, &command);\n bf_print_result(output, ordinal, command.opcode, result);\n if (command.opcode == BF_CMD_SNAPSHOT) {\n bf_view view;\n bf_snapshot(&ledger, &view);\n bf_print_view(output, ordinal, &view);\n }\n if (ferror(output)) {\n fprintf(diagnostic, "output failure\\n");\n return 2;\n }\n }\n if (fflush(output) != 0) {\n fprintf(diagnostic, "output flush failure\\n");\n return 2;\n }\n return 0;\n}\n\nint main(int argc, char **argv)\n{\n FILE *input = stdin;\n int status;\n if (argc > 2) {\n fprintf(stderr, "usage: batch-fold [script|-]\\n");\n return 2;\n }\n if (argc == 2 && strcmp(argv[1], "-") != 0) {\n input = fopen(argv[1], "rb");\n if (input == NULL) {\n fprintf(stderr, "cannot open batch script\\n");\n return 2;\n }\n }\n status = bf_execute(input, stdout, stderr);\n if (input != stdin && fclose(input) != 0) {\n fprintf(stderr, "input close failure\\n");\n status = 2;\n }\n return status;\n}\n'}
COMPILER_SECONDS = 15
CANDIDATE_SECONDS = 10
CLEANUP_SECONDS = 6
ABSENCE_SECONDS = 6
def script_operations():
operations = []
def add(*parts):
operations.append(tuple(parts))
add('SNAPSHOT')
add('ADD',0,1,1,0)
add('COMMIT')
add('CANCEL')
add('BEGIN',0)
add('BEGIN',1000001)
add('BEGIN',1)
add('BEGIN',2)
add('ADD',8,0,1,0)
add('ADD',0,-1000001,1,0)
add('ADD',0,1000001,1,0)
add('ADD',0,0,0,0)
add('ADD',0,0,1001,0)
add('ADD',0,0,1,2)
add('SNAPSHOT')
add('COMMIT')
add('SNAPSHOT')
add('BEGIN',1)
add('BEGIN',2)
add('ADD',0,12,2,0)
add('ADD',1,-5,3,0)
add('ADD',0,8,1,0)
add('SNAPSHOT')
add('COMMIT')
add('SNAPSHOT')
# A canceled nonempty batch and re-use of its ID exercise D2 independently.
add('BEGIN',3)
add('ADD',4,-93,11,0)
add('ADD',7,22,7,1)
add('SNAPSHOT')
add('CANCEL')
add('SNAPSHOT')
add('BEGIN',3)
add('CANCEL')
add('SNAPSHOT')
rng = random.Random(0xBA7C4)
for batch in range(4,22):
add('BEGIN',batch)
size = [1,2,7,64,0,13][(batch-4)%6]
for index in range(size):
channel = (index+batch)%8
value = rng.choice([-1000000,1000000,0,rng.randrange(-999999,1000000)])
weight = rng.choice([1,1000,rng.randrange(1,1001)])
flag = int((index+batch)%5 == 0)
add('ADD',channel,value,weight,flag)
if index in (0,31):
add('SNAPSHOT')
if size == 64:
add('ADD',8,0,0,2) # invalid outranks full
add('ADD',0,1,1,0)
add('SNAPSHOT')
add('CANCEL' if batch in (9,15) else 'COMMIT')
add('SNAPSHOT')
add('BEGIN',1000000)
for channel in range(8):
add('ADD',channel,-1000000,1000,0)
add('ADD',channel,1000000,1000,0)
add('ADD',channel,0,1,1)
add('COMMIT')
add('SNAPSHOT')
add('BEGIN',1000000)
add('SNAPSHOT')
return operations
def model(operations):
# Independent list-of-committed-records model; no candidate/gold execution.
committed = []
staged = None
active_id = 0
batches = 0
last = 0
result = []
for ordinal, operation in enumerate(operations,1):
opcode, *args = operation
rc = 0
if opcode == 'BEGIN':
batch = args[0]
if staged is not None: rc = 1
elif not 1 <= batch <= 1000000 or batch <= last: rc = 2
else: staged,active_id = [],batch
elif opcode == 'ADD':
channel,value,weight,flag = args
if staged is None: rc = 1
elif not (0<=channel<8 and -1000000<=value<=1000000 and 1<=weight<=1000 and flag in (0,1)): rc = 2
elif len(staged)==64: rc = 3
else: staged.append(tuple(args))
elif opcode in ('COMMIT','CANCEL'):
if staged is None: rc = 1
else:
if opcode == 'COMMIT':
committed.extend(staged)
batches += 1
last = active_id
staged,active_id = None,0
result.append(('R',ordinal,opcode,rc))
if opcode == 'SNAPSHOT':
accepted = [r for r in committed if r[3]==0]
ignored = sum(r[3] for r in committed)
result.append(('H',ordinal,len(accepted),ignored,batches,last,int(staged is not None),active_id,0 if staged is None else len(staged)))
for channel in range(8):
rows = [r for r in accepted if r[0]==channel]
values = [r[1] for r in rows]
result.append(('B',ordinal,channel,len(rows),sum(r[2] for r in rows),sum(r[1]*r[2] for r in rows),min(values) if values else 0,max(values) if values else 0))
return result
def normalize_integer(token):
if re.fullmatch(rb'[+-]?[0-9]+',token) is None:
raise ValueError('non-decimal stdout token')
negative = token.startswith(b'-')
magnitude = token.lstrip(b'+-').lstrip(b'0') or b'0'
# All observable values in this <=100000-operation contract fit 20 effective digits.
if len(magnitude)>20:
raise ValueError('stdout integer out of observable range')
value = int(magnitude)
return -value if negative else value
def compare(raw, expected):
lines = [line.strip(b' \t\r') for line in raw.split(b'\n') if line.strip(b' \t\r')]
if len(lines)!=len(expected):
raise ValueError('observation record count %d != %d'%(len(lines),len(expected)))
for index,(line,wanted) in enumerate(zip(lines,expected)):
tokens = re.split(rb'[ \t\r]+',line)
if len(tokens)!=len(wanted):
raise ValueError('observation width at row %d'%index)
observed = tuple(normalize_integer(token) if type(value) is int else token.decode('ascii') for token,value in zip(tokens,wanted))
if observed!=wanted:
raise ValueError('observation difference at row %d: %r != %r'%(index,observed,wanted))
def emit_evidence(evidence, scratch, failures, expected_rows, success):
document = {'scratch':str(scratch),'stages':evidence,'exceptions':failures,
'expected_rows':expected_rows,'behavior_success':success}
packed = base64.b64encode(bz2.compress(json.dumps(document,separators=(',',':')).encode())).decode('ascii')
sys.stderr.write('BF_EVIDENCE '+packed+'\n')
def main():
evidence = []
failures = []
success = False
root = None
expected = []
try:
operations = script_operations()
expected = model(operations)
payload = ('\n'.join(' '.join(map(str,op)) for op in operations)+'\n').encode('ascii')
candidate_bytes = (Path(sys.argv[1])/'src/ledger.c').read_bytes()
root = Path(tempfile.mkdtemp(prefix='batch-fold-'))
# Prepare everything before the first identity child. Never chmod the hook-owned root.
for name, content in SUPPORT.items():
target = root/name
target.parent.mkdir(parents=True,exist_ok=True)
target.write_text(content)
(root/'src/ledger.c').write_bytes(candidate_bytes)
for directory in (root/'src',root/'include'):
os.chmod(directory,0o755)
if os.environ.get('ROLLOUT_GRADE_ISOLATION') == 'uid65534':
for name in [*SUPPORT,'src/ledger.c']:
os.chown(root/name,65534,65534)
for directory in (root/'src',root/'include'):
os.chown(directory,65534,65534)
compiler = ['gcc','-std=c11','-O2','-Wall','-Wextra','-Wpedantic','-Iinclude',
'src/main.c','src/text.c','src/report.c','src/ledger.c','-o','batch-fold']
execute_stage('compiler',compiler,root,COMPILER_SECONDS,b'',evidence)
actual,_ = execute_stage('candidate',[str(root/'batch-fold'),'-'],root,CANDIDATE_SECONDS,payload,evidence)
compare(actual,expected)
success = True
except BaseException as error:
failures.append(['main',type(error).__name__,str(error)])
finally:
if root is not None:
try:
execute_stage('cleanup',[sys.executable,'-I','-c',CLEANUP,str(root)],'/',CLEANUP_SECONDS,b'',evidence)
except BaseException as error:
failures.append(['cleanup',type(error).__name__,str(error)])
try:
execute_stage('absence',[sys.executable,'-I','-c',ABSENCE,str(root)],'/',ABSENCE_SECONDS,b'',evidence)
except BaseException as error:
failures.append(['absence',type(error).__name__,str(error)])
emit_evidence(evidence,root,failures,len(expected),success)
if failures or not success:
print('FAIL: '+repr(failures))
return 1
print('PASS')
return 0
if __name__=='__main__':
sys.exit(main())