100 tasks from the HWE arm-linux code-gen pool (outside the lk/man/pth range given to batch 22), each with at least one independent contract-edge defect and plain public docs. Claude reviewed 100/100 USABLE against current digests; no ID or source overlap with main or batch 22. Difficulty is unmeasured. progress.md records the wrap-up: all Codex sessions and Qwen schedulers stopped for the user's rollout, Qwen results so far (57 HARD of 373), and what is left on each batch branch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
487 lines
22 KiB
Python
487 lines
22 KiB
Python
"""Independent byte/state model; candidate execution is confined to subprocesses."""
|
|
import collections
|
|
import json
|
|
from pathlib import Path
|
|
import random
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
MASK = (1 << 64) - 1
|
|
CANDIDATES = ('stream.hpp', 'stream.cpp', 'byte_codec.cpp', 'control_packet.cpp')
|
|
|
|
def hx(value):
|
|
return value.hex() or '-'
|
|
|
|
def unhex(value):
|
|
return b'' if value == '-' else bytes.fromhex(value)
|
|
|
|
def le(value):
|
|
return int(value).to_bytes(8, 'little')
|
|
|
|
def xform(value, mode):
|
|
table = bytes((x - 32 if mode == 1 and 97 <= x <= 122 else
|
|
x + 32 if mode == 2 and 65 <= x <= 90 else x) for x in range(256))
|
|
return value.translate(table)
|
|
|
|
def packet(op, *values, data=None):
|
|
payload = b''.join(le(v) for v in values)
|
|
if data is not None:
|
|
payload += le(len(data)) + data
|
|
return b'MS\x01' + bytes([op]) + payload
|
|
|
|
class Model:
|
|
def __init__(self):
|
|
self.cap = 16
|
|
self.mode = 0
|
|
self.fifo = b''
|
|
self.transactions = {}
|
|
self.counters = [0, 0, 0, 0]
|
|
|
|
@staticmethod
|
|
def valid_id(value):
|
|
return 1 <= value <= 4294967295
|
|
|
|
def state(self):
|
|
return self.counters + [len(self.fifo), len(self.transactions)]
|
|
|
|
def op(self, op, *args):
|
|
if op == 'Z':
|
|
cap, = args
|
|
if not 1 <= cap <= 4096:
|
|
return [-22]
|
|
self.__init__()
|
|
self.cap = cap
|
|
return [0]
|
|
if op == 'M':
|
|
mode, = args
|
|
if mode not in (0, 1, 2):
|
|
return [-22]
|
|
self.mode = mode
|
|
return [0]
|
|
if op == 'G':
|
|
return [self.mode]
|
|
if op == 'C':
|
|
return [self.cap]
|
|
if op == 'S':
|
|
return self.state()
|
|
if op == 'L':
|
|
ids = sorted(self.transactions)
|
|
return [len(ids)] + ids
|
|
if op in ('B', 'A', 'F', 'X', 'I'):
|
|
ident = args[0]
|
|
status = 0 if self.valid_id(ident) else -22
|
|
if status == 0 and op != 'B' and ident not in self.transactions:
|
|
status = -2
|
|
if status:
|
|
if op == 'A':
|
|
return [status, 0]
|
|
if op == 'I':
|
|
return [status, 0, 0, 0, 0, 0]
|
|
return [status]
|
|
if op == 'B':
|
|
total = args[1]
|
|
if total > 65535:
|
|
return [-7]
|
|
if ident in self.transactions:
|
|
return [-17]
|
|
if len(self.transactions) == 4:
|
|
return [-28]
|
|
self.transactions[ident] = [total, 0, self.mode]
|
|
return [0]
|
|
transaction = self.transactions[ident]
|
|
total, done, captured = transaction
|
|
if op == 'I':
|
|
return [0, ident, total, done, total - done, captured]
|
|
if op == 'F':
|
|
if done != total:
|
|
return [-115]
|
|
del self.transactions[ident]
|
|
self.counters[2] = (self.counters[2] + 1) & MASK
|
|
return [0]
|
|
if op == 'X':
|
|
del self.transactions[ident]
|
|
self.counters[3] = (self.counters[3] + 1) & MASK
|
|
return [0]
|
|
budget, data = args[1:]
|
|
if budget > 4096:
|
|
return [-22, 0]
|
|
if len(data) > 65535:
|
|
return [-7, 0]
|
|
if len(data) > total - done:
|
|
return [-90, 0]
|
|
accepted = min(len(data), budget, self.cap - len(self.fifo))
|
|
self.fifo += xform(data[:accepted], captured)
|
|
transaction[1] += accepted
|
|
self.counters[0] = (self.counters[0] + accepted) & MASK
|
|
return [0, accepted]
|
|
if op in ('R', 'P'):
|
|
count, = args
|
|
if count > 4096:
|
|
return [-22, 0, b'']
|
|
if count == 0:
|
|
return [0, 0, b'']
|
|
if not self.fifo:
|
|
return [-11, 0, b'']
|
|
data = self.fifo[:count]
|
|
if op == 'R':
|
|
self.fifo = self.fifo[len(data):]
|
|
self.counters[1] = (self.counters[1] + len(data)) & MASK
|
|
return [0, len(data), data]
|
|
raise ValueError(op)
|
|
|
|
def wire(self, raw):
|
|
if not 4 <= len(raw) <= 131072 or raw[:3] != b'MS\x01':
|
|
return [-71, b'']
|
|
opcode = raw[3]
|
|
names = {1: 'Z', 2: 'M', 3: 'G', 4: 'B', 5: 'A', 6: 'F', 7: 'X',
|
|
8: 'R', 9: 'P', 10: 'S', 11: 'I', 12: 'L', 13: 'C'}
|
|
if opcode not in names:
|
|
return [-25, b'']
|
|
body = raw[4:]
|
|
if opcode == 5:
|
|
if len(body) < 24:
|
|
return [-71, b'']
|
|
ident, budget, length = [int.from_bytes(body[i:i+8], 'little') for i in (0, 8, 16)]
|
|
if length > 65535 or length != len(body) - 24:
|
|
return [-71, b'']
|
|
result = self.op('A', ident, budget, body[24:])
|
|
else:
|
|
arity = {1: 1, 2: 1, 3: 0, 4: 2, 6: 1, 7: 1, 8: 1, 9: 1, 10: 0, 11: 1, 12: 0, 13: 0}[opcode]
|
|
if len(body) != 8 * arity:
|
|
return [-71, b'']
|
|
args = [int.from_bytes(body[i:i+8], 'little') for i in range(0, len(body), 8)]
|
|
result = self.op(names[opcode], *args)
|
|
if opcode in (3, 10, 12, 13):
|
|
return [0, b''.join(le(x) for x in result)]
|
|
status = result[0]
|
|
if status:
|
|
return [status, b'']
|
|
if opcode == 5:
|
|
return [0, le(result[1])]
|
|
if opcode in (8, 9):
|
|
return [0, le(result[1]) + result[2]]
|
|
if opcode == 11:
|
|
return [0, b''.join(le(x) for x in result[1:])]
|
|
return [0, b'']
|
|
|
|
class Scene:
|
|
def __init__(self, name):
|
|
self.name = name
|
|
self.models = [Model(), Model()]
|
|
self.selected = 0
|
|
self.commands = []
|
|
self.expected = []
|
|
self.labels = []
|
|
|
|
def add(self, op, *args, label='', clauses='C01-C18,P01-P07'):
|
|
fields = [op] + [hx(x) if isinstance(x, bytes) else str(x) for x in args]
|
|
command = ' '.join(fields)
|
|
model = self.models[self.selected]
|
|
if op == 'U':
|
|
self.selected = args[0]
|
|
result = [self.selected]
|
|
elif op == 'T':
|
|
result = [xform(args[1], args[0]), args[1]]
|
|
elif op == 'H':
|
|
result = [{b'pass': 0, b'upper': 1, b'lower': 2}.get(args[0], -22), args[0]]
|
|
elif op == 'N':
|
|
result = [{0: 'pass', 1: 'upper', 2: 'lower'}.get(args[0], 'invalid')]
|
|
elif op == 'W':
|
|
result = model.wire(args[0]) + [args[0]]
|
|
else:
|
|
result = model.op(op, *args)
|
|
if op == 'A':
|
|
result += [b'', args[2]]
|
|
rendered = op + ' ' + ' '.join(hx(x) if isinstance(x, bytes) else str(x) for x in result)
|
|
self.commands.append(command.encode('ascii') + b'\n')
|
|
self.expected.append(rendered.encode('ascii') + b'\n')
|
|
self.labels.append((label or op, clauses))
|
|
|
|
def audit(self, label='atomicity'):
|
|
for op in ('G', 'C', 'S', 'L'):
|
|
self.add(op, label=label + '/' + op, clauses='C03-C04,C12-C13')
|
|
for ident in (1, 7, 17, 4294967295):
|
|
self.add('I', ident, label=label + '/inspect', clauses='C12-C13')
|
|
self.add('P', 4096, label=label + '/peek', clauses='C11-C13')
|
|
|
|
def wire(self, opcode, *args, data=None, label='wire', clauses='C14-C16'):
|
|
self.add('W', packet(opcode, *args, data=data), label=label, clauses=clauses)
|
|
|
|
def freeze(self):
|
|
return {'name': self.name, 'input': b''.join(self.commands),
|
|
'expected': b''.join(self.expected), 'labels': self.labels}
|
|
|
|
def scenes():
|
|
output = []
|
|
s = Scene('public-short-write')
|
|
for op, args in [('Z', [3]), ('M', [1]), ('B', [17, 5]), ('A', [17, 5, b'aBcDe']),
|
|
('S', []), ('R', [3]), ('A', [17, 2, b'De']), ('F', [17]), ('S', []), ('R', [8])]:
|
|
s.add(op, *args, label='public/' + op, clauses='C03-C06,C08-C12')
|
|
output.append(s.freeze())
|
|
|
|
s = Scene('capture-and-interleave')
|
|
for first in (0, 1, 2):
|
|
for second in (0, 1, 2):
|
|
s.add('Z', 11, clauses='C03')
|
|
s.add('M', first, clauses='C04')
|
|
s.add('B', 1, 6, clauses='C06-C07')
|
|
s.add('A', 1, 3, b'aZ\x00', clauses='C07-C09')
|
|
s.add('M', second, clauses='C04,C07')
|
|
s.add('G', label='mode-control-direct', clauses='C04')
|
|
s.add('B', 7, 3, clauses='C06-C07')
|
|
s.add('A', 7, 4096, b'xY\xff', clauses='C07-C09')
|
|
s.add('A', 1, 4096, b'Bc\x80', label='begin-mode-capture-later-chunk', clauses='C07-C09')
|
|
s.audit('interleave-state')
|
|
s.add('F', 7, clauses='C10')
|
|
s.add('F', 1, clauses='C10')
|
|
s.add('M', (first + 1) % 3, clauses='C07')
|
|
s.add('P', 4096, label='old-fifo-after-mode-change', clauses='C07,C11')
|
|
s.add('R', 4, clauses='C11')
|
|
s.add('R', 4096, clauses='C11')
|
|
s.add('B', 1, 1, clauses='C06-C07')
|
|
s.add('A', 1, 1, b'z', label='reused-id-new-capture', clauses='C07')
|
|
s.add('X', 1, clauses='C10')
|
|
s.add('R', 1, label='cancel-retains-fifo', clauses='C10-C11')
|
|
s.add('S', clauses='C12')
|
|
output.append(s.freeze())
|
|
|
|
s = Scene('boundaries-and-error-atomicity')
|
|
s.audit('fresh-constructor')
|
|
for cap in (0, 1, 2, 4095, 4096, 4097, MASK):
|
|
s.add('Z', cap, label='capacity-domain', clauses='C03,C13')
|
|
s.audit('capacity-after')
|
|
s.add('Z', 7)
|
|
s.add('M', 2)
|
|
for ident in (17, 1, 4294967295, 7):
|
|
s.add('B', ident, 5, clauses='C06')
|
|
for ident, total in [(0, MASK), (MASK, MASK), (4294967296, 0), (7, 65536), (7, 5), (8, 65536), (8, 5)]:
|
|
s.add('B', ident, total, label='begin-priority', clauses='C06,C13')
|
|
s.audit('failed-begin')
|
|
for ident in (0, 4294967296, MASK, 2, 1, 4294967295):
|
|
for op in ('I', 'F', 'X'):
|
|
s.add(op, ident, label='id-domain/' + op, clauses='C10,C12-C13')
|
|
s.audit('id-after')
|
|
for mode in (-(1 << 63), -1, 0, 1, 2, 3, (1 << 63) - 1):
|
|
s.add('M', mode, label='mode-domain', clauses='C04,C13')
|
|
s.add('G', clauses='C04')
|
|
s.audit('mode-after')
|
|
s.add('Z', 4)
|
|
s.add('B', 1, 65535)
|
|
for ident, budget, data in [(0, MASK, b'x' * 65536), (9, MASK, b'x' * 65536),
|
|
(1, MASK, b'x' * 65536), (1, 4096, b'x' * 65536),
|
|
(1, 0, b''), (1, 4097, b''), (1, MASK, b''),
|
|
(1, 1, b'x' * 65535), (1, 1, b'y' * 65535),
|
|
(1, 4096, b'BcD'), (1, 4096, b'e'), (1, 0, b'fg')]:
|
|
s.add('A', ident, budget, data, label='append-domain-priority', clauses='C08-C09,C13')
|
|
s.audit('append-after')
|
|
for count in (0, 1, 4095, 4096, 4097, MASK):
|
|
s.add('P', count, clauses='C11-C13')
|
|
s.add('R', count, clauses='C11-C13')
|
|
s.audit('read-after')
|
|
for total in (0, 1, 65535, 65536, MASK):
|
|
s.add('Z', 1)
|
|
s.add('B', 4294967295, total, label='declared-total-domain', clauses='C06')
|
|
s.add('I', 4294967295, clauses='C12')
|
|
s.add('A', 4294967295, 0, b'', clauses='C08-C09')
|
|
s.add('F', 4294967295, clauses='C10')
|
|
s.add('F', 4294967295, clauses='C10')
|
|
s.add('X', 4294967295, clauses='C10')
|
|
s.add('X', 4294967295, clauses='C10')
|
|
s.audit('repeat-after')
|
|
# A failed request larger than remaining must not partially fit into FIFO.
|
|
s.add('Z', 2); s.add('B', 1, 1); s.add('A', 1, 1, b'aB', label='over-total-before-prefix', clauses='C08')
|
|
s.audit('over-total-atomic'); s.add('A', 1, 1, b'a'); s.add('A', 1, 1, b'b', label='completed-active-over-total', clauses='C08')
|
|
s.add('F', 1); s.add('B', 1, 0); s.add('F', 1); s.add('S')
|
|
# reset is observed with all state components nonempty.
|
|
s.add('M', 2); s.add('B', 7, 9); s.add('Z', 0); s.audit('invalid-reset-preserves')
|
|
s.add('Z', 4096); s.audit('valid-reset-clears')
|
|
output.append(s.freeze())
|
|
|
|
s = Scene('all-byte-values-and-pure-helpers')
|
|
s.add('Z', 7); s.add('M', 1); s.add('B', 17, 9)
|
|
s.add('A', 17, 4, b'aB\x00\xff')
|
|
s.add('M', 2); s.add('B', 7, 0)
|
|
s.audit('helpers-before')
|
|
for mode in (-(1 << 63), -1, 0, 1, 2, 3, (1 << 63) - 1):
|
|
s.add('N', mode, clauses='C04')
|
|
s.audit('helper-name-preserves-device')
|
|
for data in (b'', bytes(range(256)), b'@AZ[az{\x00\r\n\xff', b'z' * 65535):
|
|
s.add('T', mode, data, label='transform-byte-domain', clauses='C01-C02,C05')
|
|
s.audit('helper-transform-preserves-device')
|
|
for text in (b'pass', b'upper', b'lower', b'', b'PASS', b'Upper', b'lower\n', b'upper\x00', b' pass', b'pass ', b'lower\r\n', b'upp', b'uppers', bytes(range(256)), b'lower' + b'\x00' * 65536):
|
|
s.add('H', text, label='strict-mode-text', clauses='C04')
|
|
s.audit('helper-parse-preserves-device')
|
|
s.add('A', 17, 5, b'cDeFz', label='helper-followup-accepts', clauses='C02,C07-C09,C13')
|
|
s.add('R', 4096); s.add('F', 7); s.add('X', 17); s.audit('helper-followup-state')
|
|
for mode in (0, 1, 2):
|
|
s.add('Z', 4096); s.add('M', mode); s.add('B', 1, 65535)
|
|
remaining = 65535
|
|
index = 0
|
|
while remaining:
|
|
length = min(4096, remaining)
|
|
data = bytes((index + n) % 256 for n in range(length))
|
|
s.add('A', 1, 4096, data, label='large-full-transaction', clauses='C05,C07-C09')
|
|
s.add('P', 4096); s.add('R', 4096)
|
|
remaining -= length
|
|
index += length
|
|
s.add('F', 1); s.add('S'); s.add('R', 0); s.add('R', 1)
|
|
output.append(s.freeze())
|
|
|
|
s = Scene('packet-success-and-semantic-errors')
|
|
for op, args, data in [(13, [], None), (3, [], None), (10, [], None), (12, [], None),
|
|
(1, [9], None), (2, [1], None), (4, [17, 6], None),
|
|
(5, [17, 3], b'aB\x00'), (2, [2], None), (5, [17, 9], b'De\xff'),
|
|
(11, [17], None), (12, [], None), (9, [9], None), (6, [17], None),
|
|
(8, [4], None), (8, [9], None), (10, [], None), (4, [1, 2], None),
|
|
(7, [1], None), (13, [], None)]:
|
|
s.wire(op, *args, data=data, label='packet-direct-equivalent')
|
|
s.audit('packet-success-state')
|
|
for op, args, data in [(1, [0], None), (1, [4097], None), (2, [MASK], None),
|
|
(4, [0, MASK], None), (4, [1, 65536], None), (5, [0, MASK], b'x'),
|
|
(5, [2, MASK], b'x'), (6, [0], None), (7, [MASK], None),
|
|
(8, [4097], None), (9, [MASK], None), (11, [0], None), (11, [2], None)]:
|
|
s.wire(op, *args, data=data, label='packet-semantic-error')
|
|
s.audit('packet-error-atomicity')
|
|
s.wire(4, 1, 1); s.wire(5, 1, MASK, data=b'x'); s.wire(5, 1, 1, data=b'xx')
|
|
s.wire(6, 1); s.wire(5, 1, 1, data=b'x'); s.wire(6, 1); s.wire(6, 1); s.wire(7, 1)
|
|
s.wire(8, 0); s.wire(8, 1); s.wire(8, 1); s.wire(9, 0); s.wire(9, 1)
|
|
# Endpoint scalars decoded without signed narrowing.
|
|
for ident in (1, 4294967295, 4294967296, MASK):
|
|
s.wire(4, ident, 0); s.wire(11, ident); s.wire(6, ident)
|
|
for total in (65535, 65536, MASK):
|
|
s.wire(4, 17, total); s.wire(7, 17)
|
|
s.wire(1, 4096); s.wire(2, 0); s.wire(4, 1, 65535)
|
|
s.wire(5, 1, 4096, data=bytes(range(256)) * 255 + b'z' * 255)
|
|
s.wire(8, 4096); s.wire(7, 1); s.audit('packet-large-after')
|
|
output.append(s.freeze())
|
|
|
|
s = Scene('packet-framing-and-priority')
|
|
s.add('Z', 9); s.add('M', 1); s.add('B', 1, 20); s.add('A', 1, 9, b'aBcD')
|
|
for op in range(1, 14):
|
|
arity = {1: 1, 2: 1, 3: 0, 4: 2, 5: 3, 6: 1, 7: 1, 8: 1, 9: 1, 10: 0, 11: 1, 12: 0, 13: 0}[op]
|
|
good = packet(op, *([0] * arity))
|
|
mutations = [good[:-1], good + b'\x00', b'XS' + good[2:], good[:2] + b'\x00' + good[3:]]
|
|
if op == 5:
|
|
mutations += [packet(5, 0, MASK, 65536) + b'x' * 65536,
|
|
packet(5, 0, MASK, MASK), packet(5, 0, MASK, 1), packet(5, 0, MASK, 0) + b'x']
|
|
for raw in mutations:
|
|
s.add('W', raw, label='packet-malformed-before-semantics', clauses='C13-C15')
|
|
s.audit('malformed-atomic')
|
|
for raw in (b'', b'M', b'MS', b'MS\x01', b'MS\x01\x00', b'MS\x01\xff',
|
|
b'MS\x01\x00' + b'x' * 131068, b'MS\x01\x00' + b'x' * 131069,
|
|
b'MS\x01\x01' + b'x' * 131068, b'MS\x01\xff' + b'x' * 3):
|
|
s.add('W', raw, label='packet-size-and-unknown-domain', clauses='C14')
|
|
s.audit('packet-boundary-atomic')
|
|
s.add('R', 4096)
|
|
output.append(s.freeze())
|
|
|
|
rng = random.Random(0x6370747374726561)
|
|
s = Scene('deterministic-mixed-history')
|
|
for turn in range(1800):
|
|
if turn % 67 == 0:
|
|
s.add('U', rng.randrange(2), label='independent-devices', clauses='C01,P01-P02')
|
|
model = s.models[s.selected]
|
|
ident = rng.choice([0, 1, 7, 17, 2, 4294967295, 4294967296, MASK])
|
|
if turn % 101 == 0:
|
|
s.add('Z', rng.choice([1, 2, 3, 17, 4096])); continue
|
|
op = rng.choices(['M', 'B', 'A', 'F', 'X', 'R', 'P', 'S', 'I', 'L', 'G', 'C'],
|
|
[8, 12, 25, 5, 5, 18, 8, 3, 3, 3, 3, 3])[0]
|
|
args = []
|
|
if op == 'M': args = [rng.choice([-1, 0, 1, 2, 3])]
|
|
if op == 'B': args = [ident, rng.choice([0, 1, 2, 5, 19, 65535, 65536])]
|
|
if op == 'A':
|
|
if model.transactions and rng.randrange(5): ident = rng.choice(list(model.transactions))
|
|
length = rng.randrange(0, 21)
|
|
args = [ident, rng.choice([0, 1, 2, 5, 4096, 4097]), bytes(rng.randrange(256) for _ in range(length))]
|
|
if op in ('F', 'X', 'I'): args = [ident]
|
|
if op in ('R', 'P'): args = [rng.choice([0, 1, 2, 3, 17, 4096, 4097, MASK])]
|
|
code = {'Z': 1, 'M': 2, 'G': 3, 'B': 4, 'A': 5, 'F': 6, 'X': 7, 'R': 8, 'P': 9, 'S': 10, 'I': 11, 'L': 12, 'C': 13}[op]
|
|
if rng.randrange(3) == 0 and not (op == 'M' and args[0] < 0):
|
|
if op == 'A': s.wire(code, *args[:2], data=args[2], label='mixed-packet')
|
|
else: s.wire(code, *args, label='mixed-packet')
|
|
else:
|
|
s.add(op, *args, label='mixed-direct')
|
|
if turn % 31 == 0: s.audit('mixed-state')
|
|
# Force repeated circular wrap, alternating two simultaneously captured modes.
|
|
for selected in (0, 1):
|
|
s.add('U', selected); s.add('Z', 7); s.add('M', selected + 1)
|
|
for ident in (1, 7): s.add('B', ident, 500)
|
|
for turn in range(150):
|
|
ident = (1, 7)[turn % 2]
|
|
data = bytes(rng.randrange(256) for _ in range(5))
|
|
s.add('A', ident, 5, data, label='ring-wrap-acceptance')
|
|
s.add('M', turn % 3, label='ring-wrap-control')
|
|
s.add('R', 3, label='ring-wrap-read')
|
|
s.add('P', 7, label='ring-wrap-peek')
|
|
s.audit('ring-wrap-final')
|
|
output.append(s.freeze())
|
|
# Separate exact empty-stdout policy from a nonempty observation stream.
|
|
output.append(Scene('empty-input-output-policy').freeze())
|
|
return output
|
|
|
|
def brief(data, limit=100):
|
|
return repr(data[:limit]) + ('...' if len(data) > limit else '')
|
|
|
|
def main():
|
|
suite = scenes()
|
|
context = json.loads((Path(__file__).parent / 'observer.json').read_text())
|
|
candidate_data = {name: (Path(sys.argv[1]) / name).read_bytes() for name in CANDIDATES}
|
|
scratch = Path(tempfile.mkdtemp(prefix='captured-stream-', dir=Path.cwd()))
|
|
for name, data in context.items():
|
|
(scratch / name).write_text(data)
|
|
for name, data in candidate_data.items():
|
|
(scratch / name).write_bytes(data)
|
|
# shortcut: scratch belongs to the wrapper's unique cwd; wrapper exit removes the tree.
|
|
# After the first compiler starts, the trusted checker never accesses this tree again.
|
|
build = subprocess.run(['g++', '-std=c++17', '-O2', '-Wall', '-Wextra', '-pedantic',
|
|
'stream_observe.cpp', 'stream.cpp', 'byte_codec.cpp', 'control_packet.cpp',
|
|
'-o', 'observe'], cwd=scratch, capture_output=True, timeout=30)
|
|
if build.returncode:
|
|
summary = 'build=%d runs={} first=build stderr=%s' % (build.returncode, brief(build.stderr, 600))
|
|
sys.stderr.write(summary + '\n')
|
|
print(summary)
|
|
return 1
|
|
exits = collections.Counter()
|
|
failures = []
|
|
total_observations = 0
|
|
for scene in suite:
|
|
try:
|
|
run = subprocess.run([str(scratch / 'observe')], cwd=scratch, input=scene['input'],
|
|
capture_output=True, timeout=12)
|
|
except subprocess.TimeoutExpired:
|
|
exits['timeout'] += 1
|
|
failures.append(scene['name'] + ': timeout')
|
|
continue
|
|
exits[str(run.returncode)] += 1
|
|
expected = scene['expected']
|
|
equal = run.stdout == expected
|
|
total_observations += len(scene['labels'])
|
|
if run.returncode or not equal or run.stderr:
|
|
detail = '%s exit=%d stdout_equal=%s stderr_len=%d' % (scene['name'], run.returncode, equal, len(run.stderr))
|
|
if not equal:
|
|
actual_lines = run.stdout.split(b'\n')
|
|
expected_lines = expected.split(b'\n')
|
|
index = next(i for i in range(max(len(actual_lines), len(expected_lines)))
|
|
if (actual_lines[i] if i < len(actual_lines) else None) !=
|
|
(expected_lines[i] if i < len(expected_lines) else None))
|
|
label, clauses = scene['labels'][index] if index < len(scene['labels']) else ('extra-output', 'C18')
|
|
got = actual_lines[index] if index < len(actual_lines) else b'<missing>'
|
|
want = expected_lines[index] if index < len(expected_lines) else b'<end>'
|
|
detail += ' line=%d check=%s clauses=%s got=%s expected=%s' % (index + 1, label, clauses, brief(got), brief(want))
|
|
elif run.stderr:
|
|
detail += ' check=empty-stderr clauses=C01,C18 stderr=' + brief(run.stderr)
|
|
else:
|
|
detail += ' check=exit-zero clauses=C18'
|
|
failures.append(detail)
|
|
summary = 'build=0 runs=%s scenes=%d observations=%d first=%s' % (dict(exits), len(suite), total_observations, failures[0] if failures else 'none')
|
|
# The wrapper keeps only the final 4000 stderr bytes. Keep concise per-scene facts plus first rejection last.
|
|
sys.stderr.write('scene_failures=%d\n' % len(failures))
|
|
for failure in failures[:3]: sys.stderr.write(failure[:650] + '\n')
|
|
sys.stderr.write(summary[:1800] + '\n')
|
|
print(summary[:1800] if failures else 'PASS')
|
|
return 1 if failures else 0
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|