amos 2f32624d3b Batch 18 (20261026-C-hwe-codegen-18): add 100 calibrated HWE code-gen tasks; pause production for rollout
100 tasks from the HWE arm-linux code-gen pool (outside the lk/man/pth
range given to batch 22), each with at least one independent
contract-edge defect and plain public docs. Claude reviewed 100/100
USABLE against current digests; no ID or source overlap with main or
batch 22. Difficulty is unmeasured.

progress.md records the wrap-up: all Codex sessions and Qwen schedulers
stopped for the user's rollout, Qwen results so far (57 HARD of 373),
and what is left on each batch branch.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 17:52:13 +08:00

487 lines
22 KiB
Python

"""Independent byte/state model; candidate execution is confined to subprocesses."""
import collections
import json
from pathlib import Path
import random
import subprocess
import sys
import tempfile
MASK = (1 << 64) - 1
CANDIDATES = ('stream.hpp', 'stream.cpp', 'byte_codec.cpp', 'control_packet.cpp')
def hx(value):
return value.hex() or '-'
def unhex(value):
return b'' if value == '-' else bytes.fromhex(value)
def le(value):
return int(value).to_bytes(8, 'little')
def xform(value, mode):
table = bytes((x - 32 if mode == 1 and 97 <= x <= 122 else
x + 32 if mode == 2 and 65 <= x <= 90 else x) for x in range(256))
return value.translate(table)
def packet(op, *values, data=None):
payload = b''.join(le(v) for v in values)
if data is not None:
payload += le(len(data)) + data
return b'MS\x01' + bytes([op]) + payload
class Model:
def __init__(self):
self.cap = 16
self.mode = 0
self.fifo = b''
self.transactions = {}
self.counters = [0, 0, 0, 0]
@staticmethod
def valid_id(value):
return 1 <= value <= 4294967295
def state(self):
return self.counters + [len(self.fifo), len(self.transactions)]
def op(self, op, *args):
if op == 'Z':
cap, = args
if not 1 <= cap <= 4096:
return [-22]
self.__init__()
self.cap = cap
return [0]
if op == 'M':
mode, = args
if mode not in (0, 1, 2):
return [-22]
self.mode = mode
return [0]
if op == 'G':
return [self.mode]
if op == 'C':
return [self.cap]
if op == 'S':
return self.state()
if op == 'L':
ids = sorted(self.transactions)
return [len(ids)] + ids
if op in ('B', 'A', 'F', 'X', 'I'):
ident = args[0]
status = 0 if self.valid_id(ident) else -22
if status == 0 and op != 'B' and ident not in self.transactions:
status = -2
if status:
if op == 'A':
return [status, 0]
if op == 'I':
return [status, 0, 0, 0, 0, 0]
return [status]
if op == 'B':
total = args[1]
if total > 65535:
return [-7]
if ident in self.transactions:
return [-17]
if len(self.transactions) == 4:
return [-28]
self.transactions[ident] = [total, 0, self.mode]
return [0]
transaction = self.transactions[ident]
total, done, captured = transaction
if op == 'I':
return [0, ident, total, done, total - done, captured]
if op == 'F':
if done != total:
return [-115]
del self.transactions[ident]
self.counters[2] = (self.counters[2] + 1) & MASK
return [0]
if op == 'X':
del self.transactions[ident]
self.counters[3] = (self.counters[3] + 1) & MASK
return [0]
budget, data = args[1:]
if budget > 4096:
return [-22, 0]
if len(data) > 65535:
return [-7, 0]
if len(data) > total - done:
return [-90, 0]
accepted = min(len(data), budget, self.cap - len(self.fifo))
self.fifo += xform(data[:accepted], captured)
transaction[1] += accepted
self.counters[0] = (self.counters[0] + accepted) & MASK
return [0, accepted]
if op in ('R', 'P'):
count, = args
if count > 4096:
return [-22, 0, b'']
if count == 0:
return [0, 0, b'']
if not self.fifo:
return [-11, 0, b'']
data = self.fifo[:count]
if op == 'R':
self.fifo = self.fifo[len(data):]
self.counters[1] = (self.counters[1] + len(data)) & MASK
return [0, len(data), data]
raise ValueError(op)
def wire(self, raw):
if not 4 <= len(raw) <= 131072 or raw[:3] != b'MS\x01':
return [-71, b'']
opcode = raw[3]
names = {1: 'Z', 2: 'M', 3: 'G', 4: 'B', 5: 'A', 6: 'F', 7: 'X',
8: 'R', 9: 'P', 10: 'S', 11: 'I', 12: 'L', 13: 'C'}
if opcode not in names:
return [-25, b'']
body = raw[4:]
if opcode == 5:
if len(body) < 24:
return [-71, b'']
ident, budget, length = [int.from_bytes(body[i:i+8], 'little') for i in (0, 8, 16)]
if length > 65535 or length != len(body) - 24:
return [-71, b'']
result = self.op('A', ident, budget, body[24:])
else:
arity = {1: 1, 2: 1, 3: 0, 4: 2, 6: 1, 7: 1, 8: 1, 9: 1, 10: 0, 11: 1, 12: 0, 13: 0}[opcode]
if len(body) != 8 * arity:
return [-71, b'']
args = [int.from_bytes(body[i:i+8], 'little') for i in range(0, len(body), 8)]
result = self.op(names[opcode], *args)
if opcode in (3, 10, 12, 13):
return [0, b''.join(le(x) for x in result)]
status = result[0]
if status:
return [status, b'']
if opcode == 5:
return [0, le(result[1])]
if opcode in (8, 9):
return [0, le(result[1]) + result[2]]
if opcode == 11:
return [0, b''.join(le(x) for x in result[1:])]
return [0, b'']
class Scene:
def __init__(self, name):
self.name = name
self.models = [Model(), Model()]
self.selected = 0
self.commands = []
self.expected = []
self.labels = []
def add(self, op, *args, label='', clauses='C01-C18,P01-P07'):
fields = [op] + [hx(x) if isinstance(x, bytes) else str(x) for x in args]
command = ' '.join(fields)
model = self.models[self.selected]
if op == 'U':
self.selected = args[0]
result = [self.selected]
elif op == 'T':
result = [xform(args[1], args[0]), args[1]]
elif op == 'H':
result = [{b'pass': 0, b'upper': 1, b'lower': 2}.get(args[0], -22), args[0]]
elif op == 'N':
result = [{0: 'pass', 1: 'upper', 2: 'lower'}.get(args[0], 'invalid')]
elif op == 'W':
result = model.wire(args[0]) + [args[0]]
else:
result = model.op(op, *args)
if op == 'A':
result += [b'', args[2]]
rendered = op + ' ' + ' '.join(hx(x) if isinstance(x, bytes) else str(x) for x in result)
self.commands.append(command.encode('ascii') + b'\n')
self.expected.append(rendered.encode('ascii') + b'\n')
self.labels.append((label or op, clauses))
def audit(self, label='atomicity'):
for op in ('G', 'C', 'S', 'L'):
self.add(op, label=label + '/' + op, clauses='C03-C04,C12-C13')
for ident in (1, 7, 17, 4294967295):
self.add('I', ident, label=label + '/inspect', clauses='C12-C13')
self.add('P', 4096, label=label + '/peek', clauses='C11-C13')
def wire(self, opcode, *args, data=None, label='wire', clauses='C14-C16'):
self.add('W', packet(opcode, *args, data=data), label=label, clauses=clauses)
def freeze(self):
return {'name': self.name, 'input': b''.join(self.commands),
'expected': b''.join(self.expected), 'labels': self.labels}
def scenes():
output = []
s = Scene('public-short-write')
for op, args in [('Z', [3]), ('M', [1]), ('B', [17, 5]), ('A', [17, 5, b'aBcDe']),
('S', []), ('R', [3]), ('A', [17, 2, b'De']), ('F', [17]), ('S', []), ('R', [8])]:
s.add(op, *args, label='public/' + op, clauses='C03-C06,C08-C12')
output.append(s.freeze())
s = Scene('capture-and-interleave')
for first in (0, 1, 2):
for second in (0, 1, 2):
s.add('Z', 11, clauses='C03')
s.add('M', first, clauses='C04')
s.add('B', 1, 6, clauses='C06-C07')
s.add('A', 1, 3, b'aZ\x00', clauses='C07-C09')
s.add('M', second, clauses='C04,C07')
s.add('G', label='mode-control-direct', clauses='C04')
s.add('B', 7, 3, clauses='C06-C07')
s.add('A', 7, 4096, b'xY\xff', clauses='C07-C09')
s.add('A', 1, 4096, b'Bc\x80', label='begin-mode-capture-later-chunk', clauses='C07-C09')
s.audit('interleave-state')
s.add('F', 7, clauses='C10')
s.add('F', 1, clauses='C10')
s.add('M', (first + 1) % 3, clauses='C07')
s.add('P', 4096, label='old-fifo-after-mode-change', clauses='C07,C11')
s.add('R', 4, clauses='C11')
s.add('R', 4096, clauses='C11')
s.add('B', 1, 1, clauses='C06-C07')
s.add('A', 1, 1, b'z', label='reused-id-new-capture', clauses='C07')
s.add('X', 1, clauses='C10')
s.add('R', 1, label='cancel-retains-fifo', clauses='C10-C11')
s.add('S', clauses='C12')
output.append(s.freeze())
s = Scene('boundaries-and-error-atomicity')
s.audit('fresh-constructor')
for cap in (0, 1, 2, 4095, 4096, 4097, MASK):
s.add('Z', cap, label='capacity-domain', clauses='C03,C13')
s.audit('capacity-after')
s.add('Z', 7)
s.add('M', 2)
for ident in (17, 1, 4294967295, 7):
s.add('B', ident, 5, clauses='C06')
for ident, total in [(0, MASK), (MASK, MASK), (4294967296, 0), (7, 65536), (7, 5), (8, 65536), (8, 5)]:
s.add('B', ident, total, label='begin-priority', clauses='C06,C13')
s.audit('failed-begin')
for ident in (0, 4294967296, MASK, 2, 1, 4294967295):
for op in ('I', 'F', 'X'):
s.add(op, ident, label='id-domain/' + op, clauses='C10,C12-C13')
s.audit('id-after')
for mode in (-(1 << 63), -1, 0, 1, 2, 3, (1 << 63) - 1):
s.add('M', mode, label='mode-domain', clauses='C04,C13')
s.add('G', clauses='C04')
s.audit('mode-after')
s.add('Z', 4)
s.add('B', 1, 65535)
for ident, budget, data in [(0, MASK, b'x' * 65536), (9, MASK, b'x' * 65536),
(1, MASK, b'x' * 65536), (1, 4096, b'x' * 65536),
(1, 0, b''), (1, 4097, b''), (1, MASK, b''),
(1, 1, b'x' * 65535), (1, 1, b'y' * 65535),
(1, 4096, b'BcD'), (1, 4096, b'e'), (1, 0, b'fg')]:
s.add('A', ident, budget, data, label='append-domain-priority', clauses='C08-C09,C13')
s.audit('append-after')
for count in (0, 1, 4095, 4096, 4097, MASK):
s.add('P', count, clauses='C11-C13')
s.add('R', count, clauses='C11-C13')
s.audit('read-after')
for total in (0, 1, 65535, 65536, MASK):
s.add('Z', 1)
s.add('B', 4294967295, total, label='declared-total-domain', clauses='C06')
s.add('I', 4294967295, clauses='C12')
s.add('A', 4294967295, 0, b'', clauses='C08-C09')
s.add('F', 4294967295, clauses='C10')
s.add('F', 4294967295, clauses='C10')
s.add('X', 4294967295, clauses='C10')
s.add('X', 4294967295, clauses='C10')
s.audit('repeat-after')
# A failed request larger than remaining must not partially fit into FIFO.
s.add('Z', 2); s.add('B', 1, 1); s.add('A', 1, 1, b'aB', label='over-total-before-prefix', clauses='C08')
s.audit('over-total-atomic'); s.add('A', 1, 1, b'a'); s.add('A', 1, 1, b'b', label='completed-active-over-total', clauses='C08')
s.add('F', 1); s.add('B', 1, 0); s.add('F', 1); s.add('S')
# reset is observed with all state components nonempty.
s.add('M', 2); s.add('B', 7, 9); s.add('Z', 0); s.audit('invalid-reset-preserves')
s.add('Z', 4096); s.audit('valid-reset-clears')
output.append(s.freeze())
s = Scene('all-byte-values-and-pure-helpers')
s.add('Z', 7); s.add('M', 1); s.add('B', 17, 9)
s.add('A', 17, 4, b'aB\x00\xff')
s.add('M', 2); s.add('B', 7, 0)
s.audit('helpers-before')
for mode in (-(1 << 63), -1, 0, 1, 2, 3, (1 << 63) - 1):
s.add('N', mode, clauses='C04')
s.audit('helper-name-preserves-device')
for data in (b'', bytes(range(256)), b'@AZ[az{\x00\r\n\xff', b'z' * 65535):
s.add('T', mode, data, label='transform-byte-domain', clauses='C01-C02,C05')
s.audit('helper-transform-preserves-device')
for text in (b'pass', b'upper', b'lower', b'', b'PASS', b'Upper', b'lower\n', b'upper\x00', b' pass', b'pass ', b'lower\r\n', b'upp', b'uppers', bytes(range(256)), b'lower' + b'\x00' * 65536):
s.add('H', text, label='strict-mode-text', clauses='C04')
s.audit('helper-parse-preserves-device')
s.add('A', 17, 5, b'cDeFz', label='helper-followup-accepts', clauses='C02,C07-C09,C13')
s.add('R', 4096); s.add('F', 7); s.add('X', 17); s.audit('helper-followup-state')
for mode in (0, 1, 2):
s.add('Z', 4096); s.add('M', mode); s.add('B', 1, 65535)
remaining = 65535
index = 0
while remaining:
length = min(4096, remaining)
data = bytes((index + n) % 256 for n in range(length))
s.add('A', 1, 4096, data, label='large-full-transaction', clauses='C05,C07-C09')
s.add('P', 4096); s.add('R', 4096)
remaining -= length
index += length
s.add('F', 1); s.add('S'); s.add('R', 0); s.add('R', 1)
output.append(s.freeze())
s = Scene('packet-success-and-semantic-errors')
for op, args, data in [(13, [], None), (3, [], None), (10, [], None), (12, [], None),
(1, [9], None), (2, [1], None), (4, [17, 6], None),
(5, [17, 3], b'aB\x00'), (2, [2], None), (5, [17, 9], b'De\xff'),
(11, [17], None), (12, [], None), (9, [9], None), (6, [17], None),
(8, [4], None), (8, [9], None), (10, [], None), (4, [1, 2], None),
(7, [1], None), (13, [], None)]:
s.wire(op, *args, data=data, label='packet-direct-equivalent')
s.audit('packet-success-state')
for op, args, data in [(1, [0], None), (1, [4097], None), (2, [MASK], None),
(4, [0, MASK], None), (4, [1, 65536], None), (5, [0, MASK], b'x'),
(5, [2, MASK], b'x'), (6, [0], None), (7, [MASK], None),
(8, [4097], None), (9, [MASK], None), (11, [0], None), (11, [2], None)]:
s.wire(op, *args, data=data, label='packet-semantic-error')
s.audit('packet-error-atomicity')
s.wire(4, 1, 1); s.wire(5, 1, MASK, data=b'x'); s.wire(5, 1, 1, data=b'xx')
s.wire(6, 1); s.wire(5, 1, 1, data=b'x'); s.wire(6, 1); s.wire(6, 1); s.wire(7, 1)
s.wire(8, 0); s.wire(8, 1); s.wire(8, 1); s.wire(9, 0); s.wire(9, 1)
# Endpoint scalars decoded without signed narrowing.
for ident in (1, 4294967295, 4294967296, MASK):
s.wire(4, ident, 0); s.wire(11, ident); s.wire(6, ident)
for total in (65535, 65536, MASK):
s.wire(4, 17, total); s.wire(7, 17)
s.wire(1, 4096); s.wire(2, 0); s.wire(4, 1, 65535)
s.wire(5, 1, 4096, data=bytes(range(256)) * 255 + b'z' * 255)
s.wire(8, 4096); s.wire(7, 1); s.audit('packet-large-after')
output.append(s.freeze())
s = Scene('packet-framing-and-priority')
s.add('Z', 9); s.add('M', 1); s.add('B', 1, 20); s.add('A', 1, 9, b'aBcD')
for op in range(1, 14):
arity = {1: 1, 2: 1, 3: 0, 4: 2, 5: 3, 6: 1, 7: 1, 8: 1, 9: 1, 10: 0, 11: 1, 12: 0, 13: 0}[op]
good = packet(op, *([0] * arity))
mutations = [good[:-1], good + b'\x00', b'XS' + good[2:], good[:2] + b'\x00' + good[3:]]
if op == 5:
mutations += [packet(5, 0, MASK, 65536) + b'x' * 65536,
packet(5, 0, MASK, MASK), packet(5, 0, MASK, 1), packet(5, 0, MASK, 0) + b'x']
for raw in mutations:
s.add('W', raw, label='packet-malformed-before-semantics', clauses='C13-C15')
s.audit('malformed-atomic')
for raw in (b'', b'M', b'MS', b'MS\x01', b'MS\x01\x00', b'MS\x01\xff',
b'MS\x01\x00' + b'x' * 131068, b'MS\x01\x00' + b'x' * 131069,
b'MS\x01\x01' + b'x' * 131068, b'MS\x01\xff' + b'x' * 3):
s.add('W', raw, label='packet-size-and-unknown-domain', clauses='C14')
s.audit('packet-boundary-atomic')
s.add('R', 4096)
output.append(s.freeze())
rng = random.Random(0x6370747374726561)
s = Scene('deterministic-mixed-history')
for turn in range(1800):
if turn % 67 == 0:
s.add('U', rng.randrange(2), label='independent-devices', clauses='C01,P01-P02')
model = s.models[s.selected]
ident = rng.choice([0, 1, 7, 17, 2, 4294967295, 4294967296, MASK])
if turn % 101 == 0:
s.add('Z', rng.choice([1, 2, 3, 17, 4096])); continue
op = rng.choices(['M', 'B', 'A', 'F', 'X', 'R', 'P', 'S', 'I', 'L', 'G', 'C'],
[8, 12, 25, 5, 5, 18, 8, 3, 3, 3, 3, 3])[0]
args = []
if op == 'M': args = [rng.choice([-1, 0, 1, 2, 3])]
if op == 'B': args = [ident, rng.choice([0, 1, 2, 5, 19, 65535, 65536])]
if op == 'A':
if model.transactions and rng.randrange(5): ident = rng.choice(list(model.transactions))
length = rng.randrange(0, 21)
args = [ident, rng.choice([0, 1, 2, 5, 4096, 4097]), bytes(rng.randrange(256) for _ in range(length))]
if op in ('F', 'X', 'I'): args = [ident]
if op in ('R', 'P'): args = [rng.choice([0, 1, 2, 3, 17, 4096, 4097, MASK])]
code = {'Z': 1, 'M': 2, 'G': 3, 'B': 4, 'A': 5, 'F': 6, 'X': 7, 'R': 8, 'P': 9, 'S': 10, 'I': 11, 'L': 12, 'C': 13}[op]
if rng.randrange(3) == 0 and not (op == 'M' and args[0] < 0):
if op == 'A': s.wire(code, *args[:2], data=args[2], label='mixed-packet')
else: s.wire(code, *args, label='mixed-packet')
else:
s.add(op, *args, label='mixed-direct')
if turn % 31 == 0: s.audit('mixed-state')
# Force repeated circular wrap, alternating two simultaneously captured modes.
for selected in (0, 1):
s.add('U', selected); s.add('Z', 7); s.add('M', selected + 1)
for ident in (1, 7): s.add('B', ident, 500)
for turn in range(150):
ident = (1, 7)[turn % 2]
data = bytes(rng.randrange(256) for _ in range(5))
s.add('A', ident, 5, data, label='ring-wrap-acceptance')
s.add('M', turn % 3, label='ring-wrap-control')
s.add('R', 3, label='ring-wrap-read')
s.add('P', 7, label='ring-wrap-peek')
s.audit('ring-wrap-final')
output.append(s.freeze())
# Separate exact empty-stdout policy from a nonempty observation stream.
output.append(Scene('empty-input-output-policy').freeze())
return output
def brief(data, limit=100):
return repr(data[:limit]) + ('...' if len(data) > limit else '')
def main():
suite = scenes()
context = json.loads((Path(__file__).parent / 'observer.json').read_text())
candidate_data = {name: (Path(sys.argv[1]) / name).read_bytes() for name in CANDIDATES}
scratch = Path(tempfile.mkdtemp(prefix='captured-stream-', dir=Path.cwd()))
for name, data in context.items():
(scratch / name).write_text(data)
for name, data in candidate_data.items():
(scratch / name).write_bytes(data)
# shortcut: scratch belongs to the wrapper's unique cwd; wrapper exit removes the tree.
# After the first compiler starts, the trusted checker never accesses this tree again.
build = subprocess.run(['g++', '-std=c++17', '-O2', '-Wall', '-Wextra', '-pedantic',
'stream_observe.cpp', 'stream.cpp', 'byte_codec.cpp', 'control_packet.cpp',
'-o', 'observe'], cwd=scratch, capture_output=True, timeout=30)
if build.returncode:
summary = 'build=%d runs={} first=build stderr=%s' % (build.returncode, brief(build.stderr, 600))
sys.stderr.write(summary + '\n')
print(summary)
return 1
exits = collections.Counter()
failures = []
total_observations = 0
for scene in suite:
try:
run = subprocess.run([str(scratch / 'observe')], cwd=scratch, input=scene['input'],
capture_output=True, timeout=12)
except subprocess.TimeoutExpired:
exits['timeout'] += 1
failures.append(scene['name'] + ': timeout')
continue
exits[str(run.returncode)] += 1
expected = scene['expected']
equal = run.stdout == expected
total_observations += len(scene['labels'])
if run.returncode or not equal or run.stderr:
detail = '%s exit=%d stdout_equal=%s stderr_len=%d' % (scene['name'], run.returncode, equal, len(run.stderr))
if not equal:
actual_lines = run.stdout.split(b'\n')
expected_lines = expected.split(b'\n')
index = next(i for i in range(max(len(actual_lines), len(expected_lines)))
if (actual_lines[i] if i < len(actual_lines) else None) !=
(expected_lines[i] if i < len(expected_lines) else None))
label, clauses = scene['labels'][index] if index < len(scene['labels']) else ('extra-output', 'C18')
got = actual_lines[index] if index < len(actual_lines) else b'<missing>'
want = expected_lines[index] if index < len(expected_lines) else b'<end>'
detail += ' line=%d check=%s clauses=%s got=%s expected=%s' % (index + 1, label, clauses, brief(got), brief(want))
elif run.stderr:
detail += ' check=empty-stderr clauses=C01,C18 stderr=' + brief(run.stderr)
else:
detail += ' check=exit-zero clauses=C18'
failures.append(detail)
summary = 'build=0 runs=%s scenes=%d observations=%d first=%s' % (dict(exits), len(suite), total_observations, failures[0] if failures else 'none')
# The wrapper keeps only the final 4000 stderr bytes. Keep concise per-scene facts plus first rejection last.
sys.stderr.write('scene_failures=%d\n' % len(failures))
for failure in failures[:3]: sys.stderr.write(failure[:650] + '\n')
sys.stderr.write(summary[:1800] + '\n')
print(summary[:1800] if failures else 'PASS')
return 1 if failures else 0
if __name__ == '__main__':
raise SystemExit(main())