100 tasks from the HWE arm-linux code-gen pool (outside the lk/man/pth range given to batch 22), each with at least one independent contract-edge defect and plain public docs. Claude reviewed 100/100 USABLE against current digests; no ID or source overlap with main or batch 22. Difficulty is unmeasured. progress.md records the wrap-up: all Codex sessions and Qwen schedulers stopped for the user's rollout, Qwen results so far (57 HARD of 373), and what is left on each batch branch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
412 lines
18 KiB
Python
412 lines
18 KiB
Python
"""Trusted independent Fraction oracle; observer only returns raw API observations."""
|
|
from fractions import Fraction
|
|
from pathlib import Path
|
|
import math
|
|
import random
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
|
|
class Model:
|
|
def __init__(self):
|
|
self.profiles = [None] * 8
|
|
self.revision = 0
|
|
self.counters = [0] * 5
|
|
|
|
@staticmethod
|
|
def profile(value):
|
|
n, d, z, low, high = value
|
|
if not (1 <= n <= 1000000 and 1 <= d <= 1000000):
|
|
return 2, None
|
|
if not -128 <= z <= 127:
|
|
return 3, None
|
|
if not -128 <= low <= z <= high <= 127:
|
|
return 4, None
|
|
scale = Fraction(n, d)
|
|
return 0, (scale.numerator, scale.denominator, z, low, high)
|
|
|
|
@staticmethod
|
|
def valid(value):
|
|
return -1000000000 <= value[0] <= 1000000000 and 1 <= value[1] <= 1000000
|
|
|
|
def encode(self, channel, value, counters):
|
|
n, d, z, low, high = self.profiles[channel]
|
|
scaled = Fraction(value[0], value[1]) / Fraction(n, d)
|
|
rounded = math.floor(abs(scaled) + Fraction(1, 2))
|
|
if scaled < 0:
|
|
rounded = -rounded
|
|
shifted = rounded + z
|
|
if shifted < low:
|
|
counters[3] += 1
|
|
if shifted > high:
|
|
counters[4] += 1
|
|
return min(high, max(low, shifted))
|
|
|
|
def decode(self, channel, value):
|
|
n, d, z, _, _ = self.profiles[channel]
|
|
rational = (value - z) * Fraction(n, d)
|
|
return rational.numerator, rational.denominator
|
|
|
|
@staticmethod
|
|
def physical_channels(layout, channels, frames, block):
|
|
if layout == 0:
|
|
return [c for f in range(frames) for c in range(channels)]
|
|
if layout == 1:
|
|
return [c for c in range(channels) for f in range(frames)]
|
|
return [c for first in range(0, frames, block) for c in range(channels)
|
|
for f in range(first, min(frames, first + block))]
|
|
|
|
def observe(self, op, args):
|
|
if op == 'S':
|
|
channel, *profile = args
|
|
if channel >= 8:
|
|
return [1]
|
|
status, normalized = self.profile(profile)
|
|
if status:
|
|
return [status]
|
|
if self.profiles[channel] != normalized:
|
|
self.profiles[channel] = normalized
|
|
self.revision += 1
|
|
return [0]
|
|
if op == 'R':
|
|
count, null_input, supplied, *flat = args
|
|
if not 1 <= count <= 8:
|
|
return [7]
|
|
if null_input:
|
|
return [8]
|
|
profiles = []
|
|
for i in range(count):
|
|
status, profile = self.profile(flat[i * 5:i * 5 + 5])
|
|
if status:
|
|
return [status]
|
|
profiles.append(profile)
|
|
desired = profiles + [None] * (8 - count)
|
|
if desired != self.profiles:
|
|
self.profiles = desired
|
|
self.revision += 1
|
|
return [0]
|
|
if op == 'E':
|
|
channel = args[0]
|
|
if channel >= 8:
|
|
return [1]
|
|
if self.profiles[channel] is not None:
|
|
self.profiles[channel] = None
|
|
self.revision += 1
|
|
return [0]
|
|
if op == 'X':
|
|
if any(p is not None for p in self.profiles):
|
|
self.revision += 1
|
|
self.profiles = [None] * 8
|
|
return ['X']
|
|
if op == 'G':
|
|
channel = args[0]
|
|
if channel >= 8:
|
|
return [1, 901, 902, 903, 904, 905]
|
|
if self.profiles[channel] is None:
|
|
return [9, 901, 902, 903, 904, 905]
|
|
return [0, *self.profiles[channel]]
|
|
if op == 'Y':
|
|
return [int(args[0] < 8 and self.profiles[args[0]] is not None)]
|
|
if op == 'N':
|
|
return [sum(p is not None for p in self.profiles)]
|
|
if op == 'V':
|
|
return [self.revision]
|
|
if op == 'C':
|
|
return list(self.counters)
|
|
if op == 'Z':
|
|
self.counters = [0] * 5
|
|
return ['Z']
|
|
if op in ('Q', 'D', 'T'):
|
|
channel = args[0]
|
|
sentinel = [-777] if op != 'D' else [919, -313]
|
|
if channel >= 8 or op == 'T' and args[1] >= 8:
|
|
return [1, *sentinel]
|
|
if self.profiles[channel] is None or op == 'T' and self.profiles[args[1]] is None:
|
|
return [9, *sentinel]
|
|
value = args[1:] if op == 'Q' else args[-1]
|
|
if op == 'Q' and not self.valid(value) or op != 'Q' and not -128 <= value <= 127:
|
|
return [5, *sentinel]
|
|
if op == 'Q':
|
|
output = [self.encode(channel, value, self.counters)]
|
|
self.counters[0] += 1
|
|
elif op == 'D':
|
|
output = list(self.decode(channel, value))
|
|
self.counters[1] += 1
|
|
else:
|
|
output = [self.encode(args[1], self.decode(channel, value), self.counters)]
|
|
self.counters[2] += 1
|
|
return [0, *output]
|
|
if op in ('B', 'U'):
|
|
layout, channels, frames, block, count, capacity, ni, no, supplied, *flat = args
|
|
output = [-777] * 258 if op == 'B' else [item for _ in range(258) for item in (919, -313)]
|
|
status = 0
|
|
if layout not in (0, 1, 2):
|
|
status = 6
|
|
elif not (1 <= channels <= 8 and 0 <= frames <= 32 and count == channels * frames):
|
|
status = 7
|
|
elif not (1 <= block <= 16 if layout == 2 else block == 1):
|
|
status = 7
|
|
elif capacity < count or count > 0 and (ni or no):
|
|
status = 8
|
|
elif count and any(self.profiles[c] is None for c in range(channels)):
|
|
status = 9
|
|
elif op == 'B' and any(not self.valid(flat[i * 2:i * 2 + 2]) for i in range(count)):
|
|
status = 5
|
|
elif op == 'U' and any(not -128 <= flat[i] <= 127 for i in range(count)):
|
|
status = 5
|
|
if status:
|
|
return [status, *output]
|
|
for i, c in enumerate(self.physical_channels(layout, channels, frames, block)):
|
|
if op == 'B':
|
|
output[i + 1] = self.encode(c, flat[i * 2:i * 2 + 2], self.counters)
|
|
self.counters[0] += 1
|
|
else:
|
|
output[(i + 1) * 2:(i + 2) * 2] = self.decode(c, flat[i])
|
|
self.counters[1] += 1
|
|
return [0, *output]
|
|
raise RuntimeError('unknown oracle operation')
|
|
|
|
|
|
def cases():
|
|
model = Model()
|
|
inputs, outputs, labels = [], [], []
|
|
|
|
def emit(label, op, *args):
|
|
inputs.append(' '.join(map(str, (op, *args))))
|
|
outputs.append(' '.join(map(str, model.observe(op, args))))
|
|
labels.append(label)
|
|
|
|
def inspect(label):
|
|
for c in range(9):
|
|
emit(label + '/R09/get', 'G', c)
|
|
emit(label + '/R09/ready', 'Y', c)
|
|
for op in ('N', 'V', 'C'):
|
|
emit(label + '/R09/direct-' + op, op)
|
|
|
|
def configure(label, channel, profile):
|
|
emit(label, 'S', channel, *profile)
|
|
inspect(label)
|
|
|
|
def replace(label, profiles, count=None, null=0):
|
|
emit(label, 'R', len(profiles) if count is None else count, null, len(profiles),
|
|
*(field for profile in profiles for field in profile))
|
|
inspect(label)
|
|
|
|
def batch(label, op, layout, channels, frames, block, values, count=None, cap=None, ni=0, no=0):
|
|
count = len(values) if count is None else count
|
|
flat = [x for pair in values for x in pair] if op == 'B' else list(values)
|
|
emit(label, op, layout, channels, frames, block, count,
|
|
count if cap is None else cap, ni, no, len(values), *flat)
|
|
inspect(label)
|
|
|
|
inspect('R01/R06/initial')
|
|
emit('R08/reset-empty', 'X')
|
|
emit('R10/clear-empty', 'Z')
|
|
inspect('R08/R10/idempotent-empty')
|
|
for c in (0, 7, 8, 999):
|
|
emit('R11/notready-index-value-Q', 'Q', c, 1000000001, 0)
|
|
emit('R11/notready-index-value-D', 'D', c, 128)
|
|
emit('R12/notready-index-value-T', 'T', c, 8, 128)
|
|
inspect('R11/R12/failure-state')
|
|
|
|
base = (1, 1, 0, -128, 127)
|
|
configure('R02/R06/canonical', 0, (6, 8, 0, -128, 127))
|
|
configure('R06/idempotent-canonical', 0, (3, 4, 0, -128, 127))
|
|
configure('R01/R02/last-channel', 7, (1000000, 1000000, 127, 127, 127))
|
|
invalid_profiles = [
|
|
(0, 1, 0, -128, 127), (1000001, 1, 0, -128, 127),
|
|
(1, 0, 0, -128, 127), (1, 1000001, 0, -128, 127),
|
|
(1, 1, -129, -128, 127), (1, 1, 128, -128, 127),
|
|
(1, 1, 0, -129, 127), (1, 1, 0, 128, 127),
|
|
(1, 1, 0, -128, -129), (1, 1, 0, -128, 128),
|
|
(1, 1, 0, 1, 127), (1, 1, 0, -128, -1),
|
|
(1, 1, 0, 4, -4), (0, 0, 128, 5, -5),
|
|
(1, 1, 128, 5, -5), (1000001, 1000001, 0, -128, 127),
|
|
(2000000, 2, 0, -128, 127)]
|
|
for profile in invalid_profiles:
|
|
configure('R02/R06/invalid-profile-atomic', 0, profile)
|
|
configure('R06/index-priority', 8, invalid_profiles[-2])
|
|
for n, d in ((1, 1000000), (1000000, 1)):
|
|
configure('R02/scale-domain-endpoints', 7, (n, d, -128, -128, 127))
|
|
replace('R07/zero-count-before-null', [], 0, 1)
|
|
replace('R07/large-count-before-null', [], 9, 1)
|
|
replace('R07/null', [], 1, 1)
|
|
replace('R07/late-invalid-atomic', [base, base, invalid_profiles[4]])
|
|
replace('R07/profile-index-priority', [invalid_profiles[4], invalid_profiles[0]])
|
|
replace('R07/full-bank', [base] * 8)
|
|
replace('R07/canonical-whole', [(2, 2, 0, -128, 127)] * 8)
|
|
replace('R07/shrink', [(2, 255, -128, -128, 127)])
|
|
replace('R07/repeated-shrink', [(4, 510, -128, -128, 127)])
|
|
emit('R08/erase-invalid', 'E', 8)
|
|
emit('R08/erase-empty', 'E', 7)
|
|
emit('R08/erase-live', 'E', 0)
|
|
emit('R08/erase-repeat', 'E', 0)
|
|
inspect('R08/erase-state')
|
|
|
|
replace('R19/public-profiles', [base, (2, 1, 0, -128, 127)])
|
|
batch('R19/public-replay', 'B', 2, 2, 3, 2, [(2, 1)] * 6)
|
|
configure('R04/negative-half-profile', 0, (1, 1, 1, -128, 127))
|
|
emit('R04/signed-round-before-zero-negative-half', 'Q', 0, -1, 2)
|
|
inspect('R04/negative-half-state')
|
|
replace('R05/convert-half-profiles', [(1, 2, 0, -128, 127), (1, 1, 1, -128, 127)])
|
|
emit('R04/R05/convert-negative-half-direct', 'T', 0, 1, -1)
|
|
emit('R04/R05/convert-positive-half-direct', 'T', 0, 1, 1)
|
|
inspect('R04/R05/convert-half-counters')
|
|
|
|
rng = random.Random(18827010)
|
|
profiles = [(2, 255, -128, -128, 127), (1, 1000000, 127, -128, 127),
|
|
(1000000, 1, -128, -128, 127), (999983, 999979, 17, -23, 89),
|
|
(1, 1, 0, 0, 0), (1, 1, -1, -128, 127),
|
|
(1, 1, 1, -128, 127), (3, 7, 0, -128, 127)]
|
|
replace('R02/R03/scalar-profile-extremes', profiles)
|
|
values = [(0, 1), (0, 1000000), (1000000000, 1), (-1000000000, 1),
|
|
(1000000000, 1000000), (-1000000000, 1000000), (1, 1), (2, 1),
|
|
(-1, 2), (1, 2), (-499999, 1000000), (-500001, 1000000),
|
|
(499999, 1000000), (500001, 1000000), (-127, 1), (128, 1)]
|
|
values += [(rng.randint(-1000000000, 1000000000), rng.randint(1, 1000000)) for _ in range(60)]
|
|
for c in range(8):
|
|
for value in values:
|
|
emit('R03/R04/exact-scalar', 'Q', c, *value)
|
|
for code in range(-128, 128):
|
|
emit('R03/R05/all-dequant-codes', 'D', c, code)
|
|
for value in ((-1000000001, 1), (1000000001, 1), (0, 0), (0, -1), (0, 1000001)):
|
|
emit('R03/R11/value-boundaries', 'Q', c, *value)
|
|
for code in (-129, 128):
|
|
emit('R03/R11/code-boundaries', 'D', c, code)
|
|
inspect('R10/R11/scalar-counts')
|
|
for zero in (-128, -120, -1, 0, 1, 120, 127):
|
|
configure('R04/half-sign-profile', 0, (1, 1, zero, -128, 127))
|
|
for k in (-511, -255, -3, -1, 0, 1, 3, 255, 511):
|
|
emit('R04/signed-round-before-zero', 'Q', 0, k, 2)
|
|
for k in (-128 - zero, 127 - zero):
|
|
for numerator in (k * 1000000 - 1, k * 1000000, k * 1000000 + 1):
|
|
emit('R04/clip-equality-and-neighbors', 'Q', 0, numerator, 1000000)
|
|
inspect('R04/clip-observation')
|
|
replace('R05/convert-profiles', profiles)
|
|
for source in range(8):
|
|
for destination in range(8):
|
|
for code in (-128, -127, -1, 0, 1, 126, 127):
|
|
emit('R05/R12/convert-direct', 'T', source, destination, code)
|
|
for source, destination in ((0, 0), (0, 7), (7, 0)):
|
|
for code in (-129, 128):
|
|
emit('R12/convert-value', 'T', source, destination, code)
|
|
emit('R12/source-index', 'T', 8, 0, 128)
|
|
emit('R12/dest-index', 'T', 0, 8, 128)
|
|
emit('R08/counters-retained-erase', 'E', 7)
|
|
emit('R12/source-ready', 'T', 7, 0, 128)
|
|
emit('R12/dest-ready', 'T', 0, 7, 128)
|
|
inspect('R05/R12/convert-counters')
|
|
configure('R02/R06/error-retains-nonzero-counters', 0, invalid_profiles[-2])
|
|
replace('R07/error-retains-nonzero-counters', [base, invalid_profiles[0]])
|
|
emit('R08/error-retains-nonzero-counters', 'E', 8)
|
|
for c in (7, 8):
|
|
emit('R11/error-retains-nonzero-counters-Q', 'Q', c, 1000000001, 0)
|
|
emit('R11/error-retains-nonzero-counters-D', 'D', c, 128)
|
|
inspect('R06/R07/R08/R11/nonzero-error-atomicity')
|
|
emit('R10/clear', 'Z')
|
|
inspect('R10/clear-state')
|
|
emit('R10/clear-repeat', 'Z')
|
|
emit('R08/reset-live', 'X')
|
|
inspect('R08/reset-live-state')
|
|
emit('R08/reset-repeat', 'X')
|
|
inspect('R08/reset-repeat-state')
|
|
configure('R15/seed-counter-profile', 0, base)
|
|
emit('R15/seed-nonzero-counters', 'Q', 0, -1000000000, 1)
|
|
emit('R15/unready-with-counters', 'E', 0)
|
|
inspect('R15/invalid-batch-retains-counter-seed')
|
|
|
|
for op in ('B', 'U'):
|
|
empty = []
|
|
batch('R15/empty-unready-null', op, 0, 8, 0, 1, empty, ni=1, no=1)
|
|
batch('R13/layout-priority', op, 99, 0, 33, 0, empty, count=7, cap=0, ni=1, no=1)
|
|
batch('R13/negative-layout', op, -1, 0, 0, 0, empty, ni=1, no=1)
|
|
for layout, ch, fr, block, count in ((0, 0, 0, 1, 0), (0, 9, 0, 1, 0),
|
|
(0, 1, 33, 1, 33), (0, 1, 1, 2, 1), (1, 1, 1, 0, 1),
|
|
(2, 1, 1, 0, 1), (2, 1, 1, 17, 1), (0, 1, 1, 1, 0),
|
|
(2, 8, 32, 16, 257)):
|
|
batch('R14/shape-before-buffer', op, layout, ch, fr, block, empty,
|
|
count=count, cap=0, ni=1, no=1)
|
|
value = [(0, 0)] if op == 'B' else [128]
|
|
for ni, no, cap in ((0, 0, 0), (1, 0, 1), (0, 1, 1), (1, 1, 0)):
|
|
batch('R14/buffer-before-ready', op, 0, 1, 1, 1, value, cap=cap, ni=ni, no=no)
|
|
batch('R15/ready-before-value', op, 0, 1, 1, 1, value)
|
|
replace('R16/batch-profiles', profiles)
|
|
for op in ('B', 'U'):
|
|
for channels in (1, 2, 3, 8):
|
|
for frames in (0, 1, 2, 3, 7, 16, 32):
|
|
for layout, block in ((0, 1), (1, 1), (2, 1), (2, 2), (2, 3), (2, 16)):
|
|
count = channels * frames
|
|
if op == 'B':
|
|
data = [values[(i * 17 + channels + frames) % len(values)] for i in range(count)]
|
|
else:
|
|
data = [rng.randint(-128, 127) for _ in range(count)]
|
|
batch('R14/R16/R17/all-layouts-and-tails', op, layout, channels, frames,
|
|
block, data, cap=min(256, count + 2))
|
|
if channels == 2 and frames == 3 and layout == 2 and block == 2 and op == 'B':
|
|
batch('R19/public-replay', 'B', 2, 2, 3, 2, [(2, 1)] * 6)
|
|
data = [(2, 1), (3, 1), (4, 1), (1000000001, 1)] if op == 'B' else [0, -128, 127, 128]
|
|
batch('R15/late-value-atomic', op, 2, 2, 2, 2, data, cap=6)
|
|
if op == 'B':
|
|
for value in ((-1000000001, 1), (1000000001, 1), (0, 0), (0, -1), (0, 1000001)):
|
|
batch('R03/R15/batch-value-domains-atomic', op, 0, 1, 2, 1,
|
|
[(2, 1), value], cap=4)
|
|
else:
|
|
batch('R03/R15/batch-negative-code-invalid', op, 0, 1, 2, 1, [0, -129], cap=4)
|
|
emit('R15/remove-second-channel', 'E', 1)
|
|
batch('R15/late-ready-atomic', op, 1, 2, 2, 1, data, cap=6)
|
|
configure('R15/restore-second-channel', 1, profiles[1])
|
|
inspect('R10/R16/final-counters')
|
|
emit('R08/reset-retains-nonzero-counters', 'X')
|
|
inspect('R08/reset-nonzero-state')
|
|
emit('R10/clear-after-reset', 'Z')
|
|
inspect('R10/clear-after-reset-state')
|
|
return ('\n'.join(inputs) + '\n').encode(), ('\n'.join(outputs) + '\n').encode(), labels
|
|
|
|
|
|
def main():
|
|
candidate = Path(sys.argv[1])
|
|
sources = {name: (candidate / name).read_bytes()
|
|
for name in ('quant.hpp', 'numeric.cpp', 'bank.cpp', 'tensor.cpp')}
|
|
observer = Path(__file__).with_name('observer.cpp').read_bytes()
|
|
input_bytes, expected, labels = cases()
|
|
scratch = Path(tempfile.mkdtemp(prefix='aq-observe-', dir=Path.cwd()))
|
|
for name, data in sources.items():
|
|
(scratch / name).write_bytes(data)
|
|
(scratch / 'observer.cpp').write_bytes(observer)
|
|
# All trusted filesystem IO is complete. Compiler/candidate may now control scratch.
|
|
argv = ['g++', '-std=c++17', '-O2', '-Wall', '-Wextra', 'numeric.cpp', 'bank.cpp',
|
|
'tensor.cpp', 'observer.cpp', '-o', 'observe']
|
|
built = subprocess.run(argv, cwd=scratch, capture_output=True, timeout=30)
|
|
sys.stderr.write('AQ_BUILD returncode=' + str(built.returncode) + '\n')
|
|
if built.returncode:
|
|
print('build=' + str(built.returncode) + ' compile rejected')
|
|
return 1
|
|
observed = subprocess.run([str(scratch / 'observe')], input=input_bytes,
|
|
capture_output=True, timeout=20)
|
|
sys.stderr.write('AQ_RUN returncode=' + str(observed.returncode) + ' observations=' + str(len(labels)) +
|
|
' stdout_bytes=' + str(len(observed.stdout)) + ' stderr_bytes=' + str(len(observed.stderr)) + '\n')
|
|
print('build=0 run=' + str(observed.returncode) + ' observations=' + str(len(labels)))
|
|
if observed.returncode:
|
|
print('candidate exited before complete observation')
|
|
return 1
|
|
if observed.stderr:
|
|
print('R00 stderr must be empty; bytes=' + str(len(observed.stderr)))
|
|
return 1
|
|
if observed.stdout != expected:
|
|
actual_lines = observed.stdout.split(b'\n')
|
|
expected_lines = expected.split(b'\n')
|
|
for index, (actual, wanted) in enumerate(zip(actual_lines, expected_lines)):
|
|
if actual != wanted:
|
|
label = labels[index] if index < len(labels) else 'R00/trailing-output'
|
|
print('full stdout mismatch at observation=' + str(index + 1) + ' ' + label)
|
|
return 1
|
|
print('R00 full stdout length mismatch; bytes=' + str(len(observed.stdout)))
|
|
return 1
|
|
print('PASS')
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|