amos 69393b824c Add 5 more Qwen-measured tasks from batches 26 and 27 (1 HARD, 4 MEDIUM)
Batch 26 commit 7f99538 measured 1 HARD (FreeRTOS loopback and mDNS
state, where Qwen ran out of context in all three attempts), 3 MEDIUM
(canmatrix JSON round trip, Zephyr byte-value formatting, Contiki event
timer sleep) and 4 ROUTINE; two tasks still need more valid attempts.
From batch 27 the lazy integer expression task becomes MEDIUM once its
top-up attempts are added.

Each task comes from the commit Qwen measured, its digest matches, and it
is still present at the batch branch head. Attempts made while /home was
below the 20 GiB sampling floor are recorded as disk_low and left out.

Batches 26, 28 and 29 are stopped: after a login refresh on 11-03 the
Codex account no longer offers the sol models they use.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-04 15:25:47 +08:00

303 lines
15 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Protected AST generator and bytecode oracle; candidate receives arguments only."""
import json
import operator
import random
import secrets
import shutil
import subprocess
import sys
import tempfile
from fractions import Fraction
from pathlib import Path
LOW, HIGH = -2147483648, 2147483647
SEED, RANDOM_CASES = 27031109, 3200
CODEC = r'''
_type, _isinstance = type, isinstance
_int, _float, _str, _bool = int, float, str, bool
_dict, _list, _tuple, _bytes = dict, list, tuple, bytes
_repr = repr
class OrdinaryInt(int): pass
class OrdinaryString(str): pass
class OrdinaryDict(dict): pass
def pack(x):
cls = _type(x)
name = cls.__module__ + '.' + cls.__qualname__
if x is None: kind, value = 'none', None
elif _isinstance(x, _bool): kind, value = 'bool', _bool(x)
elif _isinstance(x, _int): kind, value = ('int' if cls is _int else 'int_sub'), _int(x)
elif _isinstance(x, _float): kind, value = 'float', _repr(x)
elif _isinstance(x, _str): kind, value = ('str' if cls is _str else 'str_sub'), _str(x)
elif _isinstance(x, _bytes): kind, value = 'bytes', x.hex()
elif _isinstance(x, _dict): kind, value = ('dict' if cls is _dict else 'dict_sub'), [[pack(k), pack(v)] for k, v in _dict.items(x)]
elif _isinstance(x, _list): kind, value = 'list', [pack(v) for v in _list.__iter__(x)]
elif _isinstance(x, _tuple): kind, value = 'tuple', [pack(v) for v in _tuple.__iter__(x)]
else: kind, value = 'other', None
return [kind, name, value]
def unpack(row):
kind, name, value = row
if kind == 'none': return None
if kind in ('bool', 'int', 'str'): return value
if kind == 'int_sub': return OrdinaryInt(value)
if kind == 'str_sub': return OrdinaryString(value)
if kind == 'float': return _float(value)
if kind == 'bytes': return _bytes.fromhex(value)
if kind in ('dict', 'dict_sub'):
result = _dict((unpack(k), unpack(v)) for k, v in value)
return result if kind == 'dict' else OrdinaryDict(result)
if kind == 'list': return [unpack(v) for v in value]
if kind == 'tuple': return _tuple(unpack(v) for v in value)
raise RuntimeError('transport input')
'''
exec(CODEC)
RUNNER = CODEC + r'''
import json, sys
_value_error, _name_error = ValueError, NameError
_overflow_error, _zero_error = OverflowError, ZeroDivisionError
_exception = BaseException
families = [('ValueError', _value_error), ('NameError', _name_error),
('OverflowError', _overflow_error), ('ZeroDivisionError', _zero_error)]
requests = json.load(sys.stdin)
sys.path.insert(0, sys.argv[1])
import lazy_integer
rows = []
for request in requests:
args = unpack(request)
before = pack(args)
try:
value = lazy_integer.evaluate(*args)
except _exception as error:
row = {'families': {name: _isinstance(error, cls) for name, cls in families},
'mro': [cls.__module__ + '.' + cls.__qualname__ for cls in _type(error).__mro__]}
else:
row = {'value': pack(value)}
row['before'], row['after'] = before, pack(args)
rows.append(row)
print('RESULT:' + sys.argv[2] + ':' + json.dumps(rows, ensure_ascii=True))
'''
def oracle(tree, bindings):
"""Compile a generated tree to control-flow bytecode, then interpret integers."""
program = []
def emit(node):
kind = node[0]
if kind in ('literal', 'symbol'):
program.append((kind, node[1]))
elif kind == 'prefix':
emit(node[2]); program.append(('unary', node[1]))
elif kind == 'select':
emit(node[1]); branch = len(program); program.append(None)
emit(node[2]); finish = len(program); program.append(None)
program[branch] = ('if_false', len(program))
emit(node[3]); program[finish] = ('jump', len(program))
elif node[1] in ('&&', '||'):
emit(node[2]); branch = len(program); program.append(None)
emit(node[3]); program.append(('boolean', None))
program[branch] = (node[1], len(program))
else:
emit(node[2]); emit(node[3]); program.append(('binary', node[1]))
emit(tree)
values, pc = [], 0
comparisons = {'<': operator.lt, '<=': operator.le, '>': operator.gt,
'>=': operator.ge, '==': operator.eq, '!=': operator.ne}
while pc < len(program):
kind, arg = program[pc]; pc += 1
if kind == 'jump': pc = arg; continue
if kind == 'if_false':
if values.pop() == 0: pc = arg
continue
if kind in ('&&', '||'):
first = values.pop()
if (first == 0) == (kind == '&&'):
values.append(int(kind == '||')); pc = arg
continue
if kind == 'literal': value = arg
elif kind == 'symbol':
if arg not in bindings: raise NameError(arg)
value = bindings[arg]
elif kind == 'boolean': value = int(values.pop() != 0)
elif kind == 'unary':
first = values.pop()
value = first if arg == '+' else -first if arg == '-' else int(first == 0)
else:
second, first = values.pop(), values.pop()
if arg in comparisons: value = int(comparisons[arg](first, second))
elif arg == '+': value = first + second
elif arg == '-': value = first - second
elif arg == '*': value = first * second
else:
if second == 0: raise ZeroDivisionError()
quotient = int(Fraction(first, second))
value = quotient if arg == '/' else first - quotient*second
if not LOW <= value <= HIGH: raise OverflowError()
values.append(int(value))
if len(values) != 1: raise RuntimeError('oracle stack')
return values[0]
PRECEDENCE = {'||': 1, '&&': 2, '==': 3, '!=': 3, '<': 4, '<=': 4,
'>': 4, '>=': 4, '+': 5, '-': 5, '*': 6, '/': 6, '%': 6}
def rendering(node, rng):
kind = node[0]
def wrap(tokens): return ['('] + tokens + [')']
if kind == 'literal': words, rank = [str(node[1])], 9
elif kind == 'symbol': words, rank = [node[1]], 9
elif kind == 'prefix':
child, child_rank = rendering(node[2], rng)
words, rank = [node[1]] + (wrap(child) if child_rank < 7 else child), 7
elif kind == 'binary':
left, lr = rendering(node[2], rng); right, rr = rendering(node[3], rng)
rank = PRECEDENCE[node[1]]
words = (wrap(left) if lr < rank else left) + [node[1]] + (wrap(right) if rr <= rank else right)
else:
condition, cr = rendering(node[1], rng)
yes, _ = rendering(node[2], rng); no, _ = rendering(node[3], rng)
words, rank = (wrap(condition) if cr == 0 else condition) + ['?'] + yes + [':'] + no, 0
if rng.randrange(7) == 0: return wrap(words), 9
return words, rank
MISSING = object()
def cases():
requests, expected, labels = [], [], []
def add(text, bindings=MISSING, value=None, error=None, label='manual'):
requests.append([text, {} if bindings is MISSING else bindings])
expected.append(('error', error) if error else ('value', value)); labels.append(label)
def ok(text, value, bindings=MISSING, label='semantics'): add(text, bindings, value, label=label)
def err(text, kind, bindings=MISSING, label='errors'): add(text, bindings, error=kind, label=label)
basic = [('0', 0), ('0000012', 12), (' \t\r\n + 000 \n', 0), ('1+2*3', 7),
('18/3/2', 3), ('10-3-2', 5), ('2<1<1', 1), ('2==1==0', 1),
('2<3==1', 1), ('1==2<3', 1), ('1||0&&0', 1), ('0||2&&3', 1),
('!0+!!3', 2), ('-+-2', 2), ('-(2+3)*2', -10), ('7/-3', -2),
('-7/3', -2), ('-7/-3', 2), ('-7%3', -1), ('7%-3', 1),
('-7%-3', -1), ('0/-2', 0), ('0%-2', 0), ('1?2:0?3:4', 2),
('0?2:0?3:4', 4), ('1?0?3:4:5', 4), ('(0?2:3)+1', 4),
('0?1:2?3:4', 3), ('1+2?5:6', 5), ('0?1:2+3', 5),
('0&&missing', 0), ('2||missing', 1), ('0?missing:7', 7),
('1?7:1/0', 7), ('0&&2147483648', 0), ('1||2147483648', 1),
('0?-2147483648:8', 8), ('1?8:-2147483648', 8),
('2147483647', HIGH), ('-2147483647-1', LOW),
('x%-1', 0), ('x+0', LOW), ('y*1', HIGH), ('x<y', 1),
('x==x', 1), ('x!=x', 0), ('y>=y', 1), ('x<=x', 1),
('x>y', 0), ('3&&-2', 1), ('-2||0', 1), ('0||-2', 1),
('2&&0', 0), ('0||0', 0), ('!x', 0), ('+x', LOW)]
for text, value in basic: ok(text, value, {'x': LOW, 'y': HIGH})
for a in range(-4, 5):
for b in range(-4, 5):
if b:
ok('a/b', int(Fraction(a, b)), {'a': a, 'b': b}, 'division-small-signs')
ok('a%b', a-int(Fraction(a, b))*b, {'a': a, 'b': b}, 'remainder-small-signs')
for text in ['2147483648', '-2147483648', '2147483647+1', '(2147483647+1)-1',
'x/-1', '-x', 'y*2', '0-(2147483647+2)', '0?0:2147483648']:
err(text, 'OverflowError', {'x': LOW, 'y': HIGH}, 'overflow-visited-intermediate')
for text in ['1/0', '1%0', '0/0', '0%0', '1&&1/0', '0||1%0']:
err(text, 'ZeroDivisionError', label='zero-divisor')
for text, kind in [('missing+1/0','NameError'), ('1/0+missing','ZeroDivisionError'),
('2147483648+missing','OverflowError'), ('missing+2147483648','NameError'),
('(1/0)/(missing)','ZeroDivisionError'), ('missing?1/0:0','NameError'),
('1?missing:1/0','NameError'), ('0?1/0:missing','NameError')]:
err(text, kind, label='runtime-first-error')
bad_syntax = ['', ' ', '1 2', 'a b', '2a', '1+', '+', '()', '(1', '1)',
'1**2', '1//2', '1&2', '1|2', '1=2', '1<>2', '1 & & 2',
'1 | | 2', '1 ! = 2', '1< =2', '1?2', '1?:2', '?1:2',
'1?2:', '1:2', '1?2:3:4', 'f(1)', '0x10', '1.0',
'1_0', 'é', '1', 'abc'+'x'*30, '1\v+2', '1\f+2', '1\0',
'0&&(', '1||1+', '1?7:(1+)', 'missing+(', '1/0+(',
'2147483648+(', '0?7:1?2:']
for text in bad_syntax: err(text, 'ValueError', label='whole-lexical-syntax-before-runtime')
for text in [None, True, 3, 1.0, b'1', [], {}, (), '0'*4097]:
err(text, 'ValueError', label='expression-domain')
for bindings in [None, [], (), True, {'a': True}, {'a': 1.0}, {'a': LOW-1},
{'a': HIGH+1}, {'a': 10**500}, {1: 0}, {'': 0}, {'1x': 0},
{'é': 0}, {'a'*33: 0}, {'a-b': 0}, {'a.b': 0},
{'k'+str(i): i for i in range(65)}]:
err('0', 'ValueError', bindings, 'binding-domain-unused')
err('missing', 'ValueError', {'unused': True}, 'input-before-runtime')
err('1/0', 'ValueError', {'unused': HIGH+1}, 'input-before-runtime')
ok('A_09+a_09', 5, {'A_09': 2, 'a_09': 3, 'unused': LOW}, 'names-case-unused')
ok('a'*32, 1, {'a'*32: 1}, 'name-length-endpoint')
ok('k63', 63, {'k'+str(i): i for i in range(64)}, 'binding-count-endpoint')
ok(OrdinaryString('x+1'), 3, { 'x': 2 }, 'expression-subclass')
ok('x', 3, OrdinaryDict(x=3), 'bindings-subclass')
ok('x', 4, {OrdinaryString('x'): 4}, 'binding-name-subclass')
ok('x', 5, {'x': OrdinaryInt(5)}, 'binding-int-subclass')
ok(OrdinaryString('x<y?x:y'), LOW,
OrdinaryDict({OrdinaryString('x'): OrdinaryInt(LOW), 'y': OrdinaryInt(HIGH)}), 'all-subclasses-endpoints')
err('0', 'ValueError', {'x': OrdinaryInt(HIGH+1)}, 'subclass-outside-range')
for text, value in [('+'*4095+'0', 0), ('!'*4095+'0', 1),
('('*2047+'0'+')'*2047, 0), ('0'*4096, 0),
('1+'*2047+'1 ', 2048), ('1?'*1023+'0'+':0'*1023, 0)]:
ok(text, value, label='maximum-length-deep-grammar')
err('9'*4096, 'OverflowError', label='maximum-literal-runtime')
ok('0&&'+'9'*4093, 0, label='maximum-literal-skipped')
err('('*2047+'0'+')'*2046, 'ValueError', label='deep-unclosed-syntax')
for value in [2, -3, 0, LOW, HIGH]: ok('x', value, {'x': value}, 'sequential-state')
rng = random.Random(SEED)
binary = list(PRECEDENCE)
def generate(depth):
if depth == 0 or rng.randrange(5) == 0:
if rng.randrange(3) == 0: return ('symbol', rng.choice(['a', 'b', 'c', 'missing']))
value = rng.choice([0, 0, 1, 2, 3, HIGH, HIGH+1]) if rng.randrange(12) == 0 else rng.randrange(5)
return ('literal', value)
form = rng.randrange(5)
if form == 0: return ('prefix', rng.choice(['+', '-', '!']), generate(depth-1))
if form == 1: return ('select', generate(depth-1), generate(depth-1), generate(depth-1))
return ('binary', rng.choice(binary), generate(depth-1), generate(depth-1))
for _ in range(RANDOM_CASES):
tree = generate(rng.randrange(1, 6))
bindings = {name: rng.choice([-3, -2, -1, 0, 0, 1, 2, 3]) for name in ['a', 'b', 'c']}
if rng.randrange(4) == 0: del bindings[rng.choice(['a', 'b', 'c'])]
if rng.randrange(12) == 0: bindings[rng.choice(['a', 'b', 'c'])] = rng.choice([LOW, HIGH])
words, _ = rendering(tree, rng)
text = ''.join(word + rng.choice(['', '', ' ', '\t', '\r\n']) for word in words)
try: value = oracle(tree, bindings)
except (NameError, OverflowError, ZeroDivisionError) as error: err(text, type(error).__name__, bindings, 'random-ast')
else: ok(text, value, bindings, 'random-ast')
return requests, expected, labels
def require(condition, label):
if not condition: raise AssertionError(label)
def main():
requests, expected, labels = cases()
payload = [pack(args) for args in requests]
scratch = Path(tempfile.mkdtemp(prefix='lazy-integer-check-', dir=Path.cwd()))
source = Path(sys.argv[1])/'lazy_integer.py'
require(source.is_file() and not source.is_symlink(), 'missing candidate')
shutil.copyfile(source, scratch/'lazy_integer.py')
(scratch/'runner.py').write_text(RUNNER)
nonce = secrets.token_hex(24)
# All trusted reads/writes of the candidate-writable tree end before spawn.
result = subprocess.run([sys.executable, '-I', '-B', str(scratch/'runner.py'), str(scratch), nonce],
input=json.dumps(payload, ensure_ascii=True), text=True,
capture_output=True, cwd=scratch, timeout=40)
require(result.returncode == 0, 'candidate execution failed')
prefix = 'RESULT:' + nonce + ':'
rows = [line[len(prefix):] for line in result.stdout.splitlines() if line.startswith(prefix)]
require(len(rows) == 1, 'observation marker count')
observed = json.loads(rows[0])
require(isinstance(observed, list) and len(observed) == len(expected), 'observation count')
for i, (row, want, label, raw) in enumerate(zip(observed, expected, labels, payload)):
context = str(i) + ':' + label
require(isinstance(row, dict) and row.get('before') == raw and row.get('after') == raw, context + ':input changed')
kind, value = want
if kind == 'error':
require(row.get('families', {}).get(value) is True and 'builtins.'+value in row.get('mro', []), context + ':exception '+value)
else:
actual = row.get('value', [])
require(len(actual) == 3 and actual[0] in ('int', 'int_sub') and type(actual[2]) is int and actual[2] == value, context + ':integer result')
print('validated cases=' + str(len(expected)) + ' random=' + str(RANDOM_CASES) + ' seed=' + str(SEED))
print('PASS')
if __name__ == '__main__':
try: main()
except Exception as error:
print(('FAIL: ' + type(error).__name__ + ': ' + str(error)).replace('\n', ' ')[:400])
sys.exit(1)