Batch 26 commit 7f99538 measured 1 HARD (FreeRTOS loopback and mDNS state, where Qwen ran out of context in all three attempts), 3 MEDIUM (canmatrix JSON round trip, Zephyr byte-value formatting, Contiki event timer sleep) and 4 ROUTINE; two tasks still need more valid attempts. From batch 27 the lazy integer expression task becomes MEDIUM once its top-up attempts are added. Each task comes from the commit Qwen measured, its digest matches, and it is still present at the batch branch head. Attempts made while /home was below the 20 GiB sampling floor are recorded as disk_low and left out. Batches 26, 28 and 29 are stopped: after a login refresh on 11-03 the Codex account no longer offers the sol models they use. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
303 lines
15 KiB
Python
303 lines
15 KiB
Python
"""Protected AST generator and bytecode oracle; candidate receives arguments only."""
|
||
import json
|
||
import operator
|
||
import random
|
||
import secrets
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
from fractions import Fraction
|
||
from pathlib import Path
|
||
|
||
LOW, HIGH = -2147483648, 2147483647
|
||
SEED, RANDOM_CASES = 27031109, 3200
|
||
CODEC = r'''
|
||
_type, _isinstance = type, isinstance
|
||
_int, _float, _str, _bool = int, float, str, bool
|
||
_dict, _list, _tuple, _bytes = dict, list, tuple, bytes
|
||
_repr = repr
|
||
class OrdinaryInt(int): pass
|
||
class OrdinaryString(str): pass
|
||
class OrdinaryDict(dict): pass
|
||
def pack(x):
|
||
cls = _type(x)
|
||
name = cls.__module__ + '.' + cls.__qualname__
|
||
if x is None: kind, value = 'none', None
|
||
elif _isinstance(x, _bool): kind, value = 'bool', _bool(x)
|
||
elif _isinstance(x, _int): kind, value = ('int' if cls is _int else 'int_sub'), _int(x)
|
||
elif _isinstance(x, _float): kind, value = 'float', _repr(x)
|
||
elif _isinstance(x, _str): kind, value = ('str' if cls is _str else 'str_sub'), _str(x)
|
||
elif _isinstance(x, _bytes): kind, value = 'bytes', x.hex()
|
||
elif _isinstance(x, _dict): kind, value = ('dict' if cls is _dict else 'dict_sub'), [[pack(k), pack(v)] for k, v in _dict.items(x)]
|
||
elif _isinstance(x, _list): kind, value = 'list', [pack(v) for v in _list.__iter__(x)]
|
||
elif _isinstance(x, _tuple): kind, value = 'tuple', [pack(v) for v in _tuple.__iter__(x)]
|
||
else: kind, value = 'other', None
|
||
return [kind, name, value]
|
||
def unpack(row):
|
||
kind, name, value = row
|
||
if kind == 'none': return None
|
||
if kind in ('bool', 'int', 'str'): return value
|
||
if kind == 'int_sub': return OrdinaryInt(value)
|
||
if kind == 'str_sub': return OrdinaryString(value)
|
||
if kind == 'float': return _float(value)
|
||
if kind == 'bytes': return _bytes.fromhex(value)
|
||
if kind in ('dict', 'dict_sub'):
|
||
result = _dict((unpack(k), unpack(v)) for k, v in value)
|
||
return result if kind == 'dict' else OrdinaryDict(result)
|
||
if kind == 'list': return [unpack(v) for v in value]
|
||
if kind == 'tuple': return _tuple(unpack(v) for v in value)
|
||
raise RuntimeError('transport input')
|
||
'''
|
||
exec(CODEC)
|
||
RUNNER = CODEC + r'''
|
||
import json, sys
|
||
_value_error, _name_error = ValueError, NameError
|
||
_overflow_error, _zero_error = OverflowError, ZeroDivisionError
|
||
_exception = BaseException
|
||
families = [('ValueError', _value_error), ('NameError', _name_error),
|
||
('OverflowError', _overflow_error), ('ZeroDivisionError', _zero_error)]
|
||
requests = json.load(sys.stdin)
|
||
sys.path.insert(0, sys.argv[1])
|
||
import lazy_integer
|
||
rows = []
|
||
for request in requests:
|
||
args = unpack(request)
|
||
before = pack(args)
|
||
try:
|
||
value = lazy_integer.evaluate(*args)
|
||
except _exception as error:
|
||
row = {'families': {name: _isinstance(error, cls) for name, cls in families},
|
||
'mro': [cls.__module__ + '.' + cls.__qualname__ for cls in _type(error).__mro__]}
|
||
else:
|
||
row = {'value': pack(value)}
|
||
row['before'], row['after'] = before, pack(args)
|
||
rows.append(row)
|
||
print('RESULT:' + sys.argv[2] + ':' + json.dumps(rows, ensure_ascii=True))
|
||
'''
|
||
|
||
|
||
def oracle(tree, bindings):
|
||
"""Compile a generated tree to control-flow bytecode, then interpret integers."""
|
||
program = []
|
||
def emit(node):
|
||
kind = node[0]
|
||
if kind in ('literal', 'symbol'):
|
||
program.append((kind, node[1]))
|
||
elif kind == 'prefix':
|
||
emit(node[2]); program.append(('unary', node[1]))
|
||
elif kind == 'select':
|
||
emit(node[1]); branch = len(program); program.append(None)
|
||
emit(node[2]); finish = len(program); program.append(None)
|
||
program[branch] = ('if_false', len(program))
|
||
emit(node[3]); program[finish] = ('jump', len(program))
|
||
elif node[1] in ('&&', '||'):
|
||
emit(node[2]); branch = len(program); program.append(None)
|
||
emit(node[3]); program.append(('boolean', None))
|
||
program[branch] = (node[1], len(program))
|
||
else:
|
||
emit(node[2]); emit(node[3]); program.append(('binary', node[1]))
|
||
emit(tree)
|
||
values, pc = [], 0
|
||
comparisons = {'<': operator.lt, '<=': operator.le, '>': operator.gt,
|
||
'>=': operator.ge, '==': operator.eq, '!=': operator.ne}
|
||
while pc < len(program):
|
||
kind, arg = program[pc]; pc += 1
|
||
if kind == 'jump': pc = arg; continue
|
||
if kind == 'if_false':
|
||
if values.pop() == 0: pc = arg
|
||
continue
|
||
if kind in ('&&', '||'):
|
||
first = values.pop()
|
||
if (first == 0) == (kind == '&&'):
|
||
values.append(int(kind == '||')); pc = arg
|
||
continue
|
||
if kind == 'literal': value = arg
|
||
elif kind == 'symbol':
|
||
if arg not in bindings: raise NameError(arg)
|
||
value = bindings[arg]
|
||
elif kind == 'boolean': value = int(values.pop() != 0)
|
||
elif kind == 'unary':
|
||
first = values.pop()
|
||
value = first if arg == '+' else -first if arg == '-' else int(first == 0)
|
||
else:
|
||
second, first = values.pop(), values.pop()
|
||
if arg in comparisons: value = int(comparisons[arg](first, second))
|
||
elif arg == '+': value = first + second
|
||
elif arg == '-': value = first - second
|
||
elif arg == '*': value = first * second
|
||
else:
|
||
if second == 0: raise ZeroDivisionError()
|
||
quotient = int(Fraction(first, second))
|
||
value = quotient if arg == '/' else first - quotient*second
|
||
if not LOW <= value <= HIGH: raise OverflowError()
|
||
values.append(int(value))
|
||
if len(values) != 1: raise RuntimeError('oracle stack')
|
||
return values[0]
|
||
|
||
|
||
PRECEDENCE = {'||': 1, '&&': 2, '==': 3, '!=': 3, '<': 4, '<=': 4,
|
||
'>': 4, '>=': 4, '+': 5, '-': 5, '*': 6, '/': 6, '%': 6}
|
||
def rendering(node, rng):
|
||
kind = node[0]
|
||
def wrap(tokens): return ['('] + tokens + [')']
|
||
if kind == 'literal': words, rank = [str(node[1])], 9
|
||
elif kind == 'symbol': words, rank = [node[1]], 9
|
||
elif kind == 'prefix':
|
||
child, child_rank = rendering(node[2], rng)
|
||
words, rank = [node[1]] + (wrap(child) if child_rank < 7 else child), 7
|
||
elif kind == 'binary':
|
||
left, lr = rendering(node[2], rng); right, rr = rendering(node[3], rng)
|
||
rank = PRECEDENCE[node[1]]
|
||
words = (wrap(left) if lr < rank else left) + [node[1]] + (wrap(right) if rr <= rank else right)
|
||
else:
|
||
condition, cr = rendering(node[1], rng)
|
||
yes, _ = rendering(node[2], rng); no, _ = rendering(node[3], rng)
|
||
words, rank = (wrap(condition) if cr == 0 else condition) + ['?'] + yes + [':'] + no, 0
|
||
if rng.randrange(7) == 0: return wrap(words), 9
|
||
return words, rank
|
||
|
||
|
||
MISSING = object()
|
||
|
||
|
||
def cases():
|
||
requests, expected, labels = [], [], []
|
||
def add(text, bindings=MISSING, value=None, error=None, label='manual'):
|
||
requests.append([text, {} if bindings is MISSING else bindings])
|
||
expected.append(('error', error) if error else ('value', value)); labels.append(label)
|
||
def ok(text, value, bindings=MISSING, label='semantics'): add(text, bindings, value, label=label)
|
||
def err(text, kind, bindings=MISSING, label='errors'): add(text, bindings, error=kind, label=label)
|
||
basic = [('0', 0), ('0000012', 12), (' \t\r\n + 000 \n', 0), ('1+2*3', 7),
|
||
('18/3/2', 3), ('10-3-2', 5), ('2<1<1', 1), ('2==1==0', 1),
|
||
('2<3==1', 1), ('1==2<3', 1), ('1||0&&0', 1), ('0||2&&3', 1),
|
||
('!0+!!3', 2), ('-+-2', 2), ('-(2+3)*2', -10), ('7/-3', -2),
|
||
('-7/3', -2), ('-7/-3', 2), ('-7%3', -1), ('7%-3', 1),
|
||
('-7%-3', -1), ('0/-2', 0), ('0%-2', 0), ('1?2:0?3:4', 2),
|
||
('0?2:0?3:4', 4), ('1?0?3:4:5', 4), ('(0?2:3)+1', 4),
|
||
('0?1:2?3:4', 3), ('1+2?5:6', 5), ('0?1:2+3', 5),
|
||
('0&&missing', 0), ('2||missing', 1), ('0?missing:7', 7),
|
||
('1?7:1/0', 7), ('0&&2147483648', 0), ('1||2147483648', 1),
|
||
('0?-2147483648:8', 8), ('1?8:-2147483648', 8),
|
||
('2147483647', HIGH), ('-2147483647-1', LOW),
|
||
('x%-1', 0), ('x+0', LOW), ('y*1', HIGH), ('x<y', 1),
|
||
('x==x', 1), ('x!=x', 0), ('y>=y', 1), ('x<=x', 1),
|
||
('x>y', 0), ('3&&-2', 1), ('-2||0', 1), ('0||-2', 1),
|
||
('2&&0', 0), ('0||0', 0), ('!x', 0), ('+x', LOW)]
|
||
for text, value in basic: ok(text, value, {'x': LOW, 'y': HIGH})
|
||
for a in range(-4, 5):
|
||
for b in range(-4, 5):
|
||
if b:
|
||
ok('a/b', int(Fraction(a, b)), {'a': a, 'b': b}, 'division-small-signs')
|
||
ok('a%b', a-int(Fraction(a, b))*b, {'a': a, 'b': b}, 'remainder-small-signs')
|
||
for text in ['2147483648', '-2147483648', '2147483647+1', '(2147483647+1)-1',
|
||
'x/-1', '-x', 'y*2', '0-(2147483647+2)', '0?0:2147483648']:
|
||
err(text, 'OverflowError', {'x': LOW, 'y': HIGH}, 'overflow-visited-intermediate')
|
||
for text in ['1/0', '1%0', '0/0', '0%0', '1&&1/0', '0||1%0']:
|
||
err(text, 'ZeroDivisionError', label='zero-divisor')
|
||
for text, kind in [('missing+1/0','NameError'), ('1/0+missing','ZeroDivisionError'),
|
||
('2147483648+missing','OverflowError'), ('missing+2147483648','NameError'),
|
||
('(1/0)/(missing)','ZeroDivisionError'), ('missing?1/0:0','NameError'),
|
||
('1?missing:1/0','NameError'), ('0?1/0:missing','NameError')]:
|
||
err(text, kind, label='runtime-first-error')
|
||
bad_syntax = ['', ' ', '1 2', 'a b', '2a', '1+', '+', '()', '(1', '1)',
|
||
'1**2', '1//2', '1&2', '1|2', '1=2', '1<>2', '1 & & 2',
|
||
'1 | | 2', '1 ! = 2', '1< =2', '1?2', '1?:2', '?1:2',
|
||
'1?2:', '1:2', '1?2:3:4', 'f(1)', '0x10', '1.0',
|
||
'1_0', 'é', '1', 'abc'+'x'*30, '1\v+2', '1\f+2', '1\0',
|
||
'0&&(', '1||1+', '1?7:(1+)', 'missing+(', '1/0+(',
|
||
'2147483648+(', '0?7:1?2:']
|
||
for text in bad_syntax: err(text, 'ValueError', label='whole-lexical-syntax-before-runtime')
|
||
for text in [None, True, 3, 1.0, b'1', [], {}, (), '0'*4097]:
|
||
err(text, 'ValueError', label='expression-domain')
|
||
for bindings in [None, [], (), True, {'a': True}, {'a': 1.0}, {'a': LOW-1},
|
||
{'a': HIGH+1}, {'a': 10**500}, {1: 0}, {'': 0}, {'1x': 0},
|
||
{'é': 0}, {'a'*33: 0}, {'a-b': 0}, {'a.b': 0},
|
||
{'k'+str(i): i for i in range(65)}]:
|
||
err('0', 'ValueError', bindings, 'binding-domain-unused')
|
||
err('missing', 'ValueError', {'unused': True}, 'input-before-runtime')
|
||
err('1/0', 'ValueError', {'unused': HIGH+1}, 'input-before-runtime')
|
||
ok('A_09+a_09', 5, {'A_09': 2, 'a_09': 3, 'unused': LOW}, 'names-case-unused')
|
||
ok('a'*32, 1, {'a'*32: 1}, 'name-length-endpoint')
|
||
ok('k63', 63, {'k'+str(i): i for i in range(64)}, 'binding-count-endpoint')
|
||
ok(OrdinaryString('x+1'), 3, { 'x': 2 }, 'expression-subclass')
|
||
ok('x', 3, OrdinaryDict(x=3), 'bindings-subclass')
|
||
ok('x', 4, {OrdinaryString('x'): 4}, 'binding-name-subclass')
|
||
ok('x', 5, {'x': OrdinaryInt(5)}, 'binding-int-subclass')
|
||
ok(OrdinaryString('x<y?x:y'), LOW,
|
||
OrdinaryDict({OrdinaryString('x'): OrdinaryInt(LOW), 'y': OrdinaryInt(HIGH)}), 'all-subclasses-endpoints')
|
||
err('0', 'ValueError', {'x': OrdinaryInt(HIGH+1)}, 'subclass-outside-range')
|
||
for text, value in [('+'*4095+'0', 0), ('!'*4095+'0', 1),
|
||
('('*2047+'0'+')'*2047, 0), ('0'*4096, 0),
|
||
('1+'*2047+'1 ', 2048), ('1?'*1023+'0'+':0'*1023, 0)]:
|
||
ok(text, value, label='maximum-length-deep-grammar')
|
||
err('9'*4096, 'OverflowError', label='maximum-literal-runtime')
|
||
ok('0&&'+'9'*4093, 0, label='maximum-literal-skipped')
|
||
err('('*2047+'0'+')'*2046, 'ValueError', label='deep-unclosed-syntax')
|
||
for value in [2, -3, 0, LOW, HIGH]: ok('x', value, {'x': value}, 'sequential-state')
|
||
rng = random.Random(SEED)
|
||
binary = list(PRECEDENCE)
|
||
def generate(depth):
|
||
if depth == 0 or rng.randrange(5) == 0:
|
||
if rng.randrange(3) == 0: return ('symbol', rng.choice(['a', 'b', 'c', 'missing']))
|
||
value = rng.choice([0, 0, 1, 2, 3, HIGH, HIGH+1]) if rng.randrange(12) == 0 else rng.randrange(5)
|
||
return ('literal', value)
|
||
form = rng.randrange(5)
|
||
if form == 0: return ('prefix', rng.choice(['+', '-', '!']), generate(depth-1))
|
||
if form == 1: return ('select', generate(depth-1), generate(depth-1), generate(depth-1))
|
||
return ('binary', rng.choice(binary), generate(depth-1), generate(depth-1))
|
||
for _ in range(RANDOM_CASES):
|
||
tree = generate(rng.randrange(1, 6))
|
||
bindings = {name: rng.choice([-3, -2, -1, 0, 0, 1, 2, 3]) for name in ['a', 'b', 'c']}
|
||
if rng.randrange(4) == 0: del bindings[rng.choice(['a', 'b', 'c'])]
|
||
if rng.randrange(12) == 0: bindings[rng.choice(['a', 'b', 'c'])] = rng.choice([LOW, HIGH])
|
||
words, _ = rendering(tree, rng)
|
||
text = ''.join(word + rng.choice(['', '', ' ', '\t', '\r\n']) for word in words)
|
||
try: value = oracle(tree, bindings)
|
||
except (NameError, OverflowError, ZeroDivisionError) as error: err(text, type(error).__name__, bindings, 'random-ast')
|
||
else: ok(text, value, bindings, 'random-ast')
|
||
return requests, expected, labels
|
||
|
||
|
||
def require(condition, label):
|
||
if not condition: raise AssertionError(label)
|
||
|
||
|
||
def main():
|
||
requests, expected, labels = cases()
|
||
payload = [pack(args) for args in requests]
|
||
scratch = Path(tempfile.mkdtemp(prefix='lazy-integer-check-', dir=Path.cwd()))
|
||
source = Path(sys.argv[1])/'lazy_integer.py'
|
||
require(source.is_file() and not source.is_symlink(), 'missing candidate')
|
||
shutil.copyfile(source, scratch/'lazy_integer.py')
|
||
(scratch/'runner.py').write_text(RUNNER)
|
||
nonce = secrets.token_hex(24)
|
||
# All trusted reads/writes of the candidate-writable tree end before spawn.
|
||
result = subprocess.run([sys.executable, '-I', '-B', str(scratch/'runner.py'), str(scratch), nonce],
|
||
input=json.dumps(payload, ensure_ascii=True), text=True,
|
||
capture_output=True, cwd=scratch, timeout=40)
|
||
require(result.returncode == 0, 'candidate execution failed')
|
||
prefix = 'RESULT:' + nonce + ':'
|
||
rows = [line[len(prefix):] for line in result.stdout.splitlines() if line.startswith(prefix)]
|
||
require(len(rows) == 1, 'observation marker count')
|
||
observed = json.loads(rows[0])
|
||
require(isinstance(observed, list) and len(observed) == len(expected), 'observation count')
|
||
for i, (row, want, label, raw) in enumerate(zip(observed, expected, labels, payload)):
|
||
context = str(i) + ':' + label
|
||
require(isinstance(row, dict) and row.get('before') == raw and row.get('after') == raw, context + ':input changed')
|
||
kind, value = want
|
||
if kind == 'error':
|
||
require(row.get('families', {}).get(value) is True and 'builtins.'+value in row.get('mro', []), context + ':exception '+value)
|
||
else:
|
||
actual = row.get('value', [])
|
||
require(len(actual) == 3 and actual[0] in ('int', 'int_sub') and type(actual[2]) is int and actual[2] == value, context + ':integer result')
|
||
print('validated cases=' + str(len(expected)) + ' random=' + str(RANDOM_CASES) + ' seed=' + str(SEED))
|
||
print('PASS')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
try: main()
|
||
except Exception as error:
|
||
print(('FAIL: ' + type(error).__name__ + ': ' + str(error)).replace('\n', ' ')[:400])
|
||
sys.exit(1)
|