amos 913d3c367f Add the usable task catalog and admit batch 6 tasks
All 161 task packages, including the 21 batch 6 C++17 tasks that passed
gates 1/2, are now in Git. rollout.catalog builds docs/task-catalog.{json,md}
from the task packages, a fresh trace5 calibration (128/128 PASS), every
verified controller receipt on the current task digest, documented holds
and digest-bound difficulty reviews. 123 tasks are usable (6 HARD),
5 are on hold and 33 still need a subprocess-only grader.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-18 17:43:15 +08:00

491 lines
20 KiB
Python

"""Private value oracle: translated bytes, sorted dictionaries, greedy glob.
The candidate sees only public raw observers and the generated input stream.
"""
import base64
import copy
import itertools
import json
from pathlib import Path
import random
import subprocess
import sys
import tempfile
CANDIDATES = ['sysctl_key.cpp', 'sysctl_lex.cpp', 'sysctl_compile.cpp', 'sysctl_plan.cpp']
LETTERS = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.-'
SPACE = ' \t\r\v\f'
EMPTY_CONFIGURATION = [[], [], []]
BASIS = {
'C01': [1, 4, 5, 8, 30],
'C02': [1, 2, 3, 4, 5, 30],
'C03': [5, 7, 9, 10, 11, 12, 13, 30],
'C04': [5, 7, 9, 10, 12, 18, 27],
'C05': [1, 4, 5, 14, 15, 30, 31],
'C06': [4, 5, 6, 7, 16, 17, 29, 30],
'C07': [4, 5, 6, 7, 9, 12, 16, 17, 18, 19, 20, 28, 29, 30],
'C08': [5, 8, 11, 19, 20, 22, 23, 24, 26, 27, 28],
'C09': [5, 8, 19, 20, 21, 22, 23, 26, 27, 28],
'C10': [4, 5, 8, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
'C11': [5, 8, 18, 20, 21, 22, 23, 26, 27, 28, 31, 32],
'C12': [5, 6, 7, 8, 12, 16, 17, 18, 21, 27, 28, 32],
'C13': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30],
'C14': [4, 5, 8, 16, 18, 19, 20, 25, 26, 27, 28, 30],
'C15': [33, 34, 35],
}
for _label in BASIS:
if _label != 'C15':
BASIS[_label] = sorted(set(BASIS[_label] + [4, 30]))
def key_ok(text, pattern=False):
alphabet = LETTERS + ('*?' if pattern else '')
return (1 <= len(text) <= 256 and
all(part and part not in ('.', '..') and
all(byte in alphabet for byte in part) for part in text.split('/')))
def canonical(raw, pattern):
first = next((byte for byte in raw if byte in './'), None)
value = raw.translate(str.maketrans('./', '/.')) if first == '.' else raw
return value if key_ok(value, pattern) else None
def matches(pattern, key):
# Last-star greedy recognition, independent of the public template's DP.
at = end = 0
star = checkpoint = -1
while end < len(key):
if at < len(pattern) and (pattern[at] == '?' or pattern[at] == key[end]):
at += 1
end += 1
elif at < len(pattern) and pattern[at] == '*':
star, checkpoint = at, end
at += 1
elif star >= 0:
checkpoint += 1
end, at = checkpoint, star + 1
else:
return False
return all(token == '*' for token in pattern[at:])
def text_ok(text):
return (len(text) <= 65536 and
all(byte in '\t\n\r\v\f' or 32 <= ord(byte) <= 126 for byte in text))
def parse(text):
if not text_ok(text):
return None
rules = []
for row, original in enumerate(text.split('\n')):
line = original.split('#', 1)[0].strip(SPACE)
if not line:
continue
minus = line.startswith('-')
if minus:
line = line[1:]
pair = line.split('=', 1)
if len(pair) == 1 and not minus:
return None
key = canonical(pair[0].strip(SPACE), True)
if key is None:
return None
pattern = '*' in key or '?' in key
if len(pair) == 2:
value = pair[1].strip(SPACE)
if not 1 <= len(value) <= 4096 or any(
byte != '\t' and not 32 <= ord(byte) <= 126 for byte in value):
return None
rules.append([row, 0, key, value, minus, pattern])
else:
if pattern:
return None
rules.append([row, 1, key, '', False, False])
if len(rules) > 512:
return None
return rules
def select(layers):
if len(layers) > 8 or sum(map(len, layers)) > 128:
return None
size = 0
for layer in layers:
if len(layer) > 64 or len({name for name, _ in layer}) != len(layer):
return None
for name, text in layer:
if not (1 <= len(name) <= 128 and name.endswith('.conf') and
all(byte in LETTERS for byte in name)):
return None
if text is not None:
if not text_ok(text):
return None
size += len(text)
if size > 262144:
return None
chosen = {}
for level, layer in enumerate(layers):
for name, text in layer:
chosen.setdefault(name, [level, name, text])
return [chosen[name] for name in sorted(chosen)]
def compile_layers(layers):
chosen = select(layers)
if chosen is None:
return None
located = []
for level, name, text in chosen:
if text is None:
continue
rules = parse(text)
if rules is None or len(located) + len(rules) > 512:
return None
located.extend([level, name, rule] for rule in rules)
excluded = sorted({rule[2] for _, _, rule in located if not rule[5]})
return [chosen, located, excluded]
def plan(configuration, keys):
if (len(keys) > 256 or len(set(keys)) != len(keys) or
sum(map(len, keys)) > 32768 or not all(key_ok(key) for key in keys)):
return None
rules, excluded = configuration[1], set(configuration[2])
work = sum((len(rule[2]) + 1) * (len(key) + 1)
for _, _, rule in rules if rule[1] == 0 and rule[5]
for key in keys)
if work > 16777216:
return None
values = {}
available = set(keys)
for level, name, rule in rules:
row, kind, pattern, value, ignore, is_pattern = rule
if kind == 1:
continue
if not is_pattern:
if pattern not in available:
if ignore:
continue
return None
targets = [pattern]
else:
targets = [key for key in keys if key not in excluded and matches(pattern, key)]
for key in targets:
values[key] = [key, value, ignore, level, name, row]
return [values[key] for key in sorted(values)]
def hx(text):
return text.encode('latin1').hex() or '-'
class Matrix:
def __init__(self):
self.commands, self.expected, self.labels = [], [], []
self.layers, self.configuration, self.last_plan = [], None, []
self.held_configuration, self.held_plan = None, []
self.witnesses = {}
def add(self, command, value, label):
self.commands.append(command)
self.expected.append(copy.deepcopy(value))
self.labels.append(label)
return len(self.expected) - 1
def k(self, text, label='C01'):
return self.add('K ' + hx(text), ['K', key_ok(text), True], label)
def n(self, text, pattern=False, label='C02'):
value = canonical(text, pattern)
return self.add('N %d %s' % (pattern, hx(text)),
['N', int(value is None), value or '', True], label)
def m(self, pattern, key, label='C05'):
valid = key_ok(pattern, True) and key_ok(key)
return self.add('M %s %s' % (hx(pattern), hx(key)),
['M', int(not valid), valid and matches(pattern, key), True], label)
def l(self, text, label='C03'):
value = parse(text)
return self.add('L ' + hx(text), ['L', int(value is None), value or [], True], label)
def c(self, layers, label='C07'):
self.layers = copy.deepcopy(layers)
command = ['C ' + str(len(layers))]
for layer in layers:
command.append('Y ' + str(len(layer)))
command.extend('F %s %d %s' % (hx(name), text is not None, hx(text or ''))
for name, text in layer)
self.configuration = compile_layers(layers)
return self.add('\n'.join(command), ['C', int(self.configuration is None),
self.configuration or EMPTY_CONFIGURATION, True], label)
def s(self, label='C06'):
value = select(self.layers)
return self.add('S', ['S', int(value is None), value or [], True], label)
def p(self, keys, label='C10', mode='P'):
configuration = (compile_layers(self.layers) if mode == 'P' else
self.configuration if mode == 'R' else self.held_configuration)
assert mode == 'P' or configuration is not None
value = None if configuration is None else plan(configuration, keys)
self.last_plan = value or []
return self.add('%s %d%s' % (mode, len(keys), ''.join(' ' + hx(key) for key in keys)),
[mode, int(value is None), value or [], True], label)
def h(self, label='C14'):
assert self.configuration is not None
self.held_configuration = copy.deepcopy(self.configuration)
self.held_plan = copy.deepcopy(self.last_plan)
return self.add('H', ['H', self.held_configuration, self.held_plan], label)
def q(self, label='C14'):
assert self.held_configuration is not None
return self.add('Q', ['Q', self.held_configuration, self.held_plan], label)
def cases():
c = Matrix()
c.s(); c.p([], 'C09'); c.p(['a/x'], 'C09')
c.witnesses['direction'] = c.n('net.ipv4.conf.eth0/100.forwarding')
c.c([[['10-base.conf', 'net/*=shared']], [['90-local.conf', '-net/private']]], 'C08')
c.witnesses['global'] = c.p(['net/private', 'net/public'], 'C08')
c.c([[['90-last.conf', 'a/x=last']], [['10-first.conf', 'a/x=first']]], 'C10')
c.witnesses['order'] = c.p(['a/x'], 'C10')
c.c([[['one.conf', 'a/x=first\n-a/x=last']]], 'C10')
c.witnesses['record'] = c.p(['a/x'], 'C10')
c.s(); c.p(['a/x'], mode='R'); c.h()
for text in ['', '/', '/a', 'a/', 'a//b', '.', '..', 'a/.', 'a/..', 'a/b',
'-a/x', 'A/B.c-d_e', 'a\\b', 'a[0]', 'a:*', 'a b', 'a\n',
'a' * 255, 'a' * 256, 'a' * 257]:
c.k(text); c.n(text); c.n(text, True)
for byte in range(256):
text = 'a/' + chr(byte) + 'b'
c.k(text); c.n(text); c.n(text, True)
c.l('a/x=ok #' + chr(byte), 'C04')
for text in ['a.b/c.d', 'a/b.c.d', 'a.b/c/d', 'a/b.c/d', 'a.*.b/c',
'-a.b/c', 'a..b', 'a...b', 'a/./b', 'a/../b']:
c.n(text); c.n(text, True)
patterns = ['', '*', '?', 'a/*', 'a/?', 'a/*/*', 'a/?/x', 'a/**b',
'a/*?*?*b', 'a/*/z', 'a/*.*', 'a/[x]', 'a/\\*', '/a', 'a//b']
keys = ['', 'a', 'b', 'a/x', 'a/y/z', 'a//b', 'a/.', 'a/..', 'a/bb', 'a/.b']
for pattern in patterns:
for key in keys:
c.m(pattern, key)
for width in range(1, 5):
for suffix in map(''.join, itertools.product('ab*?', repeat=width)):
for key in ['a/a', 'a/b', 'a/ab', 'a/ba', 'a/aaaa', 'a/a/b']:
c.m('a/' + suffix, key)
for size in [1, 16, 64, 127]:
c.m('a/' + '*a' * size + '*b', 'a/' + 'a' * 253)
texts = ['', '\n', '\r', '#comment\n\n', 'a/x = left=right #note',
'\v a/x\t=\t v \f\r', 'a/x=one\r\na/y=two\n',
'a/x=one\ra/y=two', 'a/x=one\va/y=two',
'a/x="left#right"', 'a/x=back\\#cut', 'a/x=one\\\na/y=two',
'a/x= #empty', 'a/x', '-', '- ', '-a/*', '-a/?',
'- a/x = v', '--a/x=v', '-a/x', 'a/*=v', '-a/*=v',
'a/x=v\nnot-an-assignment', 'a/x=v\n-a/*',
'\t#a\n\n a/x=1\n\n-a/y\n', 'a/x=tab\there',
'a/x=inner\rvalue', 'a/x=inner\vvalue', 'a/x=inner\fvalue',
'a/x=\t', 'a/x==' , 'a.x/y=z', 'a/x=' + 'v' * 4096,
'a/x=' + 'v' * 4097]
for text in texts:
c.l(text)
for text in ['#' + 'x' * 65535, '#' + 'x' * 65536,
'\n' * 65536, '\n' * 65537,
'\n'.join(['a/x=v'] * 512), '\n'.join(['a/x=v'] * 513)]:
c.l(text, 'C04')
layer_cases = [[], [[]], [[]] * 8, [[]] * 9,
[[['a.conf', None]], [['a.conf', 'invalid syntax']]],
[[['a.conf', 'a/x=high']], [['a.conf', 'invalid syntax']]],
[[['a.conf', 'a/x=high']], [['a.conf', '#\x00']]],
[[['a.conf', 'a/x=high']], [['a.conf', '#' + 'x' * 65536]]],
[[['z.conf', 'a/x=z'], ['a.conf', 'a/x=a']]],
[[['a.conf', 'a/x=1'], ['a.conf', None]]],
[[['a.conf', 'bad selected']]],
[[['a.CONF', None]]], [[['a/conf', None]]], [[['.conf', None]]],
[[['x' * 123 + '.conf', None]]], [[['x' * 124 + '.conf', None]]],
[[['a.conf', 'a/x=v\n' * 256], ['b.conf', 'a/x=v\n' * 256]]],
[[['a.conf', 'a/x=v\n' * 256], ['b.conf', 'a/x=v\n' * 257]]],
]
for layers in layer_cases:
c.c(layers); c.s(); c.p([], 'C09')
if c.configuration is not None:
c.p(['a/x'], 'C09', 'R')
for byte in range(256):
c.c([[['a' + chr(byte) + '.conf', None]]], 'C06'); c.s('C06')
for count in [64, 65, 128, 129]:
files = [[str(index) + '.conf', None] for index in range(count)]
layers = [files] if count <= 65 else [files[:64], files[64:128], files[128:]]
c.c(layers, 'C12'); c.s('C12'); c.p([], 'C12')
for extra in [0, 1]:
texts = [['%d.conf' % i, '#' + 'x' * 65535] for i in range(4)]
if extra: texts.append(['extra.conf', '#'])
c.c([texts], 'C12'); c.s('C12'); c.p([], 'C12')
scenarios = [
('a/x=exact\na/*=glob', ['a/y', 'a/x']),
('-a/x=exact\na/*=glob', ['a/y', 'a/x']),
('a/*=glob\n-a/x', ['a/y', 'a/x']),
('-a/x\na/x=exact', ['a/x']),
('a/*=glob\n-a/missing=v', ['a/x', 'a/y']),
('a/*=glob\n-a/missing', ['a/x', 'a/y']),
('-a/*=first\na/*=last', ['a/y', 'a/x']),
('a/*=first\n-a/*=last', ['a/y', 'a/x']),
('a/*=first\n-a/missing=v\na/z=last', ['a/x']),
('a/*=first\n-a/missing=v', []),
('a/*=first\na/missing=v', []),
('-a/missing=v\n-a/unknown', []),
('a/*=first\na/x=last\na/*=after', ['a/x', 'a/y']),
('-a/x=v\na/x=last', ['a/x']),
('a/x=v\n-a/x=last', ['a/x']),
('a/x=v\n-a/x', ['a/x']),
('a/x=v\n-a/x\n-a/x', ['a/x']),
('-a/x=v\na/*=unused\na/x=last', ['a/x']),
]
for text, keys in scenarios:
c.c([[['20.conf', text]]], 'C10')
c.p(keys, 'C10'); c.p(keys, 'C10', 'R')
c.c([[['empty.conf', '#empty']]], 'C09')
for keys in [[], ['a/x'], ['a/x', 'a/x'], ['bad/'], ['a/x', 'bad/'],
['a' * 256], ['a' * 257], ['a*'], ['a.x']]:
c.p(keys, 'C09'); c.p(keys, 'C09', 'R')
# Supplied known keys stay literal; only configuration KEY uses direction.
literal_key_scenarios = [
('a/x=v', ['a.x']),
('*=v', ['a.x']),
('a/b.c=v', ['a/b.c']),
('-a/x=exact\n*=glob', ['a.x']),
('*=all', ['a.b/c', 'a/x.y', 'a.x']),
]
for text, keys in literal_key_scenarios:
c.c([[['literal.conf', text]]], 'C07')
c.p(keys, 'C09'); c.p(keys, 'C09', 'R'); c.h('C14')
c.c([[['other.conf', 'unrelated/x=other']]], 'C07')
c.p(keys, 'C14', 'J'); c.q('C14')
c.c([[['empty.conf', '#empty']]], 'C09')
for count in [256, 257]:
keys = ['k' + str(i) for i in range(count)]
c.p(keys, 'C12'); c.p(keys, 'C12', 'R')
for total in [32768, 32769]:
keys = ['k' + str(i).zfill(3) + 'x' * 124 for i in range(256)]
if total == 32769: keys[0] += 'x'
c.p(keys, 'C12'); c.p(keys, 'C12', 'R')
# Exact W = 512 * 256 * (15+1) * (7+1) = 2**24.
pattern = '*' * 15
keys = ['k' + str(i).zfill(6) for i in range(256)]
c.c([[['budget.conf', '\n'.join([pattern + '=v'] * 512)]]], 'C11')
c.p(keys, 'C11'); c.p(keys, 'C11', 'R')
too_long = keys[:]; too_long[0] += 'x'
c.p(too_long, 'C11'); c.p(too_long, 'C11', 'R')
# All keys excluded and no resulting entries: W still includes every pair.
glob = '*' * 255 + '=v'
c.c([[['a.conf', '\n'.join([glob] * 130)],
['b.conf', '\n'.join([glob] * 127)],
['c.conf', '\n'.join('-' + key for key in keys[:255])]]], 'C11')
c.p(keys[:255], 'C11'); c.p(keys[:255], 'C11', 'R')
c.c([[['one.conf', '*=v\na/x=exact']]], 'C14')
c.p(['z/q', 'a/x'], 'C14'); c.h()
c.c([[['other.conf', 'q/x=next']]], 'C14'); c.p(['q/x'], 'C14')
c.q(); c.p(['a/x', 'z/q'], 'C14', 'J'); c.q()
c.p(['bad/'], 'C14', 'J'); c.q()
c.c([[['bad.conf', 'a/x=ok\nbad']]], 'C14'); c.p([], 'C14'); c.q()
c.p(['a/x'], 'C14', 'J'); c.q()
rng = random.Random(20261017020)
raw_keys = ['a/x', 'a/y', 'a/x/y', 'b/x', 'a.x/y', '-a/x']
patterns = ['a/*', 'a/?', '*', 'a/*/y', 'b/*', 'a.*', '?/*']
for index in range(180):
layers = [[], [], []]
for layer in range(rng.randrange(1, 4)):
for file_index in range(rng.randrange(0, 5)):
name = '%02d.conf' % file_index
if rng.randrange(9) == 0:
text = None
else:
lines = []
for row in range(rng.randrange(0, 7)):
key = rng.choice(raw_keys + patterns)
minus = '-' if rng.randrange(2) else ''
if rng.randrange(5) == 0:
lines.append('-' + rng.choice(raw_keys))
else:
lines.append('%s%s=%s #row%d' %
(minus, key, rng.choice(['1', 'x=y', '\tvalue']), row))
text = '\n'.join(lines)
layers[layer].append([name, text])
rng.shuffle(layers[layer])
c.c(layers, 'C13'); c.s('C13')
keys = rng.sample(['a/x', 'a/y', 'a/x/y', 'b/x', '-a/x', 'z/unused'], rng.randrange(7))
c.p(keys, 'C13')
if c.configuration is not None:
c.p(keys, 'C13', 'R')
return c
def equal(left, right):
if type(left) is not type(right):
return False
if isinstance(left, list):
return len(left) == len(right) and all(equal(a, b) for a, b in zip(left, right))
return left == right
def main():
matrix = cases()
context = json.loads((Path(__file__).resolve().parent / 'context.json').read_text())
scratch = Path(tempfile.mkdtemp(prefix='embedded-hard-sysctl-', dir='/tmp'))
for name, text in context.items():
(scratch / name).write_text(text)
for name in CANDIDATES:
(scratch / name).write_bytes((Path(sys.argv[1]) / name).read_bytes())
build = subprocess.run(['g++', '-std=c++17', '-O2', '-Wall', '-Wextra', '-pedantic',
'sysctl_observe.cpp', *CANDIDATES, '-o', 'sysctl_observe'],
cwd=scratch, text=True, encoding='utf-8', errors='replace',
capture_output=True, timeout=30)
if build.returncode:
print('FAILED_CHECKS ["C15"] build failed: ' + build.stderr[-1200:].replace('\n', ' '))
return 1
# shortcut: after execution inspect captured pipes only; trusted owner or
# container destruction removes scratch separately, without following it.
timed_out = False
try:
result = subprocess.run([str(scratch / 'sysctl_observe')], cwd=scratch,
input='\n'.join(matrix.commands) + '\n', text=True,
encoding='utf-8', errors='replace', capture_output=True, timeout=15)
output, exit_code = result.stdout, result.returncode
except subprocess.TimeoutExpired as error:
timed_out = True
output = error.stdout or b''
if isinstance(output, bytes): output = output.decode('utf-8', 'replace')
exit_code = -9
actual = output.splitlines()
failures = set(['C15'] if timed_out or exit_code else [])
count, indexes, examples = max(len(actual), len(matrix.expected)), [], []
for index in range(count):
label = matrix.labels[index] if index < len(matrix.labels) else 'C15'
wanted = matrix.expected[index] if index < len(matrix.expected) else '<end>'
try: observed = json.loads(actual[index]) if index < len(actual) else '<missing>'
except ValueError: observed = actual[index]
if index >= len(actual) or index >= len(matrix.expected) or not equal(observed, wanted):
indexes.append(index); failures.add(label)
if len(examples) < 8:
examples.append('%s observation=%d got=%s expected=%s' %
(label, index, repr(observed)[:220], repr(wanted)[:220]))
if failures:
print('FAILED_CHECKS ' + json.dumps(sorted(failures)))
for example in examples: print(example)
bitmap = bytearray((count + 7) // 8)
for index in indexes: bitmap[index // 8] |= 1 << (index % 8)
diagnostic = {'observations': count, 'bits_lsb0': base64.b64encode(bitmap).decode(),
'failed_checks': sorted(failures), 'candidate_exit': exit_code, 'timeout': timed_out}
print('MISMATCH_BITMAP ' + json.dumps(diagnostic, separators=(',', ':')), file=sys.stderr)
return 1
print('PASS')
return 0
if __name__ == '__main__':
raise SystemExit(main())