100 tasks from the HWE arm-linux code-gen pool (outside the lk/man/pth range given to batch 22), each with at least one independent contract-edge defect and plain public docs. Claude reviewed 100/100 USABLE against current digests; no ID or source overlap with main or batch 22. Difficulty is unmeasured. progress.md records the wrap-up: all Codex sessions and Qwen schedulers stopped for the user's rollout, Qwen results so far (57 HARD of 373), and what is left on each batch branch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
473 lines
18 KiB
Python
473 lines
18 KiB
Python
"""Trusted reference model; only the raw observer runs with candidate code."""
|
|
import random
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
FILES = ('deferred.hpp', 'engine_internal.hpp', 'registry.cpp', 'dispatch.cpp', 'inspect.cpp')
|
|
# Public raw observer, kept byte-identical to template/observer.cpp.
|
|
OBSERVER = '#include "deferred.hpp"\n#include <iostream>\n#include <string>\nusing namespace deferred;\nint main()\n{\n Engine engines[2];\n Engine* engine = &engines[0];\n std::string op;\n int id, priority, quota, index;\n Number value, marker;\n while (std::cin >> op) {\n if (op == "U") {\n std::cin >> index;\n engine = &engines[index];\n std::cout << index << \'\\n\';\n } else if (op == "I") {\n std::cin >> id >> priority >> quota >> value;\n std::cout << engine->install(id, priority, quota, value) << \'\\n\';\n } else if (op == "P") {\n std::cin >> id >> value;\n std::cout << engine->post(id, value) << \'\\n\';\n } else if (op == "D") {\n std::cin >> id;\n std::cout << engine->disable(id) << \'\\n\';\n } else if (op == "N") {\n std::cin >> id;\n std::cout << engine->disable_nosync(id) << \'\\n\';\n } else if (op == "E") {\n std::cin >> id;\n std::cout << engine->enable(id) << \'\\n\';\n } else if (op == "K") {\n std::cin >> id;\n std::cout << engine->kill(id) << \'\\n\';\n } else if (op == "S") {\n const Action a = engine->step();\n std::cout << a.kind << \' \' << a.id << \' \' << a.generation << \' \' << a.count << \' \' << a.sum << \' \' << a.value << \'\\n\';\n } else if (op == "V") {\n std::cin >> id >> marker;\n View v{marker, marker, marker, marker, marker, marker, marker, marker, marker, marker, marker, marker, marker, marker, marker, marker};\n const int status = engine->view(id, v);\n std::cout << status << \' \' << v.life << \' \' << v.generation << \' \' << v.priority << \' \' << v.quota << \' \' << v.cookie << \' \' << v.depth << \' \' << v.pending_count << \' \' << v.pending_sum << \' \' << v.pending_ticket << \' \' << v.active_count << \' \' << v.active_sum << \' \' << v.active_ticket << \' \' << v.completed_batches << \' \' << v.delivered_count << \' \' << v.delivered_sum << \' \' << v.delivered_value << \'\\n\';\n } else if (op == "G") {\n const Summary s = engine->summary();\n std::cout << s.installs << \' \' << s.live << \' \' << s.queued << \' \' << s.running << \' \' << s.starts << \' \' << s.finishes << \' \' << s.kills << \' \' << s.errors << \' \' << s.next_ticket << \' \' << s.events << \'\\n\';\n } else if (op == "H") {\n std::cin >> index >> marker;\n Event e{marker, marker, marker, marker, marker, marker, marker};\n const int status = engine->event(index, e);\n std::cout << status << \' \' << e.kind << \' \' << e.id << \' \' << e.generation << \' \' << e.count << \' \' << e.sum << \' \' << e.value << \' \' << e.serial << \'\\n\';\n }\n }\n return 0;\n}\n'
|
|
|
|
|
|
class Reference:
|
|
def __init__(self):
|
|
self.slots = {}
|
|
self.generations = [0] * 16
|
|
self.requests = []
|
|
self.history = []
|
|
self.serial = 0
|
|
self.ticket = 1
|
|
self.counters = dict(installs=0, starts=0, finishes=0, kills=0, errors=0)
|
|
|
|
def emit(self, kind, slot, batch=()):
|
|
self.serial += 1
|
|
count = len(batch)
|
|
total = sum(r['value'] for r in batch)
|
|
event = [kind, slot['id'], slot['generation'], count, total,
|
|
total + slot['cookie'] * count, self.serial]
|
|
self.history.append(event)
|
|
self.history = self.history[-16:]
|
|
|
|
def group(self, ident, stage):
|
|
return [r for r in self.requests if r['id'] == ident and r['stage'] == stage]
|
|
|
|
def finish(self, slot):
|
|
batch = self.group(slot['id'], 'active')
|
|
total = sum(r['value'] for r in batch)
|
|
slot['batches'] += 1
|
|
slot['count'] += len(batch)
|
|
slot['sum'] += total
|
|
slot['value'] += total + slot['cookie'] * len(batch)
|
|
self.counters['finishes'] += 1
|
|
self.emit(3, slot, batch)
|
|
action = [11, slot['id'], slot['generation'], len(batch), total,
|
|
total + slot['cookie'] * len(batch)]
|
|
for request in batch:
|
|
request['stage'] = 'done'
|
|
return action
|
|
|
|
def failed(self, code):
|
|
self.counters['errors'] += 1
|
|
return [code]
|
|
|
|
def live_error(self, ident):
|
|
if not 0 <= ident < 16:
|
|
return 1
|
|
if ident not in self.slots:
|
|
return 4
|
|
if self.slots[ident]['life'] == 2:
|
|
return 5
|
|
return 0
|
|
|
|
def run(self, command):
|
|
op, *args = command
|
|
if op == 'G':
|
|
live = [s for s in self.slots.values() if s['life'] == 1]
|
|
return [self.counters['installs'], len(live),
|
|
sum(bool(self.group(s['id'], 'pending')) for s in live),
|
|
int(any(r['stage'] == 'active' for r in self.requests)),
|
|
self.counters['starts'], self.counters['finishes'],
|
|
self.counters['kills'], self.counters['errors'],
|
|
self.ticket, len(self.history)]
|
|
if op == 'H':
|
|
index, marker = args
|
|
if not 0 <= index < 16:
|
|
return [2] + [marker] * 7
|
|
if index >= len(self.history):
|
|
return [9] + [marker] * 7
|
|
return [0] + self.history[index]
|
|
if op == 'V':
|
|
ident, marker = args
|
|
if not 0 <= ident < 16:
|
|
return [1] + [marker] * 16
|
|
if ident not in self.slots:
|
|
return [4] + [marker] * 16
|
|
s = self.slots[ident]
|
|
pending = self.group(ident, 'pending')
|
|
active = self.group(ident, 'active')
|
|
return [0, s['life'], s['generation'], s['priority'], s['quota'],
|
|
s['cookie'], len(s['gates']), len(pending),
|
|
sum(r['value'] for r in pending), pending[0]['ticket'] if pending else 0,
|
|
len(active), sum(r['value'] for r in active), active[0]['ticket'] if active else 0,
|
|
s['batches'], s['count'], s['sum'], s['value']]
|
|
if op == 'S':
|
|
running = next((s for s in self.slots.values() if self.group(s['id'], 'active')), None)
|
|
if running is not None:
|
|
return self.finish(running)
|
|
ready = []
|
|
for s in self.slots.values():
|
|
work = self.group(s['id'], 'pending')
|
|
if s['life'] == 1 and not s['gates'] and work:
|
|
ready.append((s['priority'], work[0]['ticket'], s['id'], s, work))
|
|
if not ready:
|
|
return [12, -1, 0, 0, 0, 0]
|
|
_, _, ident, s, work = min(ready)
|
|
for request in work:
|
|
request['stage'] = 'active'
|
|
self.counters['starts'] += 1
|
|
self.emit(2, s, work)
|
|
total = sum(r['value'] for r in work)
|
|
return [10, ident, s['generation'], len(work), total, total + s['cookie'] * len(work)]
|
|
ident = args[0]
|
|
if not 0 <= ident < 16:
|
|
return self.failed(1)
|
|
if op == 'I':
|
|
_, priority, quota, cookie = args
|
|
if not (0 <= priority <= 3 and 1 <= quota <= 32 and -1000000 <= cookie <= 1000000):
|
|
return self.failed(2)
|
|
if ident in self.slots and self.slots[ident]['life'] == 1:
|
|
return self.failed(3)
|
|
self.generations[ident] += 1
|
|
s = dict(id=ident, life=1, generation=self.generations[ident], priority=priority,
|
|
quota=quota, cookie=cookie, gates=[], batches=0, count=0, sum=0, value=0)
|
|
self.slots[ident] = s
|
|
self.counters['installs'] += 1
|
|
self.emit(1, s)
|
|
return [0]
|
|
if op == 'P' and not -1000000 <= args[1] <= 1000000:
|
|
return self.failed(2)
|
|
error = self.live_error(ident)
|
|
if error:
|
|
return self.failed(error)
|
|
s = self.slots[ident]
|
|
if op == 'P':
|
|
pending = self.group(ident, 'pending')
|
|
if len(pending) >= s['quota']:
|
|
return self.failed(6)
|
|
if not pending:
|
|
ticket = self.ticket
|
|
self.ticket += 1
|
|
else:
|
|
ticket = pending[0]['ticket']
|
|
self.requests.append(dict(id=ident, value=args[1], stage='pending', ticket=ticket))
|
|
elif op in ('D', 'N'):
|
|
if op == 'D' and self.group(ident, 'active'):
|
|
return self.failed(8)
|
|
if len(s['gates']) == 4:
|
|
return self.failed(6)
|
|
s['gates'].append(object())
|
|
elif op == 'E':
|
|
if not s['gates']:
|
|
return self.failed(7)
|
|
s['gates'].pop()
|
|
elif op == 'K':
|
|
if self.group(ident, 'active'):
|
|
self.finish(s)
|
|
for request in self.group(ident, 'pending'):
|
|
request['stage'] = 'canceled'
|
|
s['life'] = 2
|
|
s['gates'].clear()
|
|
self.counters['kills'] += 1
|
|
self.emit(4, s)
|
|
return [0]
|
|
|
|
|
|
class RoutedReference:
|
|
def __init__(self):
|
|
self.instances = [Reference(), Reference()]
|
|
self.selected = 0
|
|
|
|
def run(self, command):
|
|
if command[0] == 'U':
|
|
assert len(command) == 2 and command[1] in (0, 1)
|
|
self.selected = command[1]
|
|
return [self.selected]
|
|
return self.instances[self.selected].run(command)
|
|
|
|
|
|
def parse(stream):
|
|
return [(line.split()[0], *map(int, line.split()[1:])) for line in stream.strip().splitlines()]
|
|
|
|
|
|
def observe(commands, ids=(0, 15)):
|
|
"""Query APIs independently after state changes, with nonzero sentinels."""
|
|
result = []
|
|
for command in commands:
|
|
result.append(command)
|
|
result.append(('G',))
|
|
for ident in ids:
|
|
result.append(('V', ident, -71))
|
|
result.append(('H', 0, 99))
|
|
result.append(('H', 15, -99))
|
|
return result
|
|
|
|
|
|
def cases():
|
|
cases = []
|
|
independent = parse('''
|
|
G
|
|
V 0 81
|
|
V 15 -81
|
|
H 0 18
|
|
I 0 3 8 10
|
|
I 15 1 1 -2
|
|
D 15
|
|
P 0 101
|
|
P 0 -1
|
|
S
|
|
N 0
|
|
P 0 13
|
|
P 15 7
|
|
P 0 1000001
|
|
''')
|
|
def inspect_both_slots():
|
|
return [('G',), ('V', 0, 91), ('V', 15, -91)] + [('H', index, 711) for index in range(16)]
|
|
independent += inspect_both_slots()
|
|
independent += parse('''
|
|
U 1
|
|
G
|
|
V 0 82
|
|
V 15 -82
|
|
H 0 28
|
|
H 15 -28
|
|
S
|
|
I 0 0 2 -7
|
|
I 15 3 32 1000000
|
|
E 0
|
|
P 0 9
|
|
S
|
|
P 0 10
|
|
N 0
|
|
''')
|
|
independent += inspect_both_slots()
|
|
independent += [('U', 0)] + inspect_both_slots()
|
|
independent += parse('''
|
|
U 1
|
|
K 0
|
|
I 0 2 1 17
|
|
P 0 50
|
|
D 0
|
|
''')
|
|
independent += inspect_both_slots()
|
|
independent += [('U', 0)] + inspect_both_slots()
|
|
independent += parse('''
|
|
S
|
|
S
|
|
E 0
|
|
S
|
|
S
|
|
E 15
|
|
S
|
|
K 15
|
|
''')
|
|
independent += inspect_both_slots()
|
|
independent += [('U', 1)] + inspect_both_slots()
|
|
independent += parse('''
|
|
S
|
|
E 0
|
|
S
|
|
S
|
|
P 15 -1000000
|
|
S
|
|
S
|
|
K 0
|
|
I 0 0 1 0
|
|
''')
|
|
independent += inspect_both_slots()
|
|
independent += [('U', 0)] + inspect_both_slots()
|
|
cases.append(('pending_witness', parse('''
|
|
I 3 1 8 7
|
|
P 3 11
|
|
S
|
|
P 3 13
|
|
S
|
|
S
|
|
S
|
|
V 3 90
|
|
G
|
|
''')))
|
|
cases.append(('depth_pairing_witness', observe(parse('''
|
|
I 15 3 32 -1000000
|
|
D 15
|
|
D 15
|
|
P 15 1000000
|
|
P 15 -1000000
|
|
E 15
|
|
S
|
|
E 15
|
|
S
|
|
S
|
|
E 15
|
|
'''))))
|
|
domains = [('G',), ('S',), ('H', 0, 31), ('H', 15, 19), ('H', -1, -13), ('H', 16, 14)]
|
|
for ident in [-2147483648, -1, 0, 15, 16, 2147483647]:
|
|
domains += [('V', ident, 817), ('I', ident, -1, 0, 1000001), ('P', ident, 1000001)]
|
|
for op in ['D', 'N', 'E', 'K']:
|
|
domains.append((op, ident))
|
|
for priority, quota, cookie in [(-1, 1, 0), (4, 1, 0), (0, 0, 0), (3, 33, 0),
|
|
(0, 1, -1000001), (3, 32, 1000001),
|
|
(-2147483648, 2147483647, -9223372036854775808),
|
|
(2147483647, -2147483648, 9223372036854775807)]:
|
|
domains.append(('I', 0, priority, quota, cookie))
|
|
domains += parse('''
|
|
I 0 0 1 -1000000
|
|
I 15 3 32 1000000
|
|
I 0 4 1 0
|
|
I 0 0 1 0
|
|
P 0 -1000001
|
|
P 0 -1000000
|
|
P 0 1000000
|
|
P 15 1000000
|
|
S
|
|
P 0 1000000
|
|
P 0 0
|
|
N 0
|
|
N 0
|
|
N 0
|
|
N 0
|
|
D 0
|
|
N 0
|
|
S
|
|
D 0
|
|
E 0
|
|
E 0
|
|
E 0
|
|
E 0
|
|
S
|
|
K 0
|
|
P 0 9223372036854775807
|
|
P 0 -9223372036854775808
|
|
P 0 0
|
|
D 0
|
|
N 0
|
|
E 0
|
|
K 0
|
|
I 0 3 32 1000000
|
|
''')
|
|
cases.append(('domains_precedence_atomicity', observe(domains)))
|
|
lifecycle = parse('''
|
|
I 0 3 2 -19
|
|
I 15 0 32 27
|
|
I 7 0 1 0
|
|
P 0 101
|
|
P 15 17
|
|
P 15 -5
|
|
P 7 300
|
|
D 15
|
|
S
|
|
N 7
|
|
P 7 400
|
|
D 7
|
|
K 0
|
|
S
|
|
S
|
|
E 15
|
|
S
|
|
P 15 -99
|
|
N 15
|
|
K 15
|
|
S
|
|
K 7
|
|
I 15 0 32 -1000000
|
|
P 15 -1000000
|
|
S
|
|
S
|
|
K 15
|
|
I 15 3 1 1000000
|
|
P 15 1000000
|
|
S
|
|
K 15
|
|
G
|
|
''')
|
|
cases.append(('lifecycle_priority_nosync_kill', observe(lifecycle, (0, 7, 15))))
|
|
quota = [('I', 0, 0, 32, 1000000), ('D', 0)]
|
|
quota += [('P', 0, 1000000) for _ in range(32)]
|
|
quota += [('P', 0, -1000000), ('E', 0), ('S',)]
|
|
quota += [('P', 0, -1000000) for _ in range(32)]
|
|
quota += [('P', 0, 0), ('N', 0), ('S',), ('S',), ('E', 0), ('S',), ('S',), ('K', 0)]
|
|
cases.append(('quota_signed_ledger', observe(quota)))
|
|
ring = []
|
|
for i in range(16):
|
|
ring += [('I', i, i % 4, 1, i - 8), ('P', i, 100 - i), ('S',), ('S',), ('K', i)]
|
|
for index in [-1, 0, 15, 16]:
|
|
ring.append(('H', index, 1000 + i))
|
|
ring += [('H', i, -777) for i in range(16)]
|
|
ring += [('G',)] + [('V', i, 2000) for i in range(16)]
|
|
cases.append(('event_ring_reuse', ring))
|
|
randomizer = random.Random(0xD15AB1E)
|
|
commands = [('I', i, i % 4, [1, 2, 8, 32][i % 4], [-1000000, 0, 1000000][i % 3]) for i in range(16)]
|
|
for _ in range(450):
|
|
op = randomizer.choice(['I', 'P', 'P', 'P', 'D', 'N', 'E', 'K', 'S'])
|
|
ident = randomizer.choice([-1, 0, 1, 4, 7, 15, 16])
|
|
if op == 'I':
|
|
command = (op, ident, randomizer.choice([-1, 0, 1, 3, 4]),
|
|
randomizer.choice([0, 1, 2, 32, 33]), randomizer.choice([-1000001, -1000000, 0, 1000000, 1000001]))
|
|
elif op == 'P':
|
|
command = (op, ident, randomizer.choice([-1000001, -1000000, -1, 0, 1, 1000000, 1000001]))
|
|
elif op == 'S':
|
|
command = (op,)
|
|
else:
|
|
command = (op, ident)
|
|
commands.append(command)
|
|
cases.append(('seeded_combinations', observe(commands, (0, 1, 7, 15))))
|
|
# Repeated START/FINISH and same-priority FIFO, including the exact quota=1 edge.
|
|
fifo = [('I', i, 1, 1, 0) for i in [15, 0, 7, 2]]
|
|
fifo += [('P', i, i) for i in [15, 0, 7, 2]]
|
|
fifo += [('P', 15, 9), ('D', 15), ('P', 0, 4)]
|
|
fifo += [('S',), ('N', 0), ('P', 0, -7), ('S',), ('E', 0), ('E', 15)]
|
|
fifo += [('S',)] * 10
|
|
cases.append(('fifo_and_fresh_engine', observe(fifo, (0, 2, 7, 15))))
|
|
long_run = [('I', 0, 0, 32, 1000000)]
|
|
for _ in range(110):
|
|
long_run += [('P', 0, 1000000)] * 32
|
|
long_run += [('S',), ('S',), ('V', 0, 92), ('G',)]
|
|
long_run += [('S',)] * (4096 - len(long_run))
|
|
cases.append(('maximum_stream_wide_totals', long_run))
|
|
cases.append(('interleaved_engine_independence', independent))
|
|
return cases
|
|
|
|
|
|
def main():
|
|
candidate = Path(sys.argv[1])
|
|
sources = {name: (candidate / name).read_bytes() for name in FILES}
|
|
samples = []
|
|
for label, commands in cases():
|
|
assert len(commands) <= 4096
|
|
model = RoutedReference()
|
|
expected = [' '.join(map(str, model.run(command))) for command in commands]
|
|
payload = '\n'.join(' '.join(map(str, command)) for command in commands) + '\n'
|
|
samples.append((label, commands, payload, expected))
|
|
scratch = Path(tempfile.mkdtemp(prefix='deferred-check-', dir=Path.cwd()))
|
|
for name, data in sources.items():
|
|
(scratch / name).write_bytes(data)
|
|
(scratch / 'observer.cpp').write_text(OBSERVER)
|
|
# Every trusted scratch access ends here, before the first candidate-controlled process.
|
|
executable = str(scratch / 'observer')
|
|
built = subprocess.run(['g++', '-std=c++17', '-O2', '-Wall', '-Wextra', '-pedantic',
|
|
str(scratch / 'observer.cpp'), str(scratch / 'registry.cpp'),
|
|
str(scratch / 'dispatch.cpp'), str(scratch / 'inspect.cpp'),
|
|
'-o', executable], capture_output=True, text=True, timeout=30)
|
|
print('build_exit=' + str(built.returncode), file=sys.stderr)
|
|
if built.returncode:
|
|
print(built.stderr[-2000:])
|
|
print('FAIL build')
|
|
return 1
|
|
failures = []
|
|
for label, commands, payload, expected in samples:
|
|
run = subprocess.run([executable], input=payload.encode('ascii'), capture_output=True, timeout=10)
|
|
actual = run.stdout.decode('utf-8', 'replace').splitlines()
|
|
expected_bytes = ('\n'.join(expected) + '\n').encode('ascii')
|
|
if run.returncode:
|
|
failures.append(f'{label}: run_exit={run.returncode} stderr={run.stderr[-150:]!r}')
|
|
elif run.stderr:
|
|
failures.append(f'{label}: run_exit=0 stderr_only={run.stderr[:150]!r}')
|
|
elif run.stdout != expected_bytes:
|
|
at = next((i for i in range(min(len(actual), len(expected))) if actual[i] != expected[i]), min(len(actual), len(expected)))
|
|
got = actual[at] if at < len(actual) else '<missing>'
|
|
want = expected[at] if at < len(expected) else '<end>'
|
|
command = commands[at] if at < len(commands) else '<extra-output>'
|
|
failures.append(f'{label}: run_exit=0 line={at + 1} API={command} expected={want!r} got={got!r} protocol_bytes_equal=False')
|
|
print(f'case={label} run_exit={run.returncode} commands={len(commands)}', file=sys.stderr)
|
|
for failure in failures:
|
|
print(failure, file=sys.stderr)
|
|
if failures:
|
|
print('FAIL behavior: ' + failures[0])
|
|
return 1
|
|
print('PASS')
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|