amos ac3d3db9e0 Add 13 more Qwen-measured tasks from batches 26 and 27 (8 HARD, 5 MEDIUM)
Route D (real fixes in embedded open-source projects) keeps producing the
hard tasks: of 64 measured so far, 15 are HARD and 4 MEDIUM, mostly because
Qwen exhausts its context in the larger real workspaces. This adds
CANopen SRDO and ASCII timing, FreeRTOS DHCP, lwIP MQTT fragments, printf
extremal exponents, Zephyr packet buffers and heatshrink (HARD), plus
lwjson, RFM9x, SFUD and nanopb proto3 (MEDIUM).

From batch 27 (spec-dense implementation tasks) come one HARD and one
MEDIUM; that batch is winding down after 30 tasks gave 1 HARD and 3 MEDIUM.

Each task comes from the commit Qwen measured, its digest matches, and it
is still present at the batch branch head. Agent timeouts stay excluded:
a pyserial attempt timed out on Qwen's own blocking read yet graded PASS.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 16:29:21 +08:00

184 lines
8.7 KiB
Python

"""Observe public SFUD API results through a controlled software NOR bus."""
from pathlib import Path
import json
import base64
import random
import subprocess
import sys
import tempfile
ROOT = Path(__file__).resolve().parent
CAPACITY = {'mx': 4 << 20, 'small': 8 << 20, 'edge': 16 << 20, 'large': 32 << 20,
'normal32': 32 << 20, 'normal64': 64 << 20, 'sst': 1 << 20}
OBSERVER = r'''
#include "virtual_flash.h"
#include <stdio.h>
#include <string.h>
static void row(int id, int step, int init, int setup, int op, virtual_flash *v,
sfud_flash *f, const uint8_t *p, unsigned n) {
unsigned i; uint8_t status = 0xee;
sfud_read_status(f, &status);
printf("%d %d %d %d %d %u %u ", id, step, init, setup, op, status & 0xfc, v->bad_command);
if (!n) putchar('-');
for (i = 0; i < n; i++) printf("%02x", p[i]);
putchar('\n');
}
int main(void) {
char profile[32], mode[16]; unsigned status, salt, addr, n, width;
int fault, id = 0;
while (scanf("%31s %u %u %d %15s %u %u %u", profile, &status, &salt, &fault, mode, &addr, &n, &width) == 8) {
virtual_flash v; sfud_flash f; uint8_t got[1040], data[1024];
int init, setup = -9, op = -9; unsigned i;
if (n > 1024 || !vf_open(&v, &f, profile, status, salt, fault)) return 2;
init = sfud_device_init(&f);
memset(got, 0xcc, sizeof(got));
if (init || !strcmp(mode, "init")) {
row(id++, 0, init, setup, op, &v, &f, got, 0); vf_close(&v); continue;
}
if (!strcmp(mode, "read") || !strcmp(mode, "seq")) {
unsigned widths[5] = {width, 4, 2, 1, width};
unsigned times = !strcmp(mode, "seq") ? 5 : 1;
for (i = 0; i < times; i++) {
memset(got, 0xcc, sizeof(got));
setup = widths[i] ? sfud_qspi_fast_read_enable(&f, widths[i]) : 0;
op = sfud_read(&f, addr, n, got);
row(id, i, init, setup, op, &v, &f, got, n);
}
} else if (!strcmp(mode, "boundwrite")) {
memset(data, salt, sizeof(data));
op = sfud_write(&f, addr, n, data);
row(id, 0, init, 0, op, &v, &f, got, 0);
} else {
for (i = 0; i < n; i++) data[i] = (uint8_t)((i*7u + salt) ^ 0xa5u);
if (!strcmp(mode, "split")) {
setup = sfud_erase(&f, addr, n);
op = sfud_write(&f, addr, n, data);
row(id, 0, init, setup, op, &v, &f, got, 0);
setup = 0;
} else setup = sfud_erase_write(&f, addr, n, data);
op = sfud_read(&f, addr - 4, n + 8, got);
row(id, !strcmp(mode, "split") ? 1 : 0, init, setup, op, &v, &f, got, n + 8);
}
id++; vf_close(&v);
}
return 0;
}
'''
def cases_and_expectations():
rng = random.Random(8171309)
cases = []
def add(profile, status, mode, addr=0, size=0, width=0, fault=0):
cases.append((profile, status, rng.randrange(1, 256), fault, mode, addr, size, width))
# Device initialization, independently of any later I/O.
for status in (0, 4, 0x1c, 0x20, 0x3c, 0x80):
add('mx', status, 'init')
for profile in ('small', 'large', 'normal32'):
for status in (0, 4, 0x1c):
add(profile, status, 'init')
for fault in (1, 2):
for status in (0, 0x1c):
add('mx', status, 'init', fault=fault)
# Protected array updates, page and sector boundaries, and the established AAI path.
for status in (0, 4, 0x1c, 0x20, 0x3c):
for size in (1, 5, 258):
add('mx', status, 'update', (4 << 20) - 4096 + 253, size)
add('mx', 0x1c, 'split', 4090, 300)
for profile in ('small', 'large', 'sst'):
for size in (1, 7, 510):
add(profile, 0x1c if profile == 'sst' else 0, 'split', 4090, size)
# Data at both sides of the addressing threshold, every maximum width and fallback.
for profile in ('small', 'edge', 'large', 'normal32', 'normal64'):
capacity = CAPACITY[profile]
positions = (19, capacity - 33)
if capacity > 16 << 20:
positions += ((16 << 20) - 7, (16 << 20) + 11)
for addr in positions:
for width in (1, 2, 4):
add(profile, 0, 'read', addr, min(17, capacity - addr), width)
add(profile, 0, 'seq', capacity - 513, 31, 1)
add(profile, 0, 'read', capacity - 1, 1, 1)
add(profile, 0, 'read', capacity - 1, 2, 4)
add(profile, 0, 'boundwrite', capacity, 1)
expected = []
for ident, (profile, status, salt, fault, mode, addr, size, width) in enumerate(cases):
prepared = profile in ('mx', 'sst')
init = 2 if profile == 'mx' and fault else 0
final_status = status & 0xfc if init or not prepared else 0
def row(step, setup, op, data):
expected.append([str(x) for x in (ident, step, init, setup, op, final_status, 0)] + [data.hex() if data else '-'])
if init or mode == 'init':
row(0, -9, -9, b'')
elif mode in ('read', 'seq'):
outside = addr + size > CAPACITY[profile]
data = (b'\xcc' * size if outside else
bytes(((a*13) ^ (a >> 9) ^ (a >> 24) ^ salt) & 255 for a in range(addr, addr + size)))
for step in range(5 if mode == 'seq' else 1):
row(step, 0, 5 if outside else 0, data)
elif mode == 'boundwrite':
row(0, 0, 5, b'')
else:
if mode == 'split': row(0, 0, 0, b'')
first = addr // 4096 * 4096
last = (addr + size + 4095) // 4096 * 4096
payload = bytes(((j*7 + salt) ^ 0xa5) & 255 for j in range(size))
# Reads span the changed range plus its neighbours, so successful return codes
# cannot conceal a failed erase/program or corruption outside the write range.
data = bytearray()
for a in range(addr - 4, addr + size + 4):
if addr <= a < addr + size: data.append(payload[a-addr])
elif first <= a < last: data.append(255)
else: data.append(((a*13) ^ (a >> 9) ^ (a >> 24) ^ salt) & 255)
row(1 if mode == 'split' else 0, 0, 0, bytes(data))
return cases, expected
def main():
candidate = Path(sys.argv[1])
# Read all trusted support and candidate bytes before any candidate-controlled process.
sources = {name: base64.b64decode(data) for name, data in json.loads((ROOT / 'build_inputs.json').read_text()).items()}
sources['sfud/src/sfud.c'] = (candidate / 'sfud/src/sfud.c').read_bytes()
sources['observe.c'] = OBSERVER.encode()
cases, expected = cases_and_expectations()
input_text = ''.join(' '.join(map(str, case)) + '\n' for case in cases)
scratch = Path(tempfile.mkdtemp(prefix='sfud-observe-'))
for name, data in sources.items():
p = scratch / name
p.parent.mkdir(parents=True, exist_ok=True)
p.write_bytes(data)
executable = scratch / 'observe'
argv = ['gcc', '-std=c99', '-O1', '-Wall', '-Wextra', '-I' + str(scratch / 'sfud/inc'),
'-I' + str(scratch / 'support'), str(scratch / 'sfud/src/sfud.c'),
str(scratch / 'sfud/src/sfud_sfdp.c'), str(scratch / 'support/virtual_flash.c'),
str(scratch / 'observe.c'), '-o', str(executable)]
# From here on only subprocess results are inspected; never traverse scratch again.
built = subprocess.run(argv, cwd=scratch, capture_output=True, text=True, timeout=25)
if built.returncode:
print('FAIL: build'); return 1
ran = subprocess.run([str(executable)], cwd=scratch, input=input_text, capture_output=True,
text=True, timeout=60)
if ran.returncode:
print('FAIL: observation process'); return 1
observed = [line.split() for line in ran.stdout.splitlines()]
if len(observed) != len(expected):
print('FAIL: observation count'); return 1
for index, (got, wanted) in enumerate(zip(observed, expected)):
# The contract requires failure propagation, not one particular nonzero code.
if wanted[2] != '0' and len(got) == 8 and got[2] in {'1', '2', '3', '4', '5'}:
got[2] = wanted[2]
# Only the MX block-protection field is required to become unprotected;
# implementations may retain other nonvolatile status bits.
if len(got) == 8 and got[0].isdigit():
profile, _, _, fault, _, _, _, _ = cases[int(got[0])] if int(got[0]) < len(cases) else ('',0,0,0,'',0,0,0)
if profile == 'mx' and not fault and got[5].isdigit():
got[5] = str(int(got[5]) & 0x3c)
if got != wanted:
print('case row', index, 'observed', got, 'expected', wanted, file=sys.stderr)
print('FAIL: storage contract'); return 1
print('PASS'); return 0
if __name__ == '__main__':
raise SystemExit(main())