amos ac3d3db9e0 Add 13 more Qwen-measured tasks from batches 26 and 27 (8 HARD, 5 MEDIUM)
Route D (real fixes in embedded open-source projects) keeps producing the
hard tasks: of 64 measured so far, 15 are HARD and 4 MEDIUM, mostly because
Qwen exhausts its context in the larger real workspaces. This adds
CANopen SRDO and ASCII timing, FreeRTOS DHCP, lwIP MQTT fragments, printf
extremal exponents, Zephyr packet buffers and heatshrink (HARD), plus
lwjson, RFM9x, SFUD and nanopb proto3 (MEDIUM).

From batch 27 (spec-dense implementation tasks) come one HARD and one
MEDIUM; that batch is winding down after 30 tasks gave 1 HARD and 3 MEDIUM.

Each task comes from the commit Qwen measured, its digest matches, and it
is still present at the batch branch head. Agent timeouts stay excluded:
a pyserial attempt timed out on Qwen's own blocking read yet graded PASS.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 16:29:21 +08:00

110 lines
4.0 KiB
Python

"""Independent C-library oracle for the published near-extremal contract."""
import ctypes
import pathlib
import random
import shutil
import struct
import subprocess
import sys
import tempfile
TARGETS = ("src/printf/printf.c", "src/printf/printf.h")
LIBC = ctypes.CDLL(None)
LIBC.snprintf.restype = ctypes.c_int
def require(condition, message):
if not condition:
raise AssertionError(message)
def oracle(fmt, value):
buf = ctypes.create_string_buffer(256)
count = LIBC.snprintf(buf, len(buf), fmt.encode("ascii"), ctypes.c_double(value))
require(0 <= count < len(buf), "oracle buffer")
return "{}\t{}".format(count, buf.value.decode("ascii"))
def bits(value):
return struct.unpack("=Q", struct.pack("=d", value))[0]
def covered(value):
magnitude = abs(value)
return (1e-308 <= magnitude <= 3e-308 or
9e307 <= magnitude <= sys.float_info.max)
def cases():
rows = []
# Decimal boundary inputs are fixed public data; randomized nearby bit patterns
# prevent a repair tied to one printed literal.
boundary = (
1.2345678901e-308, -1.2345678901e-308,
1.2345e-308, -1.2345e-308,
1e-308, -1e-308, 2.2250738585072014e-308,
1.23e308, -1.23e308, 9.87654321e307,
)
for fmt, value in (
("%.10e", boundary[0]), ("%.10E", boundary[1]),
("%.3g", boundary[2]), ("%.3G", boundary[3]),
("%.3e", boundary[4]), ("%+.3E", boundary[7]),
("%.4e", boundary[6]), ("%#.4G", boundary[8]),
("%.5e", boundary[9]),
("% 20.3e", boundary[0]), ("%-20.3e", boundary[0]),
("%+020.3E", boundary[1]), ("%#.3G", boundary[7]),
("%e", boundary[0]), ("%G", boundary[7]),
("%.3e", sys.float_info.max),
):
rows.append((fmt, value))
rng = random.Random(0xF83521B)
formats = ("%.2e", "%.3E", "%.6e", "%.8e", "%.3g", "%.5G", "%+.4e", "%#.3g")
for _ in range(36):
base = rng.choice(boundary)
offset = rng.randrange(-500, 501)
raw = bits(base) + offset
if raw < 0 or raw > (1 << 64) - 1:
continue
value = struct.unpack("=d", struct.pack("=Q", raw))[0]
if not covered(value):
continue
rows.append((rng.choice(formats), value))
return rows
try:
test_rows = cases()
require(all(covered(value) for _, value in test_rows), "test input outside public range")
expected = [oracle(fmt, value) for fmt, value in test_rows]
stdin = "".join("{} {:016x}\n".format(fmt.replace(" ", "~"), bits(value))
for fmt, value in test_rows)
# The grade container owns this scratch lifetime. The supervising checker
# never traverses it after candidate code starts, including at function exit.
scratch = pathlib.Path(tempfile.mkdtemp(prefix="printf-grade-", dir="/tmp"))
for rel in TARGETS:
source = pathlib.Path(sys.argv[1]) / rel
require(source.is_file() and not source.is_symlink(), "missing " + rel)
target = scratch / rel
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(source, target)
shutil.copyfile(pathlib.Path(__file__).with_name("harness.c"), scratch / "harness.c")
build = subprocess.run(
["gcc", "-std=c99", "-O2", "-fno-builtin", "-Isrc", "harness.c",
"src/printf/printf.c", "-o", "observe"],
cwd=scratch, capture_output=True, text=True, timeout=90,
)
require(build.returncode == 0, "candidate build failed: " + build.stderr[-600:])
run = subprocess.run([str(scratch / "observe")], cwd=scratch,
input=stdin, capture_output=True, text=True, timeout=30)
require(run.returncode == 0, "candidate execution failed")
observed = run.stdout.splitlines()
require(len(observed) == len(expected), "observation count")
for index, (actual, want) in enumerate(zip(observed, expected)):
require(actual == want, "case {} mismatch: {} != {}".format(index, actual, want))
print("PASS")
except Exception as exc:
print("FAIL: {}: {}".format(type(exc).__name__, exc))
sys.exit(1)