The user wants 2000 tasks, most of them ones Qwen cannot solve. New batch tasks are now measured on their branch first (3 Qwen trials each), and only HARD or MEDIUM tasks reach main; ROUTINE ones stay on the batch branches. A session cut by context_limit or output_limit counts as a failure for difficulty (user decision 2026-11-03); the trajectory stays incomplete. - Batch 26 (route D, real upstream fixes in embedded open-source projects): 7 HARD of 20 (coreJSON, littlefs, CANopen, coreMQTT, LwM2M, Zephyr ring, lwIP SNTP). Most failures are Qwen exhausting its context in the larger workspaces. - Batch 25 (HWE mutation repairs): 1 HARD and 1 MEDIUM of 56. - Batch 27 (spec-dense implementation tasks): 2 MEDIUM of the first 10. Each task comes from the exact commit Qwen measured, and its digest matches the measured one. Failure reasons are not yet reviewed, so difficulty is the automatic label. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
170 lines
8.7 KiB
Python
170 lines
8.7 KiB
Python
"""Trusted expected values remain in this process; the observer only prints observations."""
|
|
from pathlib import Path
|
|
import random
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import zipfile
|
|
|
|
PRIVATE = Path(__file__).resolve().parent
|
|
CANDIDATES = ["os/services/lwm2m/lwm2m-tlv.c", "os/services/lwm2m/lwm2m-tlv.h"]
|
|
SENTINEL = 123456789
|
|
MAXIMUM = (1 << 31) - 1
|
|
|
|
def tlv(payload, kind=3, identifier=5700, length_width=None):
|
|
if length_width is None:
|
|
length_width = 0 if len(payload) < 8 else 1
|
|
first = (kind << 6) | (32 if identifier > 255 else 0) | (length_width << 3)
|
|
if length_width == 0:
|
|
first |= len(payload)
|
|
return (bytes([first]) + identifier.to_bytes(2 if identifier > 255 else 1, "big")
|
|
+ (len(payload).to_bytes(length_width, "big") if length_width else b"") + payload)
|
|
|
|
def fixed(payload, bits):
|
|
width = len(payload)
|
|
if width not in (4, 8):
|
|
return 0, 0
|
|
frac, expbits, bias = (23, 8, 127) if width == 4 else (52, 11, 1023)
|
|
word = int.from_bytes(payload, "big")
|
|
exponent = (word >> frac) & ((1 << expbits) - 1)
|
|
if exponent == (1 << expbits) - 1:
|
|
return 0, 0
|
|
if not exponent:
|
|
return width, 0
|
|
mantissa = (word & ((1 << frac) - 1)) | (1 << frac)
|
|
shift = exponent - bias - frac + bits
|
|
magnitude = mantissa << shift if shift >= 0 else mantissa >> -shift
|
|
magnitude = min(magnitude, MAXIMUM)
|
|
return width, -magnitude if word >> (frac + expbits) else magnitude
|
|
|
|
def hx(data):
|
|
return data.hex() if data else "-"
|
|
|
|
def cases():
|
|
commands, expected, labels = [], [], []
|
|
def add(command, result, label):
|
|
commands.append(command)
|
|
expected.append(str(result))
|
|
labels.append(label)
|
|
rng = random.Random(334165)
|
|
samples = []
|
|
for width in (4, 8):
|
|
frac, ebits, bias = (23, 8, 127) if width == 4 else (52, 11, 1023)
|
|
full = (1 << ebits) - 1
|
|
# Finite normal values, exponent extremes, signs, and low significand bits.
|
|
for bits in (0, 1, 4, 8, 16, 23, 30):
|
|
for exponent in (1, bias - 40, bias - bits - 1, bias - bits,
|
|
bias, bias + 3, bias + 30 - bits,
|
|
bias + 31 - bits, full - 1):
|
|
for sign in (0, 1):
|
|
for mantissa in (0, (1 << frac) - 1, rng.randrange(1 << frac)):
|
|
word = (sign << (frac + ebits)) | (exponent << frac) | mantissa
|
|
samples.append((word.to_bytes(width, "big"), bits, "normal"))
|
|
for sign in (0, 1):
|
|
for exponent, mantissa, label in ((0, 0, "zero"), (0, 1, "subnormal"),
|
|
(0, (1 << frac) - 1, "subnormal"),
|
|
(full, 0, "infinity"), (full, 1, "nan"),
|
|
(full, (1 << frac) - 1, "nan")):
|
|
word = (sign << (frac + ebits)) | (exponent << frac) | mantissa
|
|
samples.append((word.to_bytes(width, "big"), bits, label))
|
|
for length in (0, 1, 2, 3, 5, 6, 7, 9, 10, 12, 16):
|
|
samples.append((bytes(rng.randrange(256) for _ in range(length)), 8, "bad-width"))
|
|
for i, (payload, bits, label) in enumerate(samples):
|
|
consumed, value = fixed(payload, bits)
|
|
add("D %d %s" % (bits, hx(payload)), "D %d %d" % (consumed, value), "direct:" + label)
|
|
identifier = (0, 15, 255, 256, 5700, 65535)[i % 6]
|
|
length_width = (1, 2, 3)[i % 3] if i % 4 == 0 or len(payload) >= 8 else 0
|
|
packet = tlv(payload, (1, 3)[i % 2], identifier, length_width)
|
|
# Some requests include a later TLV; successful reads consume one record.
|
|
trailing = b"\xc1\x01\x00" if i % 5 == 0 else b""
|
|
result = "R %d %d %d" % (len(packet) if consumed else 0, value, len(payload) if consumed else 99)
|
|
add("R %d %s" % (bits, hx(packet + trailing)), result, "object:" + label)
|
|
for data in (b"", b"\xc4", b"\xe4\x16", b"\xd8\x01\x00",
|
|
b"\xc4\x01\x3f\x80\x00", b"\xc8\x01\x08\x00"):
|
|
add("R 8 " + hx(data), "R 0 %d 99" % SENTINEL, "truncated-object")
|
|
add("T " + hx(data), "T 0", "truncated-tlv")
|
|
# Header interpretation and trailing-record behavior, without stressing storage safety.
|
|
for length in (0, 1, 4, 7, 8, 15, 255, 256):
|
|
payload = bytes(rng.randrange(256) for _ in range(length))
|
|
for identifier in (7, 256, 65535):
|
|
for kind in (0, 1, 2, 3):
|
|
packet = tlv(payload, kind, identifier, 0 if length < 8 else (1 if length < 256 else 2))
|
|
add("T " + hx(packet + b"tail"), "T %d %d %d %d %d %s" %
|
|
(len(packet), kind, identifier, length, len(packet), hx(payload)), "generic-tlv")
|
|
for bits in (0, 4, 8, 16, 23, 30):
|
|
for value in (0, 1, -1, 320, -12345, 65535, -(1 << 22)):
|
|
for identifier, ri in ((15, 0), (5700, 0), (65535, 1)):
|
|
payload = struct.pack(">f", value / (2 ** bits))
|
|
packet = tlv(payload, 1 if ri else 3, identifier)
|
|
add("W %d %d %d %d" % (bits, value, identifier, ri),
|
|
"W %d %s %d %d 4 %d %d %d" %
|
|
(len(packet), hx(packet), len(packet), value, len(packet), 1 if ri else 3, identifier), "float-writer")
|
|
for value in (0, 1, 127, 128, 255, 32767, 32768, 65535, MAXIMUM, -1, -128, -129, -32768, -2147483648):
|
|
size = next(n for n in range(1, 5) if -(1 << (n * 8 - 1)) <= value < (1 << (n * 8 - 1)))
|
|
payload = value.to_bytes(size, "big", signed=True)
|
|
for identifier, ri in ((15, 0), (5700, 0), (65535, 1)):
|
|
packet = tlv(payload, 1 if ri else 3, identifier)
|
|
add("I 0 %d %d %d" % (value, identifier, ri),
|
|
"I %d %s %d %d %d %d %d %d" %
|
|
(len(packet), hx(packet), len(packet), value, size, len(packet), 1 if ri else 3, identifier), "integer")
|
|
for value in (0, 1, -55, 123456):
|
|
for identifier, ri in ((15, 0), (5700, 1)):
|
|
payload = bytes([int(bool(value))])
|
|
packet = tlv(payload, 1 if ri else 3, identifier)
|
|
add("B 0 %d %d %d" % (value, identifier, ri),
|
|
"B %d %s %d %d 1 %d %d %d" %
|
|
(len(packet), hx(packet), len(packet), int(bool(value)), len(packet), 1 if ri else 3, identifier), "boolean")
|
|
for payload in (b"", b"A", b"sensor state", bytes(range(1, 32))):
|
|
for capacity in (len(payload), len(payload)+1, len(payload)+5):
|
|
packet = tlv(payload)
|
|
good = capacity > len(payload)
|
|
add("S 5700 %d %s" % (capacity, hx(payload)),
|
|
"S %d %s %d %d %s" %
|
|
(len(packet), hx(packet), len(packet) if good else 0, len(payload) if good else 99,
|
|
hx(payload+b"\0") if good else "aa"), "string")
|
|
return commands, expected, labels
|
|
|
|
def main():
|
|
candidate = Path(sys.argv[1])
|
|
commands, expected, labels = cases()
|
|
with tempfile.TemporaryDirectory(prefix="lwm2m-width-") as name:
|
|
work = Path(name)
|
|
with zipfile.ZipFile(PRIVATE / "support.zip") as support:
|
|
for member in support.infolist():
|
|
destination = work / member.filename
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_bytes(support.read(member))
|
|
for relative in CANDIDATES:
|
|
destination = work / relative
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(candidate / relative, destination)
|
|
(work / "observer.c").write_bytes((PRIVATE / "observer.c").read_bytes())
|
|
argv = ["gcc", "-std=c99", "-O2", "-Iinclude", "-Ios/services/lwm2m", "observer.c",
|
|
"os/services/lwm2m/lwm2m-tlv.c", "os/services/lwm2m/lwm2m-tlv-reader.c",
|
|
"os/services/lwm2m/lwm2m-tlv-writer.c", "-o", "observe"]
|
|
compiled = subprocess.run(argv, cwd=work, capture_output=True, text=True, timeout=30)
|
|
if compiled.returncode:
|
|
print("FAIL: build")
|
|
return 1
|
|
# All scratch input was written before candidate execution. Do not read scratch afterward.
|
|
observed = subprocess.run([str(work / "observe")], cwd=work, input="\n".join(commands)+"\n",
|
|
capture_output=True, text=True, timeout=20)
|
|
if observed.returncode:
|
|
print("FAIL: observer exit")
|
|
return 1
|
|
lines = observed.stdout.splitlines()
|
|
if len(lines) != len(expected):
|
|
print("FAIL: observation count")
|
|
return 1
|
|
for number, (actual, wanted, label) in enumerate(zip(lines, expected, labels)):
|
|
if actual != wanted:
|
|
print("FAIL: behavior %s at %d" % (label, number))
|
|
return 1
|
|
print("PASS")
|
|
return 0
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|