The user wants 2000 tasks, most of them ones Qwen cannot solve. New batch tasks are now measured on their branch first (3 Qwen trials each), and only HARD or MEDIUM tasks reach main; ROUTINE ones stay on the batch branches. A session cut by context_limit or output_limit counts as a failure for difficulty (user decision 2026-11-03); the trajectory stays incomplete. - Batch 26 (route D, real upstream fixes in embedded open-source projects): 7 HARD of 20 (coreJSON, littlefs, CANopen, coreMQTT, LwM2M, Zephyr ring, lwIP SNTP). Most failures are Qwen exhausting its context in the larger workspaces. - Batch 25 (HWE mutation repairs): 1 HARD and 1 MEDIUM of 56. - Batch 27 (spec-dense implementation tasks): 2 MEDIUM of the first 10. Each task comes from the exact commit Qwen measured, and its digest matches the measured one. Failure reasons are not yet reviewed, so difficulty is the automatic label. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
277 lines
13 KiB
Python
277 lines
13 KiB
Python
"""Private deterministic checker. Only the observer child imports candidate code."""
|
|
import copy
|
|
from fractions import Fraction as F
|
|
import json
|
|
from pathlib import Path
|
|
import random
|
|
import secrets
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
SEED = 270227
|
|
RANDOM_VALID = 2400
|
|
RANDOM_INVALID = 300
|
|
|
|
# The observer receives inputs and returns raw values, exception ancestry and input
|
|
# snapshots. It has no expected results, test generation or correctness assertions.
|
|
OBSERVER = r'''
|
|
import importlib.util, json, sys
|
|
class IntChild(int): pass
|
|
class ListChild(list): pass
|
|
class DictChild(dict): pass
|
|
|
|
def unpack(node):
|
|
kind, value = node
|
|
if kind == 'dict': return {unpack(k): unpack(v) for k, v in value}
|
|
if kind == 'list': return [unpack(v) for v in value]
|
|
if kind == 'tuple': return tuple(unpack(v) for v in value)
|
|
if kind == 'int_child': return IntChild(value)
|
|
if kind == 'list_child': return ListChild(unpack(v) for v in value)
|
|
if kind == 'dict_child': return DictChild((unpack(k),unpack(v)) for k,v in value)
|
|
if kind == 'bytes': return bytes.fromhex(value)
|
|
return value
|
|
|
|
def observe(value, raw=False):
|
|
if value is None: return ['none']
|
|
if isinstance(value, bool): return ['bool',bool(value)]
|
|
if isinstance(value, int): result = ['int',str(int(value))]
|
|
elif isinstance(value, float): result = ['float',repr(float(value))]
|
|
elif isinstance(value, str): result = ['str',str(value)]
|
|
elif isinstance(value, bytes): result = ['bytes',bytes(value).hex()]
|
|
elif isinstance(value, list): result = ['list',[observe(v,raw) for v in value]]
|
|
elif isinstance(value, tuple): result = ['tuple',[observe(v,raw) for v in value]]
|
|
elif isinstance(value, dict): result = ['dict',sorted([[observe(k,raw),observe(v,raw)] for k,v in value.items()])]
|
|
else: result = ['other',repr(value)]
|
|
if raw: result += [type(value).__module__,type(value).__qualname__]
|
|
return result
|
|
|
|
spec=importlib.util.spec_from_file_location('area_candidate',sys.argv[1])
|
|
module=importlib.util.module_from_spec(spec)
|
|
sys.modules[spec.name]=module
|
|
spec.loader.exec_module(module)
|
|
rows=[]
|
|
for encoded in json.load(sys.stdin):
|
|
data=unpack(encoded)
|
|
before=observe(data,True)
|
|
try:
|
|
result=module.resample(data)
|
|
row={'value':observe(result)}
|
|
except BaseException as exc:
|
|
row={'exception':{'name':type(exc).__name__,
|
|
'TypeError':isinstance(exc,TypeError),'ValueError':isinstance(exc,ValueError),
|
|
'KeyError':isinstance(exc,KeyError)}}
|
|
row['before']=before
|
|
row['after']=observe(data,True)
|
|
rows.append(row)
|
|
print('OBSERVATIONS:'+sys.argv[2]+':'+json.dumps(rows,separators=(',',':')))
|
|
'''
|
|
|
|
|
|
def tag(value):
|
|
if value is None: return ['none']
|
|
if isinstance(value, bool): return ['bool', value]
|
|
if isinstance(value, int): return ['int', str(value)]
|
|
if isinstance(value, str): return ['str', value]
|
|
if isinstance(value, list): return ['list', [tag(v) for v in value]]
|
|
if isinstance(value, dict): return ['dict', sorted([[tag(k), tag(v)] for k,v in value.items()])]
|
|
raise AssertionError('unsupported expected value')
|
|
|
|
|
|
class InputInt(int): pass
|
|
class InputList(list): pass
|
|
class InputDict(dict): pass
|
|
|
|
|
|
def pack(value):
|
|
if type(value) is InputInt: return ['int_child', int(value)]
|
|
if type(value) is InputList: return ['list_child', [pack(v) for v in value]]
|
|
if type(value) is InputDict: return ['dict_child', [[pack(k),pack(v)] for k,v in value.items()]]
|
|
if isinstance(value, dict): return ['dict', [[pack(k),pack(v)] for k,v in value.items()]]
|
|
if isinstance(value, list): return ['list', [pack(v) for v in value]]
|
|
if isinstance(value, tuple): return ['tuple', [pack(v) for v in value]]
|
|
if isinstance(value, bytes): return ['bytes', value.hex()]
|
|
return ['scalar', value]
|
|
|
|
|
|
def reference(request):
|
|
"""Partition each cell at source grid lines; integrate constant subrectangles."""
|
|
source = request['pixels']
|
|
h, w = len(source), len(source[0]) if source else 0
|
|
x0,y0,x1,y1 = request['window']
|
|
ow,oh = request['size']
|
|
output, coverage = [], []
|
|
for j in range(oh):
|
|
values, ratios = [], []
|
|
for i in range(ow):
|
|
l, r = F(x0) + F((x1-x0)*i,ow), F(x0) + F((x1-x0)*(i+1),ow)
|
|
t, b = F(y0) + F((y1-y0)*j,oh), F(y0) + F((y1-y0)*(j+1),oh)
|
|
xs = sorted({l,r,*[F(k) for k in range(w+1) if l < k < r]})
|
|
ys = sorted({t,b,*[F(k) for k in range(h+1) if t < k < b]})
|
|
area, integral = F(0), F(0)
|
|
for a,z in zip(ys,ys[1:]):
|
|
midy = (a+z)/2
|
|
sy = midy.numerator // midy.denominator
|
|
for q,s in zip(xs,xs[1:]):
|
|
midx = (q+s)/2
|
|
sx = midx.numerator // midx.denominator
|
|
if 0 <= sy < h and 0 <= sx < w and source[sy][sx] is not None:
|
|
weight = (z-a)*(s-q)
|
|
area += weight
|
|
integral += source[sy][sx]*weight
|
|
nominal = (r-l)*(b-t)
|
|
ratio = area/nominal if nominal else F(0)
|
|
ratios.append([ratio.numerator,ratio.denominator])
|
|
if area > 0 and ratio >= F(*request['minimum']):
|
|
mean = integral/area
|
|
lower = mean.numerator // mean.denominator
|
|
distances = [(abs(mean-k), k%2, k) for k in (lower,lower+1)]
|
|
values.append(min(distances)[2])
|
|
else:
|
|
values.append(request['fill'])
|
|
output.append(values)
|
|
coverage.append(ratios)
|
|
return {'pixels':output,'coverage':coverage}
|
|
|
|
|
|
cases, expected, labels = [], [], []
|
|
|
|
def add(data, label, error=None):
|
|
cases.append(pack(data))
|
|
expected.append(error if error else tag(reference(data)))
|
|
labels.append(label)
|
|
|
|
|
|
def base(**changes):
|
|
result = dict(pixels=[[0,1],[2,None]],window=[0,0,2,2],size=[2,2],minimum=[0,1],fill=17)
|
|
result.update(changes)
|
|
return result
|
|
|
|
# Handwritten rule and cross-rule boundaries, including even/odd midpoint values.
|
|
for p in ([],[[]],[[],[],[]],[[None]],[[0]],[[65535]],[[0,1]],[[1,2]],[[2,3]],[[65534,65535]],[[None,7],[9,None]]):
|
|
for wh in ([0,0],[0,3],[3,0],[1,1],[2,3],[3,2]):
|
|
add(base(pixels=p,size=wh), 'shape-and-empty')
|
|
for window in ([0,0,0,0],[0,0,0,2],[0,0,2,0],[-2,-2,4,4],[-1,0,1,2],[2,0,3,2],[0,2,2,3],[-1,-1,0,0],[1,1,2,2],[0,0,1,1],[1,0,2,1]):
|
|
for minimum in ([0,1],[1,1],[1,2],[2,4],[3,4]):
|
|
add(base(window=window,minimum=minimum,size=[3,3]),'window-and-coverage')
|
|
for pair in ([0,1],[1,2],[2,3],[65534,65535],[0,65535]):
|
|
add(base(pixels=[pair],window=[0,0,2,1],size=[1,1]),'rounding-midpoints')
|
|
for threshold in ([0,1],[1,2],[2,4],[500,1000],[1,1]):
|
|
add(base(pixels=[[8,None]],window=[0,0,2,1],size=[1,1],minimum=threshold),'threshold-equality')
|
|
add(base(pixels=[[0,None]],window=[0,0,2,1],size=[1,1],minimum=[0,1]),'zero-sample-is-valid')
|
|
add(base(pixels=[[5]],window=[0,0,2,2],size=[1,1],minimum=[1,4]),'outside-in-denominator')
|
|
add(base(pixels=[[None]],window=[0,0,1,1],size=[1,1],minimum=[0,1]),'no-area-at-zero-threshold')
|
|
add(base(ignored={'arbitrary':[True,InputInt(3),b'abc',(None,)]}),'extra-keys-ignored')
|
|
big=10**400
|
|
for window in ([-big,-big,big,big],[big,0,big+1,1],[0,0,big,1],[0,0,big,big]):
|
|
add(base(pixels=[[65535]],window=window,size=[2,3],minimum=[0,big]),'arbitrary-integer-coordinates')
|
|
add(base(pixels=[[3]],window=[0,0,2,1],size=[1,1],minimum=[big,2*big]),'arbitrary-integer-threshold')
|
|
add(base(pixels=[[65535 if (r+c)%3 else None for c in range(32)] for r in range(32)],window=[0,0,32,32],size=[32,32],minimum=[1,1],fill=0),'maximum-dimensions')
|
|
add(base(pixels=[[r*32+c for c in range(32)] for r in range(32)],window=[-1,-1,33,33],size=[3,5]),'large-downsample')
|
|
|
|
rng=random.Random(SEED)
|
|
for trial in range(RANDOM_VALID):
|
|
h,w=rng.randrange(5),rng.randrange(5)
|
|
pixels=[[rng.choice([None,0,1,2,3,7,65535]) for _ in range(w)] for _ in range(h)]
|
|
x0,x1=sorted([rng.randrange(-2,7),rng.randrange(-2,7)])
|
|
y0,y1=sorted([rng.randrange(-2,7),rng.randrange(-2,7)])
|
|
den=rng.randrange(1,6);num=rng.randrange(den+1)
|
|
add(base(pixels=pixels,window=[x0,y0,x1,y1],size=[rng.randrange(5),rng.randrange(5)],minimum=[num,den],fill=rng.choice([0,1,2,17,65535])),'random-valid-'+str(trial))
|
|
|
|
# Invalid cases carry explicit expected exception identities, never candidate or
|
|
# gold-derived expectations. Type/length/value ordering is exercised separately.
|
|
for value in (None,[],(),True,0,'pixels',InputDict()):
|
|
add(value,'request-type','TypeError')
|
|
for missing in ('pixels','window','size','minimum','fill'):
|
|
data=base();del data[missing]
|
|
add(data,'missing-'+missing,'KeyError')
|
|
add({'pixels':False},'missing-before-invalid-field','KeyError')
|
|
for value in (None,(),{},InputList(),1):
|
|
add(base(pixels=value),'image-container','TypeError')
|
|
for value in ([None],[(0,)],[[0],InputList([0])],[[0]*33,False]):
|
|
add(base(pixels=value),'row-type-before-width','TypeError')
|
|
for value in ([[0]*33],[[0],[]],[[]]*33,[[0],[0,1]]):
|
|
add(base(pixels=value),'image-shape','ValueError')
|
|
for value in (True,False,0.0,'1',[],{},InputInt(1)):
|
|
add(base(pixels=[[value]]),'pixel-type','TypeError')
|
|
for value in (-1,65536,big,-big):
|
|
add(base(pixels=[[value]]),'pixel-range','ValueError')
|
|
for key,length in (('window',4),('size',2),('minimum',2)):
|
|
for value in (None,(),{},InputList([0]*length)):
|
|
add(base(**{key:value}),key+'-container','TypeError')
|
|
for value in ([],[0]*(length-1),[0]*(length+1)):
|
|
add(base(**{key:value}),key+'-length','ValueError')
|
|
for slot in range(length):
|
|
for bad in (True,None,0.0,InputInt(1)):
|
|
value=base()[key].copy();value[slot]=bad
|
|
add(base(**{key:value}),key+'-element-type','TypeError')
|
|
for value in ([1,0,0,1],[0,2,1,1]):
|
|
add(base(window=value),'window-order','ValueError')
|
|
for value in ([-1,1],[1,-1],[33,1],[1,33],[big,0]):
|
|
add(base(size=value),'size-range','ValueError')
|
|
for value in ([-1,1],[1,0],[0,0],[2,1],[0,-1],[big+1,big]):
|
|
add(base(minimum=value),'minimum-range','ValueError')
|
|
for value in (True,False,None,0.0,'0',InputInt(1)):
|
|
add(base(fill=value),'fill-type','TypeError')
|
|
for value in (-1,65536,big):
|
|
add(base(fill=value),'fill-range','ValueError')
|
|
|
|
# Explicit compound errors lock stage order and reject early empty shortcuts.
|
|
compound=[
|
|
(dict(pixels=[[0],[0,1]],window=False),'ValueError'),
|
|
(dict(pixels=[[-1]],window=False),'ValueError'),
|
|
(dict(pixels=[[False]],window=False),'TypeError'),
|
|
(dict(window=[2,0,1,1],size=False),'ValueError'),
|
|
(dict(window=[2,0,False,1]),'TypeError'),
|
|
(dict(window=[False],size=False),'ValueError'),
|
|
(dict(size=[-1,False],minimum=False),'TypeError'),
|
|
(dict(size=[33,0],minimum=False),'ValueError'),
|
|
(dict(minimum=[-1,False],fill=False),'TypeError'),
|
|
(dict(minimum=[2,1],fill=False),'ValueError'),
|
|
(dict(size=[0,0],fill=False),'TypeError'),
|
|
(dict(window=[0,0,0,0],minimum=[1,0]),'ValueError'),
|
|
(dict(pixels=[[-1,False]]),'ValueError'),
|
|
(dict(pixels=[[False,-1]]),'TypeError'),
|
|
(dict(pixels=[[0]*33]*33),'ValueError')]
|
|
for changes,error in compound:
|
|
add(base(**changes),'multi-error-priority',error)
|
|
for trial in range(RANDOM_INVALID):
|
|
changes,error=rng.choice(compound)
|
|
data=base(**copy.deepcopy(changes));data['unused']=[trial,True,{'x':None}]
|
|
add(data,'random-invalid-'+str(trial),error)
|
|
|
|
|
|
def require(condition,message):
|
|
if not condition:
|
|
raise AssertionError(message)
|
|
|
|
try:
|
|
source=Path(sys.argv[1])/'area_resize.py'
|
|
require(source.is_file() and not source.is_symlink(),'missing candidate')
|
|
scratch=Path(tempfile.mkdtemp(prefix='area-resize-',dir=Path.cwd()))
|
|
shutil.copyfile(source,scratch/'area_resize.py')
|
|
(scratch/'observer.py').write_text(OBSERVER,encoding='utf-8')
|
|
nonce=secrets.token_hex(16)
|
|
# The wrapper owns this temporary tree. After candidate startup the parent
|
|
# only reads captured pipes and in-memory expectations; it never visits it.
|
|
result=subprocess.run([sys.executable,'-I','-B',str(scratch/'observer.py'),str(scratch/'area_resize.py'),nonce],input=json.dumps(cases),capture_output=True,text=True,cwd=scratch,timeout=40)
|
|
require(result.returncode==0,'observer failed')
|
|
marker='OBSERVATIONS:'+nonce+':'
|
|
marked=[line[len(marker):] for line in result.stdout.splitlines() if line.startswith(marker)]
|
|
require(len(marked)==1,'observation marker count')
|
|
rows=json.loads(marked[0])
|
|
require(isinstance(rows,list) and len(rows)==len(expected),'observation count')
|
|
for index,(row,want) in enumerate(zip(rows,expected)):
|
|
label=str(index)+':'+labels[index]
|
|
require(row.get('before')==row.get('after'),'input mutation '+label)
|
|
if isinstance(want,str):
|
|
observation=row.get('exception',{})
|
|
require(observation.get(want) is True,'exception identity '+label)
|
|
else:
|
|
require(row.get('value')==want,'result mismatch '+label)
|
|
print('PASS')
|
|
except Exception as error:
|
|
print('FAIL:',type(error).__name__,str(error)[:300])
|
|
sys.exit(1)
|