amos 9a08a6141b Add 2 HARD tasks from batch 28: ESP NVS iterator versions, RT-Thread workqueue timer
Batch 28 commit b7fd210 gave ESP NVS iterator versions and RT-Thread
workqueue timer state as HARD: all 3 attempts of each ran out of model
context, which counts as a failure. Each task comes from the commit Qwen
measured, its digest matches, and it is still present at the batch
branch head.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-04 23:49:00 +08:00

183 lines
9.9 KiB
Python

"""Independent public API model; subprocesses only return raw observations."""
import json
import os
from pathlib import Path
import random
import subprocess
import sys
import tempfile
ERR = -255
BUSY = -16
ROOT = Path(__file__).resolve().parent
CANDIDATES = ['components/drivers/ipc/workqueue.c', 'src/timer.c', 'include/rtdef.h']
def cases():
rng = random.Random(609371)
result = []
def add(name, commands, expected):
result.append((name, '\n'.join(commands) + '\n', expected))
def ret(n=0):
return 'R ' + str(n)
for n in range(8):
base = rng.randrange(10, 200)
delay = rng.randrange(3, 12)
later = rng.randrange(13, 31)
value = rng.randrange(100000, 10000000000)
add('work_deadline_'+str(n), [f'TICK {base}',f'WORK 0 {value} 0 0',f'SUB 0 0 {delay}',
'DRAIN',f'ADV {base+1}',f'SUB 0 0 {later}','DRAIN',f'ADV {base+delay}',
f'ADV {base+later}',f'ADV {base+later+1}','CANCEL 0 0',f'SUB 1 0 {delay}',
'DRAIN',f'ADV {base+later+1+delay}'],
[ret(),ret(),f'W 0 {value} {base+later+1}',ret(),ret(),f'W 0 {value} {base+later+1+delay}'])
add('work_cancel_'+str(n), [f'TICK {base}',f'WORK 0 {value} 0 0',f'SUB 0 0 {delay}',
'DRAIN','CANCEL 0 0',f'ADV {base+delay+1}','SUB 0 0 0','CANCEL 0 0','DRAIN',
f'SUB 0 0 {later}','DRAIN','SYNC 0 0',f'ADV {base+delay+later+2}'], [ret()]*6)
for operation in ['SUB','CANCEL']:
commands=[f'TICK {base}',f'WORK 0 {value} 0 0',f'SUB 0 0 {delay}','DRAIN',
f'TICK {base+delay}',f'PROGRESS {n%2}',
f'RACE 0 0 {int(operation=="CANCEL")} {later}','DRAIN',
f'ADV {base+delay+later}',f'SUB 0 0 {delay}','DRAIN',f'ADV {base+delay+later+delay}']
add('dispatch_'+operation+'_'+str(n),commands,{'race':{'operation':operation,
'value':value,'old_tick':base+delay,'new_tick':base+delay+later,
'final_tick':base+delay+later+delay}})
add('submit_cancel_overlap', ['WORK 0 987321 0 0','APP 0 0 8','STEP app','CANCEL 0 0',
'DRAIN','ADV 20','SUB 0 0 3','DRAIN','ADV 23'], {'alternatives':[
[ret(),ret(),ret(),'W 0 987321 23'],
[ret(),ret(),'W 0 987321 20',ret(),'W 0 987321 23']]})
add('ready_order', ['WORK 0 123 0 0','WORK 1 456 0 0','WORK 2 789 0 0','DO 0 0','SUB 0 1 0',
'URGENT 0 2','DRAIN','SUB 0 0 4','SUB 0 1 0','ALL 0','ADV 8'],
[ret()]*3+['W 2 789 0','W 0 123 0','W 1 456 0']+[ret()]*3)
add('work_callback_resubmit', ['WORK 0 987654321 1 5','SUB 0 0 2','DRAIN','ADV 2','ADV 6','ADV 7','ADV 20'],
[ret(),'W 0 987654321 2',ret(),'W 0 987654321 7'])
add('work_callback_cancel', ['WORK 0 111222333 2 0','SUB 0 0 2','DRAIN','ADV 2','ADV 10'],
[ret(),'W 0 111222333 2',ret(BUSY)])
for soft in [0,1]:
for n in range(4):
base=rng.randrange(200,400);d=rng.randrange(3,12);next_d=rng.randrange(13,30)
add(f'timer_set_{soft}_{n}',[f'TICK {base}',f'TIMER 0 {d} {soft} 0 0 0','GET 0','START 0','GET 0',
f'ADV {base+1}',f'SET 0 {next_d}','GET 0',f'ADV {base+d}',f'ADV {base+next_d+1}',
'START 0','GET 0',f'ADV {base+2*next_d+1}','GET 0','STOP 0'],
[f'G {d} 0',ret(),f'G {d} 1',ret(),f'G {next_d} 0',ret(),f'G {next_d} 1',
f'T 0 {base+2*next_d+1}',f'G {next_d} 0',ret(ERR)])
for periodic in [0,1]:
add(f'timer_reenter_{soft}_{periodic}_{n}',[f'TICK {base}',f'TIMER 0 {d} {soft} {periodic} 1 0',
'START 0','DRAIN',f'ADV {base+d}',f'ADV {base+2*d}',
'STOP 0','START 0',f'ADV {base+3*d}'],
[ret(),f'T 0 {base+d}',ret(ERR)]+
([f'T 0 {base+2*d}',ret(ERR),ret()] if periodic else [ret(ERR)])+
[ret(),f'T 0 {base+3*d}',ret(ERR)])
add(f'timer_callback_set_{soft}_{n}',[f'TICK {base}',f'TIMER 0 {d} {soft} 1 2 {next_d}',
'START 0','DRAIN',f'ADV {base+d}','GET 0',f'ADV {base+2*d}','START 0',
f'ADV {base+2*d+next_d}'],[ret(),f'T 0 {base+d}',ret(),ret(ERR),f'G {next_d} 0',ret(),
f'T 0 {base+2*d+next_d}',ret(),ret(ERR)])
add(f'timer_callback_stop_{soft}_{n}',[f'TICK {base}',f'TIMER 0 {d} {soft} 1 3 0','START 0','DRAIN',
f'ADV {base+d}','GET 0',f'ADV {base+3*d}'],[ret(),f'T 0 {base+d}',ret(),f'G {d} 0'])
add(f'timer_periodic_{soft}',[f'TIMER 0 3 {soft} 1 0 0','START 0','DRAIN','ADV 3','ADV 6','MODE 0 0',
'ADV 9','GET 0','START 0','ADV 12','GET 0','MODE 0 1','START 0','ADV 15','ADV 18','STOP 0','ADV 21'],
[ret(),'T 0 3','T 0 6',ret(),'T 0 9','G 3 0',ret(),'T 0 12','G 3 0',ret(),ret(),'T 0 15','T 0 18',ret()])
add(f'timer_order_{soft}',[f'TIMER 0 5 {soft} 0 0 0',f'TIMER 1 5 {soft} 0 0 0',
f'TIMER 2 5 {soft} 0 0 0','START 2','START 0','START 1','DRAIN','ADV 5'],
[ret(),ret(),ret(),'T 2 5','T 0 5','T 1 5'])
add(f'timer_restart_{soft}',[f'TIMER 0 5 {soft} 0 0 0','START 0','DRAIN','ADV 2',
'START 0','DRAIN','ADV 5','ADV 7'],[ret(),ret(),'T 0 7'])
add(f'timer_callback_set_oneshot_{soft}',[f'TIMER 0 4 {soft} 0 2 7','START 0','DRAIN',
'ADV 4','GET 0','ADV 9','START 0','ADV 16'],[ret(),'T 0 4',ret(),ret(ERR),
'G 7 0',ret(),'T 0 16',ret(),ret(ERR)])
add(f'timer_other_{soft}',[f'TIMER 0 3 {soft} 0 4 1',f'TIMER 1 4 {soft} 0 0 0','START 0','DRAIN',
'ADV 3','ADV 6','ADV 7'],[ret(),'T 0 3',ret(),'T 1 7'])
add(f'timer_zero_{soft}',[f'TIMER 0 0 {soft} 0 0 0','START 0','DRAIN','ADV 0','GET 0'],
[ret(),'T 0 0','G 0 0'])
add(f'timer_wrap_{soft}',['TICK 4294967292',f'TIMER 0 6 {soft} 0 0 0','START 0','DRAIN','ADV 1','ADV 2','GET 0'],
[ret(),'T 0 2','G 6 0'])
return result
def matches(lines, expected):
if not isinstance(expected, dict) or 'race' not in expected:
alternatives=expected["alternatives"] if isinstance(expected,dict) else [expected]
return lines in alternatives
# Only externally observed histories decide. P says whether a notification was seen;
# it establishes neither publication nor how far dispatch has progressed.
model=expected['race']
returns=[]; operations=[]; progress=[]; callbacks=[]
try:
for index,line in enumerate(lines):
fields=line.split()
if fields[0]=='R' and len(fields)==2:
returns.append((index,int(fields[1])))
elif fields[0]=='O' and len(fields)==2:
operations.append((index,int(fields[1])))
elif fields[0]=='P' and len(fields)==2:
progress.append((index,int(fields[1])))
elif fields[0]=='W' and len(fields)==4 and int(fields[1])==0 and int(fields[2])==model['value']:
callbacks.append((index,int(fields[3])))
else:
return False
except (ValueError,IndexError):
return False
if len(returns)!=2 or any(value!=0 for _,value in returns):
return False
if len(operations)!=1 or operations[0][1] not in (0,ERR):
return False
if len(progress)!=1 or progress[0][1] not in (0,1):
return False
op_index,op_result=operations[0]
if not (returns[0][0]==0 < progress[0][0] < op_index < returns[1][0]):
return False
old=[index for index,tick in callbacks if tick==model['old_tick']]
new=[index for index,tick in callbacks if tick==model['new_tick']]
final=[index for index,tick in callbacks if tick==model['final_tick']]
if len(callbacks)!=len(old)+len(new)+len(final) or len(final)!=1:
return False
if final[0]<=returns[1][0] or any(index>=returns[1][0] for index in old+new):
return False
if any(index<=op_index for index in new) or len(old)>1:
return False
if op_result==ERR:
return len(old)==1 and not new
if model['operation']=='SUB':
# Success must establish the requested new deadline. A committed old callback
# can already have happened or finish once while the operations overlap.
return len(new)==1
# A successful cancel cannot be followed by entering the canceled work callback.
return not new and all(index<op_index for index in old)
def main():
tests=cases()
support=json.loads((ROOT/'support.json').read_text())
raw=(ROOT/'observe.c').read_text()
sources={path:(Path(sys.argv[1])/path).read_bytes() for path in CANDIDATES}
scratch=Path(tempfile.mkdtemp(prefix='rt-work-observe-'))
for path,content in support.items():
out=scratch/path;out.parent.mkdir(parents=True,exist_ok=True);out.write_text(content)
for path,content in sources.items():
out=scratch/path;out.parent.mkdir(parents=True,exist_ok=True);out.write_bytes(content)
(scratch/'observe.c').write_text(raw)
for p in scratch.rglob('*'):
os.chmod(p,0o777 if p.is_dir() else 0o666)
# All source and inputs are prepared. The parent never accesses this scratch again after spawning.
compile_argv=['gcc','-D__RT_KERNEL_SOURCE__','-std=c11','-O1','-Iport','-Iinclude',
'-Icomponents/drivers/include','components/drivers/ipc/workqueue.c','src/timer.c',
'port/port.c','observe.c','-o',str(scratch/'observe')]
built=subprocess.run(compile_argv,cwd=scratch,capture_output=True,text=True,timeout=30)
if built.returncode:
print('build failed');return 1
for name,commands,expected in tests:
got=subprocess.run([str(scratch/'observe')],input=commands,capture_output=True,text=True,timeout=3)
lines=got.stdout.splitlines()
if got.returncode or not matches(lines,expected):
print('case',name,'exit',got.returncode,'expected',expected,'observed',lines)
return 1
print('cases',len(tests))
print('PASS');return 0
if __name__=='__main__':
try:
sys.exit(main())
except subprocess.TimeoutExpired:
print('candidate timeout');sys.exit(1)