amos 73a5240295 Batches 21 and 18: add 74 calibrated tasks (40 Bos redo, 34 HWE code-gen)
Batch 21 (20261027-A-bos-redo-21) rebuilt 40 of the 53 rejected Bos-qiao
environments: hidden assertions that ran inside the candidate's process
now run in a trusted parent that reads observations from a low-UID
worker; gold bytes are unchanged. 9 were excluded for scope (ROS2,
oversize, security core), 3 are blocked because the source requires an
exec bit the platform does not freeze, 1 lacks requests/bs4 in trace6.

Batch 18 (20261026-C-hwe-codegen-18) adds its last 34 of 134 tasks; the
first 100 landed in 2f32624d.

Claude reviewed 74/74 USABLE against current digests, canonical
families, no ID or source overlap with main, plain public docs. Only the
new task directories are taken; the stale pre-batch-20 copies of
concur-release-capsule and resource-temp-link on the batch 21 branch and
both branches' progress notes stay as main has them. Difficulty is
unmeasured.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 10:23:12 +08:00

282 lines
14 KiB
Python

"""Independent memory model. Observer prints raw API returns; expectations stay here."""
import copy
import hashlib
import json
import os
from pathlib import Path
import random
import re
import subprocess
import sys
import tempfile
NAMES = ['OK','BAD_ID','BAD_CAPACITY','BAD_BYTE','BAD_COUNT','BAD_STAGE','BAD_ENABLED','EXISTS','MISSING','WRITE_ERROR','FLUSH_ERROR','CLOSE_ERROR']
LO, HI = -(1 << 63), (1 << 63)-1
VERBS = {'OPEN':2,'PUT':3,'ARM':3,'FLUSH':1,'CLOSE':1,'VIEW':1,'STATS':0}
COVER = {'api':{}, 'cli':{}, 'api_endpoints':set(), 'cli_endpoints':set()}
ENDPOINTS = {'OPEN':[(0,3),(1,8)],'PUT':[(0,3),(0,255),(0,16)],'ARM':[(0,3),(1,3),(0,1)],'FLUSH':[(0,3)],'CLOSE':[(0,3)],'VIEW':[(0,3)],'STATS':[]}
def new_slot(i):
return dict(code=8,id=i,seen=0,live=0,capacity=0,pending=b'',durable=b'',history=b'',last=0,faults=[0,0,0],writes=0,flushes=0,closes=0)
class Model:
def __init__(self):
self.slots = [new_slot(i) for i in range(4)]
self.total = [0]*6
def drain(self,v):
while v['pending']:
v['writes'] += 1
v['history'] += b'W'
self.total[3] += 1
if v['faults'][0]:
return 9
amount = min(2,len(v['pending']))
v['durable'] += v['pending'][:amount]
v['pending'] = v['pending'][amount:]
return 0
def flush(self,v):
result = self.drain(v)
if result == 0:
v['flushes'] += 1
v['history'] += b'F'
self.total[4] += 1
result = 10 if v['faults'][1] else 0
v['last'] = result
return result
def call(self,op,args,kind):
before = copy.deepcopy((self.slots,self.total))
result = self._call(op,args)
code = result['code'] if op == 'VIEW' else 0 if op == 'STATS' else result
COVER[kind].setdefault(op,set()).add(code)
if code == 0:
for j,(low,high) in enumerate(ENDPOINTS[op]):
if args[j] in (low,high):
COVER[kind+'_endpoints'].add((op,j,args[j]))
if code in range(1,9):
assert before == (self.slots,self.total), ('model rejection changed state',op,args)
return result
def _call(self,op,a):
if op == 'STATS':
return list(self.total)
i=a[0]
if i < 0 or i > 3:
if op == 'VIEW':
v=new_slot(i);v['code']=1;return v
return 1
if op == 'OPEN' and not 1 <= a[1] <= 8: return 2
if op == 'PUT' and not 0 <= a[1] <= 255: return 3
if op == 'PUT' and not 0 <= a[2] <= 16: return 4
if op == 'ARM' and not 1 <= a[1] <= 3: return 5
if op == 'ARM' and not 0 <= a[2] <= 1: return 6
v=self.slots[i]
if op == 'VIEW': return copy.deepcopy(v)
if op == 'OPEN':
if v['live']: return 7
v=new_slot(i);v.update(code=0,seen=1,live=1,capacity=a[1]);self.slots[i]=v
self.total[0]+=1;self.total[1]+=1
return 0
if not v['live']: return 8
if op == 'ARM':
v['faults'][a[1]-1]=a[2];return 0
if op == 'PUT':
for _ in range(a[2]):
if len(v['pending']) == v['capacity']:
code=self.drain(v)
if code:
v['last']=code;return code
v['pending']+=bytes([a[1]])
v['last']=0;return 0
code=self.flush(v)
if op == 'CLOSE':
v['closes']+=1;self.total[5]+=1;v['history']+=b'C'
if code == 0 and v['faults'][2]: code=11
v['last']=code;v['pending']=b'';v['live']=0
self.total[1]-=1;self.total[2]+=1
return code
def frame(b): return str(len(b))+':'+b.hex()
def raw_view(v):
parts=['V',v['code'],v['id'],v['seen'],v['live'],v['capacity'],frame(v['pending']),frame(v['durable']),frame(v['history']),v['last'],*v['faults'],v['writes'],v['flushes'],v['closes']]
return ' '.join(map(str,parts))+'\n'
def raw_stats(t): return 'S '+' '.join(map(str,t))+'\n'
def raw_expected(commands):
sessions=[Model(),Model()];out=[]
for sid,op,args in commands:
result=sessions[sid].call(op,args,'api')
out.append(raw_view(result) if op=='VIEW' else raw_stats(result) if op=='STATS' else 'R '+str(result)+'\n')
for m in sessions:
out.extend(raw_view(m.call('VIEW',[i],'api')) for i in range(4))
out.append(raw_stats(m.call('STATS',[],'api')))
return ''.join(out).encode()
def cli_view(v):
if v['code']: return NAMES[v['code']]
parts=['VIEW',v['id'],v['seen'],v['live'],v['capacity'],v['pending'].hex() or '-',v['durable'].hex() or '-',v['history'].hex() or '-',NAMES[v['last']],*v['faults'],v['writes'],v['flushes'],v['closes']]
return ' '.join(map(str,parts))
def cli_expected(payload,m=None):
m=Model() if m is None else m
out=[];count=0
lines=payload.split(b'\n')
if lines[-1:]==[b'']: lines.pop()
for line in lines:
if count == 64:
out.append('LIMIT');break
count+=1
words=re.split(b'[ \t]+',line.strip(b' \t'))
verb=words[0].decode('ascii','replace')
args=[];valid=verb in VERBS and len(words)==VERBS.get(verb,-1)+1
if valid:
for token in words[1:]:
if not re.fullmatch(b'-?[0-9]+',token): valid=False;break
value=int(token)
if not LO<=value<=HI: valid=False;break
args.append(value)
if not valid:
out.append('SYNTAX');continue
r=m.call(verb,args,'cli')
out.append(cli_view(r) if verb=='VIEW' else 'STATS '+' '.join(map(str,r)) if verb=='STATS' else NAMES[r])
return count,(''.join(x+'\n' for x in out)).encode()
def base_background():
return [('OPEN',[0,4]),('PUT',[0,73,7]),('OPEN',[2,3]),('PUT',[2,0,2]),('ARM',[0,1,1]),('FLUSH',[0]),('ARM',[0,1,0])]
def make_sequences():
sequences=[]
# Each endpoint succeeds in its own nonzero background and is observed after mutation.
for verb,domains in ENDPOINTS.items():
if verb=='STATS': continue
for j,(low,high) in enumerate(domains):
for value in (low,high):
target=3 if j==0 and value==3 else 0
bg=[('OPEN',[2,3]),('PUT',[2,91,5])]
if verb!='OPEN': bg += [('OPEN',[target,4]),('PUT',[target,42,3])]
args={'OPEN':[target,4],'PUT':[target,65,5],'ARM':[target,2,0],'FLUSH':[target],'CLOSE':[target],'VIEW':[target]}[verb]
args[j]=value
sequences.append(bg+[(verb,args),('VIEW',[target]),('STATS',[])])
# Invalid numeric domains independently, with full observation after every attempted call.
for verb,domains in ENDPOINTS.items():
if verb=='STATS': continue
for j,(low,high) in enumerate(domains):
for bad in (low-1,high+1,LO,HI):
args={'OPEN':[1,4],'PUT':[0,65,2],'ARM':[0,2,0],'FLUSH':[0],'CLOSE':[0],'VIEW':[0]}[verb]
args[j]=bad
sequences.append(base_background()+[(verb,args),('VIEW',[0]),('VIEW',[2]),('STATS',[])])
# Validation order with multiple simultaneous errors, including a live ID.
sequences.append(base_background()+[('OPEN',[-1,0]),('OPEN',[0,0]),('PUT',[-1,-1,-1]),('PUT',[0,-1,-1]),('ARM',[-1,0,-1]),('ARM',[0,0,-1]),('VIEW',[0]),('STATS',[])])
# Independent fault paths, retry/clear, zero puts, retained and reopened records.
for stage in (1,2,3):
for extra in (0,1):
sequences.append(base_background()+[('ARM',[0,stage,1]),('ARM',[0,3,extra]),('FLUSH',[0]),('PUT',[0,255,0]),('PUT',[0,7,8]),('CLOSE',[0]),('CLOSE',[0]),('PUT',[0,1,1]),('FLUSH',[0]),('ARM',[0,1,0]),('OPEN',[0,8]),('PUT',[0,0,16]),('CLOSE',[0])])
sequences += [base_background()+[('OPEN',[0,4]),('OPEN',[1,2]),('PUT',[1,0,16]),('ARM',[0,1,1]),('FLUSH',[0]),('ARM',[0,1,0]),('FLUSH',[0]),('CLOSE',[0]),('VIEW',[0])],
[('VIEW',[0]),('CLOSE',[0]),('FLUSH',[3]),('ARM',[1,1,1]),('PUT',[3,0,0]),('STATS',[])]]
# Every CLI verb's reachable errors include failures with nonzero background.
sequences += [base_background()+[('PUT',[1,0,1]),('ARM',[1,1,0]),('CLOSE',[1]),('FLUSH',[1]),('VIEW',[1]),('ARM',[0,1,1]),('PUT',[0,65,16]),('CLOSE',[0])],
base_background()+[('ARM',[0,2,1]),('CLOSE',[0])],
base_background()+[('ARM',[0,3,1]),('CLOSE',[0])]]
rng=random.Random(116125)
for _ in range(12):
seq=base_background()
for n in range(26):
verb=rng.choice(['OPEN','PUT','ARM','FLUSH','CLOSE','VIEW','STATS'])
id=rng.randrange(4)
args={'OPEN':[id,rng.randrange(1,9)],'PUT':[id,rng.randrange(256),rng.randrange(17)],'ARM':[id,rng.randrange(1,4),rng.randrange(2)],'FLUSH':[id],'CLOSE':[id],'VIEW':[id],'STATS':[]}[verb]
seq.append((verb,args))
sequences.append(seq)
return sequences
def line(op,args): return op+' '+ ' '.join(map(str,args)) if args else op
def cases():
sequences=make_sequences()
commands=[]
for seq in sequences:
# Reopen/close existing slots between sequences; session 1 stays live and different.
commands.extend((0,'CLOSE',[i]) for i in range(4))
commands.extend((0,op,args) for op,args in seq)
commands=[(1,'OPEN',[0,8]),(1,'PUT',[0,255,13])]+commands+[(1,'VIEW',[0]),(1,'CLOSE',[0]),(0,'STATS',[])]
raw=''.join(str(s)+' '+line(op,a)+'\n' for s,op,a in commands).encode()
raw_out=raw_expected(commands)
payloads=[b'',b'STATS',b'OPEN 0 4\nPUT 0 0 1\nSTATS\n',b'STATS\n']
payloads.extend(('\n'.join(line(op,a) for op,a in seq)+'\n').encode() for seq in sequences)
# Wrong arity and numeric lexing for every verb/parameter, surrounded by visible state.
malformed=[]
good={'OPEN':['1','4'],'PUT':['0','65','2'],'ARM':['0','2','0'],'FLUSH':['0'],'CLOSE':['0'],'VIEW':['0'],'STATS':[]}
for verb,params in good.items():
malformed.extend([verb+' '+' '.join(params[:-1]),verb+' '+' '.join(params+['0'])])
for j in range(len(params)):
for bad in ('x','+1','-','1x','9223372036854775808','-9223372036854775809','99999999999999999999999999999999','-99999999999999999999999999999999','1\r','1\x00'):
a=list(params);a[j]=bad;malformed.append(verb+' '+' '.join(a))
malformed += ['', 'unknown', 'stats', '\t', 'PUT 0 65 2\r']
for bad in malformed:
payloads.append(('OPEN 0 4\nPUT 0 73 3\nOPEN 2 2\nPUT 2 90 5\nARM 0 1 1\nARM 0 2 1\nARM 0 3 1\nFLUSH 0\n'+bad+'\nVIEW 0\nVIEW 2\nSTATS\n').encode())
payloads.append(b' \tOPEN\t000 04 \nPUT 0 -0 01\nVIEW 0\nSTATS')
# Limit prefix has accepted/error/syntax lines; line64 mutates and line65 close is suppressed.
mixed=b'OPEN 0 8\nPUT 0 65 3\nOPEN 0 1\nbad\n'+b'STATS\n'*58+b'VIEW 0\nPUT 0 255 2\nCLOSE 0\nVIEW 0\n'
assert len(mixed.splitlines())==66
payloads += [mixed,b'STATS\n',b'OPEN 0 8\n'+b'PUT 0 1 1\n'*63,b'STATS\n']
# Observe Nth mutation through stream output using VIEW as line64 after line63 mutation.
# Separate direct observer scene below also exposes final state after run_cli via a documented VIEW line.
payloads.append(b'OPEN 0 8\n'+b'STATS\n'*61+b'PUT 0 255 2\nVIEW 0\nCLOSE 0\nSTATS\n')
cli_in=b'';cli_out=b''
entries=[(p,True) for p in payloads]
entries += [(mixed,True),(b'VIEW 0\nSTATS\nPUT 0 0 0\n',False),(b'STATS\n'*64+b'CLOSE 0\n',False),(b'VIEW 0\nSTATS',False)]
model=None
for p,reset in entries:
if reset: model=Model()
n,out=cli_expected(p,model)
cli_in+=(str(len(p))+' '+str(int(reset))+'\n').encode()+p
cli_out+=('C '+str(n)+' '+frame(out)+'\n').encode()
cli_out+=''.join(raw_view(model._call('VIEW',[i])) for i in range(4)).encode()
cli_out+=raw_stats(model._call('STATS',[])).encode()
smoke=b'OPEN 0 4\nPUT 0 65 6\nCLOSE 0\nVIEW 0\nSTATS\n'
result=[('raw',['./observe'],raw,raw_out),('cli',['./observe','cli'],cli_in,cli_out)]
for label,p in [('smoke',smoke),('empty',b''),('limit',mixed)]:
result.append((label,['./app'],p,cli_expected(p)[1]))
# Reachable Code sets are independently checked on actual model CLI paths.
wanted={'OPEN':{0,1,2,7},'PUT':{0,1,3,4,8,9},'ARM':{0,1,5,6,8},'FLUSH':{0,1,8,9,10},'CLOSE':{0,1,8,9,10,11},'VIEW':{0,1,8},'STATS':{0}}
for kind in ('api','cli'):
assert COVER[kind] == wanted,(kind,COVER[kind])
required={(verb,j,v) for verb,ds in ENDPOINTS.items() for j,(low,high) in enumerate(ds) for v in (low,high)}
assert required <= COVER[kind+'_endpoints'],(kind,required-COVER[kind+'_endpoints'])
return result
def digest(data): return [len(data),hashlib.sha256(data).hexdigest()[:16]]
def check(candidate):
scenes=cases()
# All trusted IO and candidate reads finish before the first compiler is spawned.
scratch=Path(tempfile.mkdtemp(prefix='io-',dir=Path.cwd()))
for name in ('io.hpp','backend.cpp','stream.cpp','cli.cpp','main.cpp'):
(scratch/name).write_bytes((candidate/name).read_bytes())
(scratch/'observer.cc').write_bytes((Path(__file__).parent/'observer.cc').read_bytes())
common=['c++','-std=c++17','-O0','backend.cpp','stream.cpp','cli.cpp']
builds=[('build_observer',common+['observer.cc','-o','observe']),('build_app',common+['main.cpp','-o','app'])]
rows=[];failed=False
def run(label,argv,data,expected,compile=False):
nonlocal failed
process=subprocess.Popen(argv,cwd=scratch,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
timeout=False
try:
out,err=process.communicate(data,timeout=20 if compile else 12)
except subprocess.TimeoutExpired:
timeout=True;process.kill();out,err=process.communicate()
reasons=[]
if process.returncode: reasons.append('exit')
if timeout: reasons.append('timeout')
if not compile and out!=expected: reasons.append('stdout')
if not compile and err: reasons.append('stderr')
failed |= bool(reasons)
rows.append([label,process.pid,argv,process.returncode,timeout,digest(data),digest(expected),digest(out),digest(err),reasons])
for label,argv in builds: run(label,argv,b'',b'',True)
if not failed:
for label,argv,data,expected in scenes: run(label,argv,data,expected)
meta={'cwd':str(scratch),'columns':['scene','pid','argv','exit','timeout','stdin','expected','actual','stderr','reasons'],'rows':rows}
encoded=json.dumps(meta,separators=(',',':'))
assert len(encoded.encode()) < 3900,len(encoded)
sys.stderr.write(encoded+'\n')
print('FAIL behavior' if failed else 'PASS')
return 1 if failed else 0
if __name__=='__main__':
raise SystemExit(check(Path(sys.argv[1])))