amos 73a5240295 Batches 21 and 18: add 74 calibrated tasks (40 Bos redo, 34 HWE code-gen)
Batch 21 (20261027-A-bos-redo-21) rebuilt 40 of the 53 rejected Bos-qiao
environments: hidden assertions that ran inside the candidate's process
now run in a trusted parent that reads observations from a low-UID
worker; gold bytes are unchanged. 9 were excluded for scope (ROS2,
oversize, security core), 3 are blocked because the source requires an
exec bit the platform does not freeze, 1 lacks requests/bs4 in trace6.

Batch 18 (20261026-C-hwe-codegen-18) adds its last 34 of 134 tasks; the
first 100 landed in 2f32624d.

Claude reviewed 74/74 USABLE against current digests, canonical
families, no ID or source overlap with main, plain public docs. Only the
new task directories are taken; the stale pre-batch-20 copies of
concur-release-capsule and resource-temp-link on the batch 21 branch and
both branches' progress notes stay as main has them. Difficulty is
unmeasured.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 10:23:12 +08:00

313 lines
16 KiB
Python

import hashlib
import json
import os
from pathlib import Path
import random
import re
import subprocess
import sys
import tempfile
HARNESS = r'''#include "jump.hpp"
#include <array>
#include <cstdint>
#include <iomanip>
#include <iostream>
#include <locale>
#include <memory>
#include <sstream>
#include <string>
using namespace jumpmodel;
void f(const FrameView& v) { std::cout<<v.code<<','<<v.id<<','<<v.local<<','<<v.returned<<','<<v.token; }
void c(const ContextView& v) { std::cout<<v.code<<','<<v.state<<','<<v.owner<<','<<v.token<<','<<v.resumes; }
std::string unhex(const std::string& x) {
if(x=="-")return {};
std::string out;
for(std::size_t i=0;i<x.size();i+=2) out.push_back(static_cast<char>(std::stoi(x.substr(i,2),nullptr,16)));
return out;
}
std::string hex(const std::string& x) {
static const char* digits="0123456789abcdef";
std::string out;
for(unsigned char ch:x) {out.push_back(digits[ch>>4]);out.push_back(digits[ch&15]);}
return out.empty()?"-":out;
}
struct Punct:std::numpunct<char> {
char do_thousands_sep()const override{return '~';}
std::string do_grouping()const override{return "\1";}
};
void snapshot(Machine& m) {
std::cout<<"|"<<m.depth()<<"|";f(m.top());
for(int i=0;i<8;++i) {std::cout<<'|';f(m.frame(i));}
for(int i=0;i<8;++i) {std::cout<<'|';c(m.context(i));}
std::cout<<'\n';
}
int main() {
std::array<std::unique_ptr<Machine>,2> machines;
for(auto& machine:machines)machine.reset(new Machine);
std::string line;
while(std::getline(std::cin,line)) {
std::istringstream row(line);
char kind;int index;row>>kind>>index;
if(kind=='N') {machines[index].reset(new Machine);std::cout<<'N';snapshot(*machines[index]);continue;}
Machine& m=*machines[index];
if(kind=='C') {
std::string data;row>>data;
std::istringstream input(unhex(data));
std::ostringstream output;
output.imbue(std::locale(std::locale::classic(),new Punct));
output<<std::hex<<std::showbase<<std::showpos<<std::uppercase<<std::boolalpha;
output.fill('~');output.width(19);
const CliReply value=run_cli(m,input,output);
std::cout<<"C,"<<value.lines<<','<<value.errors<<','<<static_cast<int>(value.limited)<<','<<hex(output.str());
snapshot(m);continue;
}
std::string op;int a=0,b=0;row>>op>>a>>b;
if(op=="FRAME") {std::cout<<'F'<<',';f(m.frame(a));}
else if(op=="TOP") {std::cout<<'F'<<',';f(m.top());}
else if(op=="CONTEXT") {std::cout<<'C'<<',';c(m.context(a));}
else if(op=="DEPTH") {std::cout<<"D,"<<m.depth();}
else {
Reply value;
if(op=="ENTER")value=m.enter(a,b);
else if(op=="STORE")value=m.store(a,b);
else if(op=="SAVE")value=m.save(a);
else if(op=="JUMP")value=m.jump(a,b);
else if(op=="CLEAR")value=m.clear(a);
else value=m.leave();
std::cout<<"R,"<<value.code<<','<<value.value;
}
snapshot(m);
}
}
'''
HEADER = None
MAIN = None
SMOKE = None
SMOKE_EXPECTED = None
class Model:
def __init__(self):
self.frames=[]
self.slots=[[0,0,-1,0,0] for _ in range(8)]
self.next=1
def invalidate(self):
live={f[4] for f in self.frames}
for s in self.slots:
if s[1]==1 and s[3] not in live:s[1]=2
def frame(self,index):
if not 0<=index<=7:return [1,-1,0,0,0]
return self.frames[index][:] if index<len(self.frames) else [4,-1,0,0,0]
def context(self,slot):
return self.slots[slot][:] if 0<=slot<=7 else [1,0,-1,0,0]
def top(self):return self.frames[-1][:] if self.frames else [4,-1,0,0,0]
def call(self,op,args):
if op=='FRAME':return 'F',self.frame(*args)
if op=='CONTEXT':return 'C',self.context(*args)
if op=='TOP':return 'F',self.top()
if op=='DEPTH':return 'D',[len(self.frames)]
bounds={'ENTER':[(0,31),(-1000,1000)],'STORE':[(0,31),(-1000,1000)],'SAVE':[(0,7)],'JUMP':[(0,7),(-128,128)],'CLEAR':[(0,7)],'LEAVE':[]}
if any(not lo<=x<=hi for x,(lo,hi) in zip(args,bounds[op])):return 'R',[1,-1]
if op=='ENTER':
id,local=args
if any(f[1]==id for f in self.frames):return 'R',[2,-1]
if len(self.frames)==8:return 'R',[3,-1]
self.frames.append([0,id,local,0,self.next]);self.next+=1
return 'R',[0,len(self.frames)]
if op=='LEAVE':
if not self.frames:return 'R',[4,-1]
old=self.frames.pop();self.invalidate();return 'R',[0,old[1]]
if op=='STORE':
id,local=args
for f in self.frames:
if f[1]==id:
old=f[2];f[2]=local;return 'R',[0,old]
return 'R',[5,-1]
if op=='SAVE':
if not self.frames:return 'R',[4,-1]
f=self.frames[-1];self.slots[args[0]]=[0,1,f[1],f[4],0];f[3]=0
return 'R',[0,0]
if op=='CLEAR':
slot=args[0];old=self.slots[slot][1];self.slots[slot]=[0,0,-1,0,0];return 'R',[0,old]
slot,value=args;s=self.slots[slot]
if s[1]==0:return 'R',[6,-1]
if s[1]==2:return 'R',[7,-1]
index=next(i for i,f in enumerate(self.frames) if f[4]==s[3])
self.frames=self.frames[:index+1];self.invalidate();value=value if value else 1
self.frames[-1][3]=value;s[4]+=1
return 'R',[0,value]
def snapshot(self):
items=[[len(self.frames)],self.top()]+[self.frame(i) for i in range(8)]+[self.context(i) for i in range(8)]
return ''.join('|'+','.join(map(str,x)) for x in items)+'\n'
ARITY={'ENTER':2,'STORE':2,'JUMP':2,'SAVE':1,'CLEAR':1,'FRAME':1,'CONTEXT':1,'LEAVE':0,'TOP':0,'DEPTH':0}
DOMAINS={'ENTER':[(0,31),(-1000,1000)],'STORE':[(0,31),(-1000,1000)],'JUMP':[(0,7),(-128,128)],'SAVE':[(0,7)],'CLEAR':[(0,7)],'FRAME':[(0,7)],'CONTEXT':[(0,7)]}
def cli_model(model,text,trace=None):
lines=text.split('\n')
if lines[-1]=='':lines.pop()
count=errors=0;limited=False;out=[]
for line in lines:
if count==64:limited=True;out.append('LIMIT\n');break
count+=1
words=[x for x in re.split('[ \t]+',line) if x]
bad=not words or words[0] not in ARITY or len(words)!=ARITY.get(words[0],-2)+1
values=[]
if not bad:
for word in words[1:]:
if not re.fullmatch('-?[0-9]+',word) or not -2147483648<=int(word)<=2147483647:bad=True;break
values.append(int(word))
if bad:
out.append('SYNTAX\n');errors+=1
if trace is not None:trace.append(('SYNTAX',None,None))
continue
kind,value=model.call(words[0],values)
if words[0]!='DEPTH' and value[0]!=0:errors+=1
out.append(kind+' '+' '.join(map(str,value))+'\n')
if trace is not None:trace.append((words[0],values,value))
return ''.join(out),(count,errors,int(limited))
class Suite:
def __init__(self):
self.models=[Model(),Model()];self.commands=[];self.answers=[]
self.api=[];self.cli=[];self.endpoint_guards=[];self.failure_preservation=0
def new(self,index=0):
self.models[index]=Model();self.commands.append('N '+str(index));self.answers.append('N'+self.models[index].snapshot())
def a(self,op,*args,index=0):
model=self.models[index];before=model.snapshot();kind,value=model.call(op,args)
self.commands.append('A '+str(index)+' '+op+(' '+' '.join(map(str,args)) if args else ''))
self.answers.append(kind+','+','.join(map(str,value))+model.snapshot());self.api.append([op,list(args),value])
if op!='DEPTH' and value[0]!=0:
assert model.snapshot()==before;self.failure_preservation+=1
return value
def c(self,text,index=0):
trace=[];output,result=cli_model(self.models[index],text,trace);self.cli.extend(trace)
self.commands.append('C '+str(index)+' '+(text.encode().hex() or '-'))
self.answers.append('C,'+','.join(map(str,result))+','+(output.encode().hex() or '-')+self.models[index].snapshot())
return trace,result
def setup(self,op,args,via):
self.new()
def run(cmd,*a):
if via=='API':return self.a(cmd,*a)
return self.c(cmd+(' '+' '.join(map(str,a)) if a else '')+'\n')[0][-1][2]
if op=='ENTER':
run('ENTER',17,48);run('SAVE',3)
elif op=='STORE':
run('ENTER',args[0],73);run('SAVE',3);run('ENTER',19 if args[0]!=19 else 20,-21)
elif op=='FRAME':
for id in range(8):run('ENTER',id,100+id)
run('SAVE',3)
elif op in ['SAVE','JUMP','CLEAR','CONTEXT']:
run('ENTER',17,73);run('SAVE',args[0]);run('JUMP',args[0],11);run('ENTER',18,-22)
return run
def build(self):
self.new();self.new(1)
self.c('ENTER 3 40\nSAVE 0\nENTER 9 70\nSAVE 1\nJUMP 0 5\nCONTEXT 1\nTOP\nDEPTH\n')
self.a('JUMP',1,8);self.a('ENTER',9,99);self.a('CONTEXT',1);self.a('SAVE',1);self.a('JUMP',1,0)
self.a('ENTER',3,321,index=1);self.a('SAVE',0,index=1);self.a('JUMP',0,-128,index=1);self.a('TOP');self.a('TOP',index=1)
lifecycle=[('ENTER',(2,82)),('SAVE',(0,)),('SAVE',(7,)),('ENTER',(5,65)),('SAVE',(1,)),('SAVE',(2,)),('JUMP',(1,22)),('ENTER',(9,77)),('SAVE',(3,)),('STORE',(2,0)),('STORE',(5,-1000)),('JUMP',(1,-1)),('CONTEXT',(2,)),('CONTEXT',(3,)),('FRAME',(0,)),('FRAME',(1,)),('LEAVE',()),('JUMP',(2,128)),('ENTER',(5,1000)),('CONTEXT',(1,)),('SAVE',(1,)),('JUMP',(1,0)),('CLEAR',(7,)),('TOP',()),('DEPTH',())]
self.new()
for op,args in lifecycle:self.a(op,*args)
self.new()
for op,args in lifecycle:self.c(op+(' '+' '.join(map(str,args)) if args else '')+'\n')
for via in ['API','CLI']:
for op,domains in DOMAINS.items():
defaults=[(lo+hi)//2 for lo,hi in domains]
for position,(lo,hi) in enumerate(domains):
for value in [lo,hi]:
args=defaults[:];args[position]=value;run=self.setup(op,args,via)
actual=run(op,*args);assert actual[0]==0,(via,op,args,actual)
self.endpoint_guards.append([via,op,position,value,actual])
for value in [lo-1,hi+1,-2147483648,2147483647]:
args=defaults[:];args[position]=value
safe=defaults[:]
run=self.setup(op,safe,via)
actual=run(op,*args);assert actual[0]==1,(via,op,args,actual)
self.new()
for op,args in [('LEAVE',()),('TOP',()),('SAVE',(0,)),('STORE',(0,1)),('FRAME',(0,)),('JUMP',(0,1)),('JUMP',(-1,129))]:self.a(op,*args)
for i in range(8):self.a('ENTER',i,i+30);self.a('SAVE',i)
self.a('ENTER',0,0);self.a('ENTER',8,0);self.a('ENTER',0,1001)
self.a('STORE',0,0);self.a('FRAME',0);self.a('JUMP',0,0)
for i in range(1,8):self.a('CONTEXT',i);self.a('JUMP',i,0)
for i in range(8):self.a('CLEAR',i);self.a('CONTEXT',i);self.a('CLEAR',i)
self.a('LEAVE');self.a('ENTER',0,-1000);self.a('TOP');self.a('SAVE',7);self.a('LEAVE');self.a('JUMP',7,128);self.a('SAVE',7)
self.new();self.c('LEAVE\nTOP\nSAVE 0\nSTORE 0 1\nFRAME 0\nFRAME 0\nJUMP 0 1\n')
self.c(''.join('ENTER %d %d\nSAVE %d\n'%(i,i+30,i) for i in range(8)))
self.c('ENTER 0 4\nENTER 8 4\nENTER 0 1001\nSTORE 0 0\nFRAME 0\nJUMP 0 0\nJUMP 1 1\nCONTEXT 1\nCLEAR 1\nJUMP 1 1\n')
for op,arity in ARITY.items():
self.c(op+'\n' if arity else op+' 1\n')
self.c(op+(' '+' '.join('0' for _ in range(arity+1)))+'\n')
if arity:
for pos in range(arity):
for bad in ['x','+1','-','1x','2147483648','-2147483649','1\r']:
args=['0']*arity;args[pos]=bad;self.c(op+' '+' '.join(args)+'\n')
self.c('\n\t \nunknown 1\nenter 1 1\nDEPTH\r\n');self.c('');self.c('DEPTH');self.c('ENTER 00031 -0\nTOP\n')
for n in [63,64,65,70]:
self.new();prefix=['ENTER 4 19','BAD','JUMP 7 1']+['DEPTH']*60
text='\n'.join(prefix+['STORE 4 0','STORE 4 99','LEAVE']+['DEPTH']*4)
text='\n'.join(text.split('\n')[:n])+'\n'
self.c(text);self.a('FRAME',0);self.c('DEPTH\n');self.c('STORE 4 -1000\nTOP\n')
rng=random.Random(218)
for block in range(14):
self.new(block%2)
for i in range(40):
index=block%2;op=rng.choice(list(ARITY));args=[]
for lo,hi in DOMAINS.get(op,[]):args.append(rng.randint(lo,hi))
if i%3==0:self.c(op+(' '+' '.join(map(str,args)) if args else '')+'\n',index=index)
else:self.a(op,*args,index=index)
seen_api={row[0] for row in self.api};assert seen_api==set(ARITY)
codes={}
for op,args,result in self.cli:
if op=='SYNTAX':continue
codes.setdefault(op,set()).add(0 if op=='DEPTH' else result[0])
required={'ENTER':{0,1,2,3},'STORE':{0,1,5},'JUMP':{0,1,6,7},'SAVE':{0,1,4},'CLEAR':{0,1},'FRAME':{0,1,4},'CONTEXT':{0,1},'LEAVE':{0,4},'TOP':{0,4},'DEPTH':{0}}
assert codes==required,(codes,required)
assert len(self.endpoint_guards)==sum(len(d)*4 for d in DOMAINS.values())
return ('\n'.join(self.commands)+'\n').encode(),''.join(self.answers).encode(),{'api':len(self.api),'cli_calls':sum(1 for x in self.commands if x.startswith('C ')),'cli_results':len(self.cli),'endpoint_guards':self.endpoint_guards,'cli_code_sets':{k:sorted(v) for k,v in codes.items()},'failed_api_state_checks':self.failure_preservation}
def digest(data):return [len(data),hashlib.sha256(data).hexdigest()[:16]]
def main():
global HEADER,MAIN,SMOKE,SMOKE_EXPECTED
source=Path(sys.argv[1]).resolve();own=Path(__file__).resolve().parent
# Complete trusted file IO before the first compiler; no scratch reads follow.
candidates={name:(source/name).read_bytes() for name in ['jump.hpp','engine.cpp','console.cpp']}
MAIN=(own/'app-main.dat').read_bytes()
SMOKE=(own/'smoke.dat').read_bytes();SMOKE_EXPECTED=(own/'smoke-expected.dat').read_bytes()
raw_input,raw_expected,coverage=Suite().build()
limit_text=('ENTER 4 19\nBAD\nJUMP 7 1\n'+'DEPTH\n'*60+'STORE 4 0\nSTORE 4 99\nLEAVE\n').encode()
limit_expected=cli_model(Model(),limit_text.decode())[0].encode()
scratch=Path(tempfile.mkdtemp(prefix='jump-',dir=Path.cwd()))
for name,data in candidates.items():(scratch/name).write_bytes(data)
(scratch/'observer.cpp').write_text(HARNESS);(scratch/'main.cpp').write_bytes(MAIN)
env=os.environ.copy();env['JUMP_GRADER_HINT']=str(Path(__file__).resolve())
records=[];failures=[]
def process(title,argv,data,expected):
proc=subprocess.Popen(argv,cwd=str(scratch),env=env,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
timed=False
try:out,err=proc.communicate(data,timeout=20)
except subprocess.TimeoutExpired:
timed=True;proc.kill();out,err=proc.communicate()
reasons=[]
if timed:reasons.append('timeout')
if proc.returncode!=0:reasons.append('exit')
if expected is not None and out!=expected:reasons.append('stdout')
if err and expected is not None:reasons.append('stderr')
row=[title,proc.pid,argv,str(scratch),proc.returncode,timed,digest(data),digest(expected or b''),digest(out),digest(err),reasons]
records.append(row);failures.extend(title+':'+r for r in reasons)
return proc.returncode==0 and not timed
built=process('build-observer',['g++','-std=c++17','-O2','-Wall','-Wextra','-pedantic','-I.','engine.cpp','console.cpp','observer.cpp','-o','observer'],b'',None)
app=process('build-app',['g++','-std=c++17','-O2','-Wall','-Wextra','-pedantic','-I.','engine.cpp','console.cpp','main.cpp','-o','jump-demo'],b'',None)
if built:process('raw',['./observer'],raw_input,raw_expected)
if app:
process('smoke',['./jump-demo'],SMOKE,SMOKE_EXPECTED)
process('empty',['./jump-demo'],b'',b'')
process('limit',['./jump-demo'],limit_text,limit_expected)
meta={'schema':['name','pid','argv','cwd','exit','timeout','stdin_len_sha16','expected_len_sha16','actual_len_sha16','stderr_len_sha16','reasons'],'processes':records,'failures':failures}
payload=json.dumps(meta,separators=(',',':'))
assert len(payload.encode())<3900,len(payload)
sys.stderr.write(payload+'\n')
print('FAIL' if failures else 'PASS')
return 1 if failures else 0
if __name__=='__main__':raise SystemExit(main())