amos 73a5240295 Batches 21 and 18: add 74 calibrated tasks (40 Bos redo, 34 HWE code-gen)
Batch 21 (20261027-A-bos-redo-21) rebuilt 40 of the 53 rejected Bos-qiao
environments: hidden assertions that ran inside the candidate's process
now run in a trusted parent that reads observations from a low-UID
worker; gold bytes are unchanged. 9 were excluded for scope (ROS2,
oversize, security core), 3 are blocked because the source requires an
exec bit the platform does not freeze, 1 lacks requests/bs4 in trace6.

Batch 18 (20261026-C-hwe-codegen-18) adds its last 34 of 134 tasks; the
first 100 landed in 2f32624d.

Claude reviewed 74/74 USABLE against current digests, canonical
families, no ID or source overlap with main, plain public docs. Only the
new task directories are taken; the stale pre-batch-20 copies of
concur-release-capsule and resource-temp-link on the batch 21 branch and
both branches' progress notes stay as main has them. Difficulty is
unmeasured.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 10:23:12 +08:00

340 lines
20 KiB
Python

"""Private independent transition model; candidate code is only a child process."""
import hashlib
import json
from pathlib import Path
import random
import re
import subprocess
import sys
import tempfile
OBSERVER = '#include "runtime.hpp"\n#include <iostream>\n#include <sstream>\n#include <string>\n#include <vector>\n#include <iomanip>\nusing namespace cancelrt;\nstatic void raw(const Reply& r) {\n std::cout << static_cast<int>(r.code) << \' \' << static_cast<int>(r.stop)\n << \' \' << r.total << \' \' << r.done << \' \' << r.produced << \' \' << r.factor\n << \' \' << r.bias << \' \' << r.attempts << \' \' << r.releases << \' \' << r.active\n << \' \' << r.sum << \' \' << r.values.size();\n for (auto v:r.values) std::cout << \' \' << v;\n std::cout << \' \' << r.ids.size();\n for (auto v:r.ids) std::cout << \' \' << v;\n std::cout << \'\\n\';\n}\nstatic std::string decode(const std::string& s) {\n if (s=="-") return {};\n std::string out;\n for (std::size_t i=0;i<s.size();i+=2) out.push_back(static_cast<char>(std::stoi(s.substr(i,2),nullptr,16)));\n return out;\n}\nstatic void bytes(const std::string& s) {\n std::cout << \' \' << s.size() << \' \';\n if(s.empty()) { std::cout << \'-\'; return; }\n const char* digits="0123456789abcdef";\n for(unsigned char c:s) std::cout << digits[c>>4] << digits[c&15];\n}\nint main() {\n Runtime runtimes[2];\n std::string op;\n while (std::cin >> op) {\n if(op=="D") { raw(Reply{}); } else if(op=="C") {\n std::string text; std::cin >> text;\n std::istringstream in(decode(text)); std::ostringstream out,err;\n out<<std::hex<<std::showpos<<std::showbase<<std::setfill(\'~\')<<std::setw(23);\n err<<std::hex<<std::showpos<<std::showbase<<std::setfill(\'!\')<<std::setw(19);\n int result=run_cli(in,out,err);\n std::cout << "C " << result; bytes(out.str()); bytes(err.str()); std::cout << \'\\n\';\n } else if(op=="P") {\n Reply r; int code,stop; std::size_t n;\n std::cin >> code >> stop >> r.total >> r.done >> r.produced >> r.factor\n >> r.bias >> r.attempts >> r.releases >> r.active >> r.sum >> n;\n r.code=static_cast<Code>(code); r.stop=static_cast<Stop>(stop);\n for(std::size_t i=0;i<n;++i) { std::int64_t v;std::cin>>v;r.values.push_back(v); }\n std::cin>>n;\n for(std::size_t i=0;i<n;++i) { int v;std::cin>>v;r.ids.push_back(v); }\n std::ostringstream out;out<<std::hex<<std::showpos<<std::showbase<<std::setfill(\'~\')<<std::setw(23);print_reply(out,r);std::cout<<"P";bytes(out.str());std::cout<<\'\\n\';\n } else {\n int instance;std::string verb;std::cin>>instance>>verb;\n std::int64_t a=0,b=0,c=0,d=0;Reply r;\n Runtime& rt=runtimes[instance];\n if(verb=="create") { std::cin>>a>>b>>c;r=rt.create(a,b,c); }\n else if(verb=="configure") {std::cin>>a>>b>>c;r=rt.configure(a,b,c);}\n else if(verb=="write") {std::cin>>a>>b>>c>>d;r=rt.write(a,b,c,d);}\n else if(verb=="read") {std::cin>>a>>b>>c>>d;r=rt.read(a,b,c,d);}\n else if(verb=="inspect") {std::cin>>a;r=rt.inspect(a);}\n else if(verb=="run") {std::cin>>a>>b>>c;r=rt.run(a,b,c);}\n else if(verb=="rewind") {std::cin>>a;r=rt.rewind(a);}\n else if(verb=="erase") {std::cin>>a;r=rt.erase(a);}\n else if(verb=="list") {r=rt.list();}\n raw(r);\n }\n }\n}\n'
DOMAINS = {
"create": [(1,31),(0,32),(-1000,1000)],
"configure": [(1,31),(-8,8),(-1000,1000)],
"write": [(1,31),(0,32),(0,32),(-1000,1000)],
"read": [(1,31),(0,32),(0,32),(0,1)],
"inspect": [(1,31)], "run": [(1,31),(0,32),(-1,32)],
"rewind": [(1,31)], "erase": [(1,31)], "list": [],
}
NATIVE = [-(1<<63),(1<<63)-1]
def empty(code=0):
return [code,0,0,0,0,0,0,0,0,0,0,[],[]]
def line(r):
return (" ".join(map(str,r[:11]+[len(r[11])]+r[11]+[len(r[12])]+r[12]))+"\n").encode()
class Model:
def __init__(self):
self.jobs = {}
def call(self,verb,args):
for value,(lo,hi) in zip(args,DOMAINS[verb]):
if not lo <= value <= hi:
return empty(1)
if verb == "list":
r=empty();r[12]=sorted(self.jobs);return r
ident=args[0]
if verb == "create":
if ident in self.jobs:
return empty(2)
_,total,seed=args
self.jobs[ident] = {"src":[seed+i for i in range(total)],"out":[],"factor":1,"bias":0,"attempts":0,"releases":0}
elif ident not in self.jobs:
return empty(3)
if verb == "erase":
del self.jobs[ident];return empty()
j=self.jobs[ident]
if verb in ("write","read"):
_,offset,count,_=args
data=j["src"] if verb=="write" or args[3]==0 else j["out"]
if offset+count > len(data):
return empty(1)
if verb in ("configure","write") and j["out"]:
return empty(4)
produced=0;stop=0;values=[]
if verb=="configure":
j["factor"],j["bias"]=args[1:]
elif verb=="write":
j["src"][offset:offset+count]=[args[3]]*count
elif verb=="read":
values=data[offset:offset+count]
elif verb=="rewind":
j["out"]=[];j["attempts"]=0;j["releases"]=0
elif verb=="run":
_,budget,cancel=args
remaining=len(j["src"])-len(j["out"])
if not remaining:
stop=1
else:
j["attempts"]+=1
produced=min(remaining,budget)
if cancel>=0:
produced=min(produced,cancel)
stop=1 if produced==remaining else (3 if cancel>=0 and cancel<budget and cancel<remaining else 2)
start=len(j["out"])
j["out"].extend(j["src"][i]*j["factor"]+j["bias"]+i for i in range(start,start+produced))
j["releases"]+=1
return [0,stop,len(j["src"]),len(j["out"]),produced,j["factor"],j["bias"],j["attempts"],j["releases"],0,sum(j["out"]),values,[]]
class Cases:
def __init__(self):
self.models=[Model(),Model()]
self.commands=[];self.expected=[]
self.api_codes={v:set() for v in DOMAINS};self.cli_codes={v:set() for v in DOMAINS}
self.api_success=set();self.cli_success=set()
self.observations=0;self.cli_calls=0;self.cli_lines=0
def api(self,verb,args=(),instance=0,observe=True):
r=self.models[instance].call(verb,list(args))
self.commands.append("O "+str(instance)+" "+verb+(" "+" ".join(map(str,args)) if args else ""))
self.expected.append(line(r));self.observations+=1
self.api_codes[verb].add(r[0])
if r[0]==0:
self.api_success.update((verb,i,a) for i,a in enumerate(args))
if observe and r[0]:
self.all_state()
return r
def all_state(self):
for n,m in enumerate(self.models):
self.api("list",instance=n,observe=False)
for ident,j in sorted(m.jobs.items()):
self.api("inspect",[ident],n,False)
self.api("read",[ident,0,len(j["src"]),0],n,False)
self.api("read",[ident,0,len(j["out"]),1],n,False)
def clear(self):
for n,m in enumerate(self.models):
for ident in list(m.jobs):
self.api("erase",[ident],n)
def cli(self,data):
self.cli_calls+=1
expected=self.cli_model(data)
self.commands.append("C "+(data.hex() or "-"))
self.expected.append(b"C 0 "+str(len(expected)).encode()+b" "+(expected.hex().encode() or b"-")+b" 0 -\n")
return expected
def cli_model(self,data):
model=Model();result=[]
physical=data.split(b"\n")
if physical[-1:]==[b""]:
physical.pop()
for index,raw in enumerate(physical):
if index==4096:
result.append(b"LIMIT\n");break
self.cli_lines+=1
words=re.split(rb"[ \t\r]+",raw.strip(b" \t\r"))
verb=words[0].decode("ascii", "replace")
if verb not in DOMAINS or len(words)!=len(DOMAINS[verb])+1:
result.append(b"SYNTAX\n");continue
args=[]
for token in words[1:]:
if not re.fullmatch(rb"[+-]?[0-9]+",token):
break
value=int(token)
if not NATIVE[0]<=value<=NATIVE[1]:
break
args.append(value)
if len(args)!=len(DOMAINS[verb]):
result.append(b"SYNTAX\n");continue
r=model.call(verb,args)
self.cli_codes[verb].add(r[0])
if r[0]==0:
self.cli_success.update((verb,i,a) for i,a in enumerate(args))
result.append(line(r))
return b"".join(result)
def formatter(self):
self.commands.append("D");self.expected.append(line(empty()))
vectors=[([],[]),([0,-1,1,NATIVE[0],NATIVE[1]],[31,1,0,-1])]
for code in [0,1,2,3,4,-17,99,-2147483648,2147483647]:
for stop in [0,1,2,3,-9,76]:
for vals,ids in vectors:
r=[code,stop,31,17,9,-8,-1000,21,19,1,-1234567890123,vals,ids]
encoded=line(r)
self.commands.append("P "+encoded.decode().strip())
self.expected.append(b"P "+str(len(encoded)).encode()+b" "+encoded.hex().encode()+b"\n")
def prepare(c,verb,args,cli=False):
ident=args[0] if args else 1
if cli:
seq=[]
def emit(v,a=()): seq.append(v+(" "+" ".join(map(str,a)) if a else ""))
else:
c.clear()
def emit(v,a=()): c.api(v,a)
if verb != "create":
emit("create",[ident,32,19])
if (verb=="read" and args[3]==1) or verb=="rewind":
emit("run",[ident,32,-1])
emit(verb,args)
if verb=="configure":
emit("run",[ident,32,-1])
if verb!="erase":
emit("inspect",[ident])
emit("read",[ident,0,32,0])
if verb in ("read","run"):
emit("read",[ident,0,32,1])
emit("list")
if cli:
return ("\n".join(seq)+"\n").encode()
def generate():
c=Cases()
defaults={"create":[1,32,19],"configure":[1,2,17],"write":[1,0,1,27],"read":[1,0,0,0],"inspect":[1],"run":[1,32,-1],"rewind":[1],"erase":[1],"list":[]}
for verb,domains in DOMAINS.items():
prepare(c,verb,defaults[verb])
c.cli(prepare(c,verb,defaults[verb],True))
for index,(lo,hi) in enumerate(domains):
for value in (lo,hi):
args=defaults[verb].copy();args[index]=value
if verb in ("read","write"):
if index==1 and value==32: args[2]=0
if index==2 and value==32: args[1]=0
prepare(c,verb,args)
c.cli(prepare(c,verb,args,True))
# Keep nonzero data and two objects live during every invalid call.
c.clear()
c.api("create",[1,32,57]);c.api("run",[1,2,-1]);c.api("create",[31,32,-71])
c.api("create",[1,32,99],1);c.api("configure",[1,-3,7],1);c.api("run",[1,5,-1],1)
for verb,domains in DOMAINS.items():
for index,(lo,hi) in enumerate(domains):
for value in [lo-1,hi+1,*NATIVE]:
args=defaults[verb].copy();args[index]=value
c.api(verb,args)
prefix="create 1 32 57\nrun 1 2 -1\ncreate 31 32 -71\n"
suffix="\ninspect 1\nread 1 0 32 0\nread 1 0 2 1\ninspect 31\nread 31 0 32 0\nlist\n"
c.cli((prefix+verb+" "+" ".join(map(str,args))+suffix).encode())
# All reachable model codes, including error priority and payload/state preservation.
error_paths=[("create",[1,32,12]),("create",[1,-1,12]),("configure",[2,3,9]),("configure",[1,3,9]),("configure",[1,9,9]),("write",[2,0,1,9]),("write",[1,31,2,9]),("write",[1,0,1,9]),("write",[1,32,0,0]),("read",[2,0,1,0]),("read",[1,32,1,0]),("read",[1,0,3,1]),("inspect",[2]),("run",[2,32,-1]),("rewind",[2]),("erase",[2])]
prefix=b"create 1 32 57\nrun 1 2 -1\ncreate 31 32 -71\n"
suffix=b"\ninspect 1\nread 1 0 32 0\nread 1 0 2 1\ninspect 31\nread 31 0 32 0\nlist\n"
for verb,args in error_paths:
c.api(verb,args)
c.cli(prefix+(verb+" "+" ".join(map(str,args))).encode()+suffix)
# Cancellation before a step, after prefixes, and at budget/completion boundaries.
for total in [0,1,2,7,32]:
for budget in [0,1,2,7,32]:
for cancel in [-1,0,1,2,7,32]:
c.clear();c.api("create",[7,total,31]);c.api("configure",[7,-3,9])
c.api("run",[7,budget,cancel]);c.all_state()
c.api("run",[7,32,-1]);c.all_state()
c.api("run",[7,1,0]);c.all_state()
c.api("rewind",[7]);c.api("write",[7,0,total,0]);c.api("read",[7,0,total,0]);c.api("run",[7,32,-1]);c.all_state()
c.cli((f"create 7 {total} 31\nconfigure 7 -3 9\nrun 7 {budget} {cancel}\ninspect 7\nread 7 0 {total} 0\nrun 7 32 -1\nread 7 0 {total} 1\nrun 7 1 0\nrewind 7\nwrite 7 0 {total} 0\nread 7 0 {total} 0\nrun 7 32 -1\nread 7 0 {total} 1\n").encode())
c.clear();c.api("create",[1,32,45]);c.api("create",[1,32,-53],1)
c.api("write",[1,0,32,0]);c.api("read",[1,0,32,0]);c.api("read",[1,0,32,0],1)
c.api("configure",[1,-8,1000]);c.api("run",[1,1,0],1);c.api("run",[1,2,-1]);c.all_state()
c.api("run",[1,2,0]);c.all_state()
c.api("run",[1,2,1]);c.all_state()
c.api("rewind",[1]);c.api("rewind",[1]);c.all_state()
c.cli(b"create 1 7 45\ncreate 31 4 -53\nconfigure 1 -8 1000\nrun 1 2 -1\nrun 1 2 0\nread 1 0 2 1\nrun 1 2 1\nread 1 0 3 1\nrun 1 32 -1\nread 1 0 7 1\nrewind 1\nrewind 1\ninspect 1\nread 1 0 7 0\ninspect 31\nread 31 0 4 0\nlist\n")
rng=random.Random(10620261027)
for _ in range(1100):
n=rng.randrange(2);verb=rng.choice(list(DOMAINS));args=[]
for lo,hi in DOMAINS[verb]:
args.append(rng.choice([lo,hi,rng.randint(lo,hi),lo-1,hi+1]))
if args and rng.randrange(3): args[0]=rng.choice([1,7,31])
c.api(verb,args)
c.formatter()
# Each verb's fixed arity, malformed number in each position, and fresh streams.
base=b"create 1 4 67\ncreate 31 3 -9\n"
tail=b"inspect 1\nread 1 0 4 0\ninspect 31\nread 31 0 3 0\nlist\n"
for verb,args in defaults.items():
good=verb+(" "+" ".join(map(str,args)) if args else "")
bad=[good+" 0"]
if args: bad.append(verb+(" "+" ".join(map(str,args[:-1])) if len(args)>1 else ""))
else: bad.append("list 0 0")
for index in range(len(args)):
for token in ["x","+","-","1x","1.0","0x1","9223372036854775808","-9223372036854775809","1\v"]:
words=list(map(str,args));words[index]=token;bad.append(verb+" "+" ".join(words))
for command in bad:
c.cli(base+command.encode()+b"\n"+tail)
for data in [b"",b"list",b"\n",b"\r\t \n",b"LIST\n",b"list\x00\n",b"create +01 0002 -0009\r\nrun 01 +02 -01\nread 1 0 2 1",b"list\nlist\n"]:
c.cli(data)
mixed=[b"create 1 32 59",b"create 31 4 -7",b"",b"unknown",b"inspect 2"]
while len(mixed)<4095:
mixed.append([b"list",b"inspect 1",b"",b"run 2 1 -1",b"write 1 0 x 9"][len(mixed)%5])
boundary=b"\n".join(mixed+[b"run 1 32 -1",b"rewind 1",b"erase 31"])+b"\n"
c.cli(boundary)
c.cli(b"\n".join(mixed+[b"run 1 32 -1"]))
c.cli(b"create 1 1 9\nrun 1 1 -1\n")
c.cli(b"inspect 1\nlist\n")
required={"create":{0,1,2},"configure":{0,1,3,4},"write":{0,1,3,4},"read":{0,1,3},"inspect":{0,1,3},"run":{0,1,3},"rewind":{0,1,3},"erase":{0,1,3},"list":{0}}
# BUSY from run is unreachable for compliant synchronous runtimes.
assert c.api_codes==required, (c.api_codes,required)
assert c.cli_codes==required, (c.cli_codes,required)
for verb,domains in DOMAINS.items():
for index,(lo,hi) in enumerate(domains):
for value in (lo,hi):
assert (verb,index,value) in c.api_success,("api endpoint",verb,index,value)
assert (verb,index,value) in c.cli_success,("cli endpoint",verb,index,value)
main_expected=c.cli_model(boundary)
a=[];ae=[];q=[];qe=[]
for cmd,expect in zip(c.commands,c.expected):
target,target_e=(q,qe) if cmd.startswith("C ") else (a,ae)
target.append(cmd);target_e.append(expect)
stats={"api_observations":c.observations,"formatter_cases":109,"numeric_parameters":sum(map(len,DOMAINS.values())),"cli_calls":c.cli_calls,"cli_dispatched_physical_lines":c.cli_lines,"api_codes":{k:sorted(v) for k,v in c.api_codes.items()},"cli_codes":{k:sorted(v) for k,v in c.cli_codes.items()},"successful_endpoints":len(c.api_success),"cli_successful_endpoints":len(c.cli_success)}
smoke=b"create 7 4 7\nrun 7 1 -1\nrun 7 3 -1\nread 7 0 4 1\n"
smoke_expected=c.cli_model(smoke)
stats["cli_dispatched_physical_lines"]=c.cli_lines
return [("api",("\n".join(a)+"\n").encode(),b"".join(ae)),("streams",("\n".join(q)+"\n").encode(),b"".join(qe)),("main",boundary,main_expected),("empty",b"",b""),("smoke",smoke,smoke_expected)],stats
def digest(data):
return [len(data),hashlib.sha256(data).hexdigest()[:16]]
def execute(argv,data,expected,seconds,records,label):
p=subprocess.Popen(argv,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
timed=False
try:
out,err=p.communicate(data,timeout=seconds)
except subprocess.TimeoutExpired:
timed=True;p.kill();out,err=p.communicate()
reasons=[]
if timed: reasons.append("timeout")
if p.returncode!=0: reasons.append("exit")
difference = None
if out!=expected:
reasons.append("stdout")
limit=min(len(out),len(expected))
at=next((i for i in range(limit) if out[i]!=expected[i]),limit)
difference=[at,expected[:at].count(b"\n")+1]
if err: reasons.append("stderr")
records.append({"k":label,"pid":p.pid,"argv":argv,"rc":p.returncode,"to":timed,"in":digest(data),"exp":digest(expected),"out":digest(out),"err":digest(err),"why":reasons,"diff":difference})
return not reasons
def main():
candidate=Path(sys.argv[1])
scenes,stats=generate()
scratch=Path(tempfile.mkdtemp(prefix="cancel-",dir=Path.cwd()))
# Complete trusted filesystem IO before the first compiler is launched.
for name in ["runtime.hpp","store.cpp","engine.cpp","cli.cpp","main.cpp"]:
(scratch/name).write_bytes((candidate/name).read_bytes())
(scratch/"observer.cpp").write_text(OBSERVER)
root=str(scratch)
working_directory=str(Path.cwd())
sources=[str(scratch/name) for name in ["store.cpp","engine.cpp","cli.cpp"]]
compiles=[(["g++","-std=c++17","-O2","-Wall","-Wextra","-pedantic",*sources,str(scratch/"observer.cpp"),"-o",root+"/observer"],"raw-build"),(["g++","-std=c++17","-O2","-Wall","-Wextra","-pedantic",*sources,str(scratch/"main.cpp"),"-o",root+"/runtime"],"cli-build")]
records=[];build_ok=True
for argv,label in compiles:
# Compiler diagnostics are retained but warnings do not reject legal candidates.
p=subprocess.Popen(argv,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
timed=False
try: out,err=p.communicate(b"",timeout=30)
except subprocess.TimeoutExpired:
timed=True;p.kill();out,err=p.communicate()
reasons=(["timeout"] if timed else [])+(["exit"] if p.returncode else [])
records.append({"k":label,"pid":p.pid,"argv":argv,"rc":p.returncode,"to":timed,"in":digest(b""),"exp":digest(b""),"out":digest(out),"err":digest(err),"why":reasons})
build_ok=build_ok and not reasons
ok=build_ok
if build_ok:
for label,data,expected in scenes:
argv=[root+("/runtime" if label in ("main","empty","smoke") else "/observer")]
ok=execute(argv,data,expected,10,records,label) and ok
metadata={"v":1,"cwd":working_directory,"records":records}
payload=json.dumps(metadata,separators=(",",":"))
if len(payload.encode())>=3900:
raise RuntimeError("metadata exceeds retained evidence budget: "+str(len(payload)))
sys.stderr.write(payload+"\n")
print("PASS" if ok else "behavior mismatch")
return 0 if ok else 1
if __name__=="__main__":
raise SystemExit(main())