amos 60319fadd5 Merge only Qwen-hard and -medium tasks: 11 from batches 25, 26 and 27
The user wants 2000 tasks, most of them ones Qwen cannot solve. New batch
tasks are now measured on their branch first (3 Qwen trials each), and only
HARD or MEDIUM tasks reach main; ROUTINE ones stay on the batch branches.
A session cut by context_limit or output_limit counts as a failure for
difficulty (user decision 2026-11-03); the trajectory stays incomplete.

- Batch 26 (route D, real upstream fixes in embedded open-source projects):
  7 HARD of 20 (coreJSON, littlefs, CANopen, coreMQTT, LwM2M, Zephyr ring,
  lwIP SNTP). Most failures are Qwen exhausting its context in the larger
  workspaces.
- Batch 25 (HWE mutation repairs): 1 HARD and 1 MEDIUM of 56.
- Batch 27 (spec-dense implementation tasks): 2 MEDIUM of the first 10.

Each task comes from the exact commit Qwen measured, and its digest matches
the measured one. Failure reasons are not yet reviewed, so difficulty is
the automatic label.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-03 10:51:08 +08:00

384 lines
24 KiB
Python

"""Independent finite-state oracle; all candidate execution is in child processes."""
import copy
import hashlib
import json
from pathlib import Path
import random
import shutil
import subprocess
import sys
import tempfile
OBSERVER = r'''
#include "sensor.hpp"
#include <algorithm>
#include <iostream>
#include <memory>
#include <sstream>
using Bytes=std::vector<std::uint8_t>;
static std::string rawhex(const Bytes& bytes) {
std::string out; const char* digits="0123456789abcdef";
for(auto value:bytes) { out+=digits[value>>4]; out+=digits[value&15]; }
return out.empty()?"-":out;
}
static Bytes rawbytes(std::string text) {
if(text=="-") return {};
Bytes out;
for(std::size_t i=0;i+1<text.size();i+=2) out.push_back(static_cast<std::uint8_t>(std::stoul(text.substr(i,2),nullptr,16)));
return out;
}
static std::string texthex(const std::string& text) {return rawhex(Bytes(text.begin(),text.end()));}
struct Action {int count=999; int error=0; int notice=0;};
static std::vector<Action> actions(std::string text) {
std::vector<Action> out; if(text=="-") return out;
std::replace(text.begin(),text.end(),',',' '); std::istringstream input(text); std::string token;
while(input>>token) {std::replace(token.begin(),token.end(),'/',' '); std::istringstream row(token); Action a; row>>a.count>>a.error>>a.notice; out.push_back(a);}
return out;
}
struct Host: sensor::Runtime {
sensor::Session* session=nullptr;
Bytes source,output;
std::vector<Action> reads,writes;
std::size_t ri=0,wi=0;
bool callback=false;
std::vector<std::string> events;
void notification(int reason) {callback=true; session->notify(reason); callback=false;}
std::string prefix() {return callback?"CB:":"MAIN:";}
sensor::IoResult read(std::uint64_t offset,std::uint8_t* data,std::size_t size) override {
Action a=ri<reads.size()?reads[ri++]:Action{};
if(a.notice!=0) notification(a.notice);
int count=a.count;
if(count>=0) {
std::size_t available=offset<source.size()?source.size()-offset:0;
count=static_cast<int>(std::min(size,available));
if(a.count!=999 && a.count!=998) count=std::min(count,a.count);
if(a.count==998) count=static_cast<int>(size)+1;
for(std::size_t i=0;i<std::min(size,static_cast<std::size_t>(count)) && i<available;++i) data[i]=source[offset+i];
}
events.push_back(prefix()+"R:"+std::to_string(offset)+":"+std::to_string(size)+":"+std::to_string(count)+":"+std::to_string(a.error)+":"+std::to_string(a.notice));
return {count,a.error};
}
sensor::IoResult write(std::uint64_t offset,const std::uint8_t* data,std::size_t size) override {
Action a=wi<writes.size()?writes[wi++]:Action{};
if(a.notice!=0) notification(a.notice);
int count=a.count;
if(count>=0) { count=a.count==998?static_cast<int>(size)+1:static_cast<int>(a.count==999?size:std::min(size,static_cast<std::size_t>(a.count))); }
if(count>0) output.insert(output.end(),data,data+std::min(size,static_cast<std::size_t>(count)));
events.push_back(prefix()+"W:"+std::to_string(offset)+":"+rawhex(Bytes(data,data+size))+":"+std::to_string(count)+":"+std::to_string(a.error)+":"+std::to_string(a.notice));
return {count,a.error};
}
void log(const std::string& text) override {events.push_back(prefix()+"L:"+text);}
void reserve(std::size_t size) override {events.push_back(prefix()+"A:"+std::to_string(size));}
void release() override {events.push_back(prefix()+"F");}
};
static void dump(int id,const sensor::Session& session,Host& host) {
auto s=session.snapshot();
std::cout<<"S "<<id<<' '<<static_cast<int>(s.state)<<' '<<s.config.block<<' '<<s.config.channel<<' '<<s.config.limit<<' '<<s.epoch<<' '<<s.sequence<<' '<<s.stop<<' '<<s.pending<<' '<<s.error<<' '<<s.input<<' '<<s.output<<' '<<s.records<<' '<<s.reads<<' '<<s.writes<<' '<<s.interrupts<<' '<<s.discarded<<' '<<s.buffered<<' '<<s.remaining<<' '<<s.owned<<' '<<texthex(sensor::describe(s))<<'\n';
for(const auto& event:host.events) std::cout<<"V "<<id<<' '<<event<<'\n';
host.events.clear();
std::cout<<"O "<<id<<' '<<rawhex(host.output)<<'\n';
}
int main() {
Host hosts[2];
sensor::Session first(hosts[0]),second(hosts[1]);
sensor::Session* sessions[]={&first,&second};
hosts[0].session=&first; hosts[1].session=&second;
sensor::Mailbox mailbox;
sensor::Catalogue cats[2];
std::string line;
while(std::getline(std::cin,line)) {
std::istringstream in(line); std::string op; in>>op;
if(op=="H") {int id; std::string data,r,w; in>>id>>data>>r>>w; auto& h=hosts[id]; h.source=rawbytes(data); h.output.clear(); h.reads=actions(r); h.writes=actions(w); h.ri=h.wi=0; std::cout<<"H "<<id<<'\n'; continue;}
if(op=="M") {std::string action; int reason=0; in>>action>>reason; if(action=="P") mailbox.publish(reason); if(action=="C") mailbox.clear(); int taken=action=="T"?mailbox.take():-1; std::cout<<"M "<<taken<<' '<<mailbox.peek()<<'\n'; continue;}
if(op=="Q") {sensor::Config c; in>>c.block>>c.channel>>c.limit; std::cout<<"Q "<<int(sensor::validate(c))<<'\n'; continue;}
if(op=="E") {unsigned ch; std::uint32_t epoch,seq; std::string bytes; in>>ch>>epoch>>seq>>bytes; sensor::Frame f{static_cast<std::uint8_t>(ch),epoch,seq,rawbytes(bytes)}; std::cout<<"E "<<rawhex(sensor::encode(f))<<'\n'; continue;}
if(op=="D") {std::string bytes; in>>bytes; sensor::Frame f{77,88,99,{1,2}}; auto code=sensor::decode(rawbytes(bytes),f); std::cout<<"D "<<int(code)<<' '<<unsigned(f.channel)<<' '<<f.epoch<<' '<<f.sequence<<' '<<rawhex(f.payload)<<'\n'; continue;}
if(op=="X") {std::string text; in>>text; if(text=="-") text.clear(); Bytes bytes{7,8}; auto code=sensor::unhex(text,bytes); std::cout<<"X "<<int(code)<<' '<<rawhex(bytes)<<'\n'; continue;}
if(op=="G") {std::string text; in>>text; std::cout<<"G "<<sensor::hex(rawbytes(text))<<'\n'; continue;}
if(op=="K" || op=="B") {int id; std::string text; in>>id>>text; auto code=op=="K"?cats[id].accept(rawbytes(text)):cats[id].accept_stream(rawbytes(text)); std::cout<<op<<' '<<int(code)<<'\n'; continue;}
if(op=="J") {int a,b; in>>a>>b; std::cout<<"J "<<int(cats[a].merge(cats[b]))<<'\n'; continue;}
if(op=="L") {int id,ch; in>>id>>ch; sensor::Entry e{77,88,99,100,101}; auto code=cats[id].lookup(ch,e); std::cout<<"L "<<int(code)<<' '<<cats[id].contains(ch)<<' '<<e.epoch<<' '<<e.next<<' '<<e.records<<' '<<e.bytes<<' '<<e.sum<<'\n'; continue;}
if(op=="Y") {int id,ch; in>>id>>ch; std::cout<<"Y "<<int(cats[id].erase(ch))<<'\n'; continue;}
if(op=="U") {int id; in>>id; cats[id].clear(); std::cout<<"U\n"; continue;}
if(op=="T") {int id; in>>id; std::cout<<"T "<<cats[id].size()<<' '<<cats[id].records()<<' '<<cats[id].bytes()<<' '<<texthex(cats[id].summary()); for(int ch:cats[id].channels()) std::cout<<' '<<ch; std::cout<<'\n'; continue;}
int id; in>>id; auto& session=*sessions[id]; int code=0;
if(op=="C") {sensor::Config config; in>>config.block>>config.channel>>config.limit; code=int(session.configure(config));}
if(op=="A") {std::uint32_t epoch; in>>epoch; code=int(session.start(epoch));}
if(op=="N") {int reason; in>>reason; hosts[id].notification(reason);}
if(op=="P") {int quota; in>>quota; code=int(session.pump(quota));}
if(op=="Z") code=int(session.reset());
std::cout<<op<<' '<<id<<' '<<code<<'\n';
dump(id,session,hosts[id]);
}
}
'''
def hx(value):
return bytes(value).hex() or '-'
def frame(channel, epoch, sequence, payload):
if not 1 <= len(payload) <= 32:
return b''
data=bytes([165,channel])+epoch.to_bytes(4,'little')+sequence.to_bytes(4,'little')+bytes([len(payload)])+bytes(payload)
return data+bytes([sum(data[1:])%256])
def decode(data):
if not 13 <= len(data) <= 44 or data[0]!=165 or not 1 <= data[10] <=32 or len(data)!=data[10]+12 or sum(data[1:-1])%256!=data[-1]:
return None
return data[1],int.from_bytes(data[2:6],'little'),int.from_bytes(data[6:10],'little'),data[11:-1]
class Session:
def __init__(self):
self.config=[4,1,0]
self.state=0
self.clear()
self.source=b''; self.out=b''; self.r=[]; self.w=[]; self.events=[]
def clear(self):
self.epoch=self.seq=self.stop=self.pending=self.error=0
self.ib=self.ob=self.records=self.reads=self.writes=self.interrupts=self.discarded=0
self.buffer=b''; self.packet=b''; self.sent=0; self.eof=False; self.owned=False
def note(self,reason):
if self.state==1 and 1<=reason<=31 and not self.pending:
self.pending=reason
def fail(self,code,error):
self.state=4; self.error=error
self.events+=['MAIN:L:fault:'+str(error),'MAIN:F']; self.owned=False
return code
def end(self):
self.state=3; self.events+=['MAIN:L:end','MAIN:F']; self.owned=False
def take(self):
if not self.pending:return False
self.stop=self.pending; self.pending=0; self.discarded=len(self.buffer)
self.buffer=b''; self.packet=b''; self.sent=0; self.state=2
self.events+=['MAIN:L:stop:'+str(self.stop),'MAIN:F'];self.owned=False
return True
def pump(self,quota):
if quota<0 or quota>32:return 1
if self.state!=1:return 3
if self.take() or quota==0:return 0
completed=0
for guard in range(10000):
if self.state!=1:return 0
if self.take():return 0
if not self.packet and self.eof:self.end();return 0
if self.config[2] and self.records>=self.config[2]:self.end();return 0
if completed>=quota:return 0
writing=bool(self.packet)
plan=self.w if writing else self.r
n,error,notice=plan.pop(0) if plan else (999,0,0)
self.note(notice)
if writing:
tail=self.packet[self.sent:]; size=len(tail); self.writes+=1
if n>=0:n=size+1 if n==998 else size if n==999 else min(size,n)
if n>0:self.out+=tail[:n]
self.events.append('MAIN:W:'+str(self.ob)+':'+hx(tail)+':'+str(n)+':'+str(error)+':'+str(notice))
else:
size=self.config[0]-len(self.buffer); self.reads+=1
if n>=0:n=size+1 if n==998 else min(size,max(0,len(self.source)-self.ib),n)
self.events.append('MAIN:R:'+str(self.ib)+':'+str(size)+':'+str(n)+':'+str(error)+':'+str(notice))
if n==-1:
if error==4:self.interrupts+=1;continue
if error==11:return 0
return self.fail(4,error)
if n<0 or n>size or error:return self.fail(5,0)
if writing:
if n==0:return self.fail(4,5)
self.sent+=n;self.ob+=n
if self.sent==len(self.packet):
self.packet=b'';self.sent=0;self.buffer=b'';self.records+=1;self.seq+=1;completed+=1
else:
if n==0:
self.eof=True
if not self.buffer:self.end();return 0
else:
self.buffer+=self.source[self.ib:self.ib+n];self.ib+=n
if self.buffer and (self.eof or len(self.buffer)==self.config[0]):
self.packet=frame(self.config[1],self.epoch,self.seq,self.buffer);self.sent=0
raise AssertionError('oracle guard')
def snap(self,ident):
values=[self.state,*self.config,self.epoch,self.seq,self.stop,self.pending,self.error,self.ib,self.ob,self.records,self.reads,self.writes,self.interrupts,self.discarded,len(self.buffer),len(self.packet)-self.sent,int(self.owned)]
names=['channel','block','limit','epoch','sequence','stop','pending','error','input','output','records','reads','writes','interrupts','discarded','buffered','remaining','owned']
descvalues=[self.config[1],self.config[0],self.config[2],*values[4:]]
desc=['idle','running','stopped','ended','fault'][self.state]+' '+ ' '.join(k+'='+str(v) for k,v in zip(names,descvalues))
result=['S '+str(ident)+' '+' '.join(map(str,values))+' '+hx(desc.encode())]
result+=['V '+str(ident)+' '+e for e in self.events]
self.events=[]
result+=['O '+str(ident)+' '+hx(self.out)]
return result
def raw(text):return b'' if text=='-' else bytes.fromhex(text)
def plan(text):return [] if text=='-' else [tuple(map(int,t.split('/'))) for t in text.split(',')]
def oracle(lines):
sessions=[Session(),Session()]; cats=[{},{}];mail=0;out=[]
def accept(cat,data):
f=decode(data)
if f is None:return 5
ch,ep,seq,payload=f
old=cat.get(ch)
if old is None or old[0]!=ep:
if seq:return 3
old=[ep,0,0,0,0]
if seq!=old[1]:return 3
cat[ch]=[ep,seq+1,old[2]+1,old[3]+len(payload),old[4]+sum(payload)]
return 0
for line in lines:
p=line.split();op=p[0]
if op=='H':
ident=int(p[1]);s=sessions[ident];s.source=raw(p[2]);s.out=b'';s.r=plan(p[3]);s.w=plan(p[4]);out.append('H '+str(ident));continue
if op=='M':
action=p[1];reason=int(p[2]);taken=-1
if action=='P' and 1<=reason<=31 and not mail:mail=reason
if action=='C':mail=0
if action=='T':taken=mail;mail=0
out.append('M '+str(taken)+' '+str(mail));continue
if op=='Q':
b,ch,lim=map(int,p[1:]);out.append('Q '+str(0 if 1<=b<=32 and 0<=ch<=255 and 0<=lim<=100000 else 1));continue
if op=='E':out.append('E '+hx(frame(int(p[1]),int(p[2]),int(p[3]),raw(p[4]))));continue
if op=='D':
f=decode(raw(p[1]));f=(77,88,99,b'\x01\x02') if f is None else f
out.append('D '+str(5 if decode(raw(p[1])) is None else 0)+' '+' '.join(map(str,f[:3]))+' '+hx(f[3]));continue
if op=='X':
text='' if p[1]=='-' else p[1]
valid=len(text)%2==0 and all(c in '0123456789abcdefABCDEF' for c in text)
out.append('X '+str(0 if valid else 1)+' '+hx(bytes.fromhex(text) if valid else b'\x07\x08'));continue
if op=='G':out.append('G '+raw(p[1]).hex());continue
if op in ('K','B'):
ident=int(p[1]);data=raw(p[2]);cat=cats[ident];code=0
if op=='K':code=accept(cat,data)
else:
nextcat=copy.deepcopy(cat);cursor=0
while cursor<len(data):
if len(data)-cursor<11:code=5;break
length=data[cursor+10]+12
if length>len(data)-cursor:code=5;break
code=accept(nextcat,data[cursor:cursor+length])
if code:break
cursor+=length
if not code:cats[ident]=nextcat
out.append(op+' '+str(code));continue
if op=='J':
a,b=map(int,p[1:]);collision=bool(cats[a].keys() & cats[b].keys())
if not collision:cats[a].update(copy.deepcopy(cats[b]))
out.append('J '+str(3 if collision else 0));continue
if op=='L':
ident,ch=map(int,p[1:]);e=cats[ident].get(ch);code=1 if not 0<=ch<=255 else 3 if e is None else 0
out.append('L '+str(code)+' '+str(int(e is not None))+' '+' '.join(map(str,e if e is not None else [77,88,99,100,101])));continue
if op=='Y':
ident,ch=map(int,p[1:]);code=1 if not 0<=ch<=255 else 3 if ch not in cats[ident] else 0
if not code:del cats[ident][ch]
out.append('Y '+str(code));continue
if op=='U':cats[int(p[1])]={};out.append('U');continue
if op=='T':
cat=cats[int(p[1])];summary=';'.join(str(ch)+':'+':'.join(map(str,cat[ch])) for ch in sorted(cat))
out.append('T '+str(len(cat))+' '+str(sum(e[2] for e in cat.values()))+' '+str(sum(e[3] for e in cat.values()))+' '+hx(summary.encode())+''.join(' '+str(ch) for ch in sorted(cat)));continue
ident=int(p[1]);s=sessions[ident];code=0
if op=='C':
config=list(map(int,p[2:]));b,ch,lim=config
code=1 if not(1<=b<=32 and 0<=ch<=255 and 0<=lim<=100000) else 2 if s.state==1 else 0
if not code:s.config=config
elif op=='A':
epoch=int(p[2]);code=1 if epoch==0 else 2 if s.state==1 else 0
if not code:
s.clear();s.epoch=epoch;s.state=1;s.owned=True;s.events+=['MAIN:A:32','MAIN:L:start:'+str(epoch)]
elif op=='N':s.note(int(p[2]))
elif op=='P':code=s.pump(int(p[2]))
elif op=='Z':
code=2 if s.state==1 else 0
if not code:s.clear();s.config=[4,1,0];s.state=0
out.append(op+' '+str(ident)+' '+str(code));out+=s.snap(ident)
return ('\n'.join(out)+'\n').encode()
def scenes():
rng=random.Random(11022026002)
flow=['S 0','S 1','P 0 0','N 0 2','Z 0']
for reason in [-10,0,1,31,32,2147483647]:flow+=['M P '+str(reason),'M P 4','M T 0','M T 0','M P 3','M C 0']
for config in [(0,0,0),(33,255,0),(1,-1,0),(32,256,0),(1,1,-1),(32,1,100001)]:flow.append('C 0 '+' '.join(map(str,config)))
flow+=['A 0 0','H 0 010203040506070809 1/0/0,2/0/0,-1/11/0 -','C 0 4 255 0','A 0 4294967295','C 0 0 0 0','C 0 1 0 0','A 0 0','A 0 1','Z 0','P 0 -1','P 0 33','P 0 0','P 0 1','P 0 1','N 0 31','N 0 1','N 0 0','S 0','P 0 0','N 0 9','P 0 1','Z 0']
for index in range(48):
ident=index%2;b=rng.choice([1,2,3,4,7,16,31,32]);ch=rng.choice([0,1,17,255]);limit=rng.choice([0,1,2,100000]);length=rng.choice([0,1,b-1,b,b+1,b*3+2]);source=bytes(rng.randrange(256) for _ in range(length))
reads='-1/4/0,1/0/0,-1/4/0,2/0/0,-1/11/0' if index%3==0 else '1/0/0,2/0/0'
writes='1/0/0,-1/4/0,2/0/0,-1/11/0,-1/4/0' if index%3==1 else '1/0/0,2/0/0'
flow+=['N '+str(ident)+' 7','P '+str(ident)+' 0','Z '+str(ident),'C '+str(ident)+' '+str(b)+' '+str(ch)+' '+str(limit),'H '+str(ident)+' '+hx(source)+' '+reads+' '+writes,'A '+str(ident)+' '+str(index+1)]
flow+=['P '+str(ident)+' '+str(q) for q in [0,1,2,32,1,32,32]]
flow+=['S '+str(ident),'N '+str(ident)+' 1','P '+str(ident)+' 0','Z '+str(ident)]
# Notice inside the backend call, including interrupted and paused transfers.
for which in ['read','write']:
for n,error in [(-1,4),(-1,11),(1,0),(999,0)]:
r=f'{n}/{error}/5' if which=='read' else '-'
w=f'{n}/{error}/7' if which=='write' else '-'
flow+=['Z 0','C 0 4 8 0','H 0 000102030405060708 '+r+' '+w,'A 0 53','P 0 32','S 0','P 0 0','Z 0']
# Failure and malformed backend result paths, followed by restart without reset.
for which in ['read','write']:
for n,error in [(-1,5),(-1,0),(-2,0),(998,0),(1,9),(0,0)]:
r=f'{n}/{error}/0' if which=='read' else '-'
w=f'{n}/{error}/0' if which=='write' else '-'
flow+=['Z 0','H 0 1011121314151617 '+r+' '+w,'A 0 9','P 0 32','S 0','P 0 32','C 0 1 0 1','H 0 aa - -','A 0 10','P 0 1','S 0','Z 0','N 0 2','P 0 0','Z 0']
# Interrupted writes after partial output and stop after partial record.
flow+=['Z 0','H 0 000102030405060708 - 1/0/0,-1/11/0','A 0 76','P 0 1','N 0 3','P 0 32','Z 0']
flow+=['Z 0','H 0 010203 - -','C 0 1 0 100000','A 0 4','P 0 32','H 0 0102030405 - -','A 0 5','N 0 7','P 0 0','H 0 aabb - -','A 0 6','P 0 32','Z 0']
utility=['Q '+' '.join(map(str,c)) for c in [(1,0,0),(32,255,100000),(0,1,0),(33,1,0),(1,-1,0),(1,256,0),(1,1,-1),(1,1,100001)]]
for size in [0,1,2,31,32,33]:
data=bytes(rng.randrange(256) for _ in range(size))
for ch,ep,seq in [(0,0,0),(255,4294967295,4294967295),(23,17,99)]:
utility+=['E '+str(ch)+' '+str(ep)+' '+str(seq)+' '+hx(data)]
packet=frame(ch,ep,seq,data)
utility+=['D '+hx(packet)]
if packet:
for i in [0,1,2,6,10,11,len(packet)-1]:
changed=bytearray(packet);changed[i]^=1;utility+=['D '+hx(changed)]
utility+=['D '+hx(packet[:-1]),'D '+hx(packet+b'\x00')]
utility+=['X '+x for x in ['-','00','aBcD','abcdefABCDEF0123456789','f','0z','+1','gg','000']]
utility+=['G '+hx(bytes(range(256))),'G -']
cat=['T 0','T 1','J 0 1','J 0 0','L 0 -1','L 0 256','L 0 0','Y 0 0','Y 0 -1','Y 0 256','B 0 -']
for ch in [255,0,17,1]:
cat+=['K 0 '+hx(frame(ch,7,1,b'ab')),'T 0','K 0 '+hx(frame(ch,7,0,b'ab')),'K 0 '+hx(frame(ch,7,0,b'ab')),'K 0 '+hx(frame(ch,7,1,b'\x00\xff')),'L 0 '+str(ch),'K 0 '+hx(frame(ch,8,1,b'a')),'K 0 '+hx(frame(ch,8,0,b'x')),'T 0']
cat+=['J 1 0','T 1','J 0 1','J 0 0','Y 1 17','T 1','L 1 17','U 1','T 1']
for index in range(32):
ch=rng.randrange(256);ep=rng.randrange(100);payload=bytes(rng.randrange(256) for _ in range(rng.randrange(1,33)))
stream=frame(ch,ep,0,payload)+frame(ch,ep,1,payload[::-1])
cat+=['U 1','B 1 '+hx(stream),'T 1','L 1 '+str(ch),'U 1','B 1 '+hx(stream+b'\x00'),'T 1','B 1 '+hx(stream[:-1]),'T 1','B 1 '+hx(frame(ch,ep,0,payload)+frame(ch,ep,2,payload)),'T 1']
cat+=['U 0','U 1','K 0 '+hx(frame(0,0,0,b'a')),'K 1 '+hx(frame(255,1,0,b'b')),'J 0 1','T 0','T 1','K 1 00','T 1','Y 0 0','Y 0 255','T 0','U 0','T 0']
cat+=['B 0 '+hx(frame(19,4,0,b'new')+b'\x00'),'T 0','K 0 '+hx(frame(3,1,0,b'a')[:-1]+b'\x00'),'T 0','B 0 '+hx(frame(3,1,0,b'a')[:-1]+b'\x00'),'T 0']
smoke=['H 0 41424344454647 - -','A 0 7','P 0 1','N 0 2','P 0 0']
callback=['H 0 0102030405 1/0/0,-1/11/0 -','A 0 5','P 0 1','N 0 31','N 0 1','P 0 0']
retry=['H 0 0102030405060708 '+','.join(['-1/4/0']*20+['2/0/0','-1/4/0'])+' 1/0/0,-1/4/0,2/0/0,-1/4/0','A 0 11','P 0 32']
return [('smoke',smoke),('callback',callback),('retry',retry),('sessions',flow),('codec',utility),('catalogue',cat)]
def digest(data):return {'n':len(data),'sha256':hashlib.sha256(data).hexdigest()[:16]}
def main():
candidate=Path(sys.argv[1])
scratch=Path(tempfile.mkdtemp(prefix='sensor-observe-',dir=Path.cwd()))
names=['include/sensor.hpp','src/codec.cpp','src/mailbox.cpp','src/session.cpp','src/catalogue.cpp']
for name in names:
p=scratch/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes((candidate/name).read_bytes())
(scratch/'observe.cpp').write_text(OBSERVER)
# Freeze every input and expected observation before the first child starts.
prepared=[(name,('\n'.join(lines)+'\n').encode(),oracle(lines)) for name,lines in scenes()]
reports=[];failures=[]
argv=['g++','-std=c++17','-O2','-Iinclude','src/codec.cpp','src/mailbox.cpp','src/session.cpp','src/catalogue.cpp','observe.cpp','-o','observe']
try:
built=subprocess.run(argv,cwd=scratch,capture_output=True,timeout=25)
reports.append({'scene':'build','exit':built.returncode,'stdout':digest(built.stdout),'stderr':digest(built.stderr)})
if built.returncode:failures.append('build')
else:
for name,data,expected in prepared:
try:
result=subprocess.run(['./observe'],cwd=scratch,input=data,capture_output=True,timeout=12)
reasons=[]
if result.returncode:reasons.append('exit')
if result.stdout!=expected:reasons.append('stdout')
if result.stderr:reasons.append('stderr')
reports.append({'scene':name,'exit':result.returncode,'stdin':digest(data),'expected':digest(expected),'stdout':digest(result.stdout),'stderr':digest(result.stderr),'reasons':reasons})
failures += [name+':'+reason for reason in reasons]
except subprocess.TimeoutExpired as error:
reports.append({'scene':name,'timeout':True,'stdout':digest(error.stdout or b''),'stderr':digest(error.stderr or b'')});failures.append(name+':timeout');break
except subprocess.TimeoutExpired:
failures.append('build:timeout')
sys.stderr.write(json.dumps({'checks':reports,'failures':failures},separators=(',',':'))+'\n')
print('FAIL '+','.join(failures) if failures else 'PASS')
return int(bool(failures))
if __name__=='__main__':raise SystemExit(main())