The user wants 2000 tasks, most of them ones Qwen cannot solve. New batch tasks are now measured on their branch first (3 Qwen trials each), and only HARD or MEDIUM tasks reach main; ROUTINE ones stay on the batch branches. A session cut by context_limit or output_limit counts as a failure for difficulty (user decision 2026-11-03); the trajectory stays incomplete. - Batch 26 (route D, real upstream fixes in embedded open-source projects): 7 HARD of 20 (coreJSON, littlefs, CANopen, coreMQTT, LwM2M, Zephyr ring, lwIP SNTP). Most failures are Qwen exhausting its context in the larger workspaces. - Batch 25 (HWE mutation repairs): 1 HARD and 1 MEDIUM of 56. - Batch 27 (spec-dense implementation tasks): 2 MEDIUM of the first 10. Each task comes from the exact commit Qwen measured, and its digest matches the measured one. Failure reasons are not yet reviewed, so difficulty is the automatic label. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
384 lines
24 KiB
Python
384 lines
24 KiB
Python
"""Independent finite-state oracle; all candidate execution is in child processes."""
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import random
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
OBSERVER = r'''
|
|
#include "sensor.hpp"
|
|
#include <algorithm>
|
|
#include <iostream>
|
|
#include <memory>
|
|
#include <sstream>
|
|
using Bytes=std::vector<std::uint8_t>;
|
|
static std::string rawhex(const Bytes& bytes) {
|
|
std::string out; const char* digits="0123456789abcdef";
|
|
for(auto value:bytes) { out+=digits[value>>4]; out+=digits[value&15]; }
|
|
return out.empty()?"-":out;
|
|
}
|
|
static Bytes rawbytes(std::string text) {
|
|
if(text=="-") return {};
|
|
Bytes out;
|
|
for(std::size_t i=0;i+1<text.size();i+=2) out.push_back(static_cast<std::uint8_t>(std::stoul(text.substr(i,2),nullptr,16)));
|
|
return out;
|
|
}
|
|
static std::string texthex(const std::string& text) {return rawhex(Bytes(text.begin(),text.end()));}
|
|
struct Action {int count=999; int error=0; int notice=0;};
|
|
static std::vector<Action> actions(std::string text) {
|
|
std::vector<Action> out; if(text=="-") return out;
|
|
std::replace(text.begin(),text.end(),',',' '); std::istringstream input(text); std::string token;
|
|
while(input>>token) {std::replace(token.begin(),token.end(),'/',' '); std::istringstream row(token); Action a; row>>a.count>>a.error>>a.notice; out.push_back(a);}
|
|
return out;
|
|
}
|
|
struct Host: sensor::Runtime {
|
|
sensor::Session* session=nullptr;
|
|
Bytes source,output;
|
|
std::vector<Action> reads,writes;
|
|
std::size_t ri=0,wi=0;
|
|
bool callback=false;
|
|
std::vector<std::string> events;
|
|
void notification(int reason) {callback=true; session->notify(reason); callback=false;}
|
|
std::string prefix() {return callback?"CB:":"MAIN:";}
|
|
sensor::IoResult read(std::uint64_t offset,std::uint8_t* data,std::size_t size) override {
|
|
Action a=ri<reads.size()?reads[ri++]:Action{};
|
|
if(a.notice!=0) notification(a.notice);
|
|
int count=a.count;
|
|
if(count>=0) {
|
|
std::size_t available=offset<source.size()?source.size()-offset:0;
|
|
count=static_cast<int>(std::min(size,available));
|
|
if(a.count!=999 && a.count!=998) count=std::min(count,a.count);
|
|
if(a.count==998) count=static_cast<int>(size)+1;
|
|
for(std::size_t i=0;i<std::min(size,static_cast<std::size_t>(count)) && i<available;++i) data[i]=source[offset+i];
|
|
}
|
|
events.push_back(prefix()+"R:"+std::to_string(offset)+":"+std::to_string(size)+":"+std::to_string(count)+":"+std::to_string(a.error)+":"+std::to_string(a.notice));
|
|
return {count,a.error};
|
|
}
|
|
sensor::IoResult write(std::uint64_t offset,const std::uint8_t* data,std::size_t size) override {
|
|
Action a=wi<writes.size()?writes[wi++]:Action{};
|
|
if(a.notice!=0) notification(a.notice);
|
|
int count=a.count;
|
|
if(count>=0) { count=a.count==998?static_cast<int>(size)+1:static_cast<int>(a.count==999?size:std::min(size,static_cast<std::size_t>(a.count))); }
|
|
if(count>0) output.insert(output.end(),data,data+std::min(size,static_cast<std::size_t>(count)));
|
|
events.push_back(prefix()+"W:"+std::to_string(offset)+":"+rawhex(Bytes(data,data+size))+":"+std::to_string(count)+":"+std::to_string(a.error)+":"+std::to_string(a.notice));
|
|
return {count,a.error};
|
|
}
|
|
void log(const std::string& text) override {events.push_back(prefix()+"L:"+text);}
|
|
void reserve(std::size_t size) override {events.push_back(prefix()+"A:"+std::to_string(size));}
|
|
void release() override {events.push_back(prefix()+"F");}
|
|
};
|
|
static void dump(int id,const sensor::Session& session,Host& host) {
|
|
auto s=session.snapshot();
|
|
std::cout<<"S "<<id<<' '<<static_cast<int>(s.state)<<' '<<s.config.block<<' '<<s.config.channel<<' '<<s.config.limit<<' '<<s.epoch<<' '<<s.sequence<<' '<<s.stop<<' '<<s.pending<<' '<<s.error<<' '<<s.input<<' '<<s.output<<' '<<s.records<<' '<<s.reads<<' '<<s.writes<<' '<<s.interrupts<<' '<<s.discarded<<' '<<s.buffered<<' '<<s.remaining<<' '<<s.owned<<' '<<texthex(sensor::describe(s))<<'\n';
|
|
for(const auto& event:host.events) std::cout<<"V "<<id<<' '<<event<<'\n';
|
|
host.events.clear();
|
|
std::cout<<"O "<<id<<' '<<rawhex(host.output)<<'\n';
|
|
}
|
|
int main() {
|
|
Host hosts[2];
|
|
sensor::Session first(hosts[0]),second(hosts[1]);
|
|
sensor::Session* sessions[]={&first,&second};
|
|
hosts[0].session=&first; hosts[1].session=&second;
|
|
sensor::Mailbox mailbox;
|
|
sensor::Catalogue cats[2];
|
|
std::string line;
|
|
while(std::getline(std::cin,line)) {
|
|
std::istringstream in(line); std::string op; in>>op;
|
|
if(op=="H") {int id; std::string data,r,w; in>>id>>data>>r>>w; auto& h=hosts[id]; h.source=rawbytes(data); h.output.clear(); h.reads=actions(r); h.writes=actions(w); h.ri=h.wi=0; std::cout<<"H "<<id<<'\n'; continue;}
|
|
if(op=="M") {std::string action; int reason=0; in>>action>>reason; if(action=="P") mailbox.publish(reason); if(action=="C") mailbox.clear(); int taken=action=="T"?mailbox.take():-1; std::cout<<"M "<<taken<<' '<<mailbox.peek()<<'\n'; continue;}
|
|
if(op=="Q") {sensor::Config c; in>>c.block>>c.channel>>c.limit; std::cout<<"Q "<<int(sensor::validate(c))<<'\n'; continue;}
|
|
if(op=="E") {unsigned ch; std::uint32_t epoch,seq; std::string bytes; in>>ch>>epoch>>seq>>bytes; sensor::Frame f{static_cast<std::uint8_t>(ch),epoch,seq,rawbytes(bytes)}; std::cout<<"E "<<rawhex(sensor::encode(f))<<'\n'; continue;}
|
|
if(op=="D") {std::string bytes; in>>bytes; sensor::Frame f{77,88,99,{1,2}}; auto code=sensor::decode(rawbytes(bytes),f); std::cout<<"D "<<int(code)<<' '<<unsigned(f.channel)<<' '<<f.epoch<<' '<<f.sequence<<' '<<rawhex(f.payload)<<'\n'; continue;}
|
|
if(op=="X") {std::string text; in>>text; if(text=="-") text.clear(); Bytes bytes{7,8}; auto code=sensor::unhex(text,bytes); std::cout<<"X "<<int(code)<<' '<<rawhex(bytes)<<'\n'; continue;}
|
|
if(op=="G") {std::string text; in>>text; std::cout<<"G "<<sensor::hex(rawbytes(text))<<'\n'; continue;}
|
|
if(op=="K" || op=="B") {int id; std::string text; in>>id>>text; auto code=op=="K"?cats[id].accept(rawbytes(text)):cats[id].accept_stream(rawbytes(text)); std::cout<<op<<' '<<int(code)<<'\n'; continue;}
|
|
if(op=="J") {int a,b; in>>a>>b; std::cout<<"J "<<int(cats[a].merge(cats[b]))<<'\n'; continue;}
|
|
if(op=="L") {int id,ch; in>>id>>ch; sensor::Entry e{77,88,99,100,101}; auto code=cats[id].lookup(ch,e); std::cout<<"L "<<int(code)<<' '<<cats[id].contains(ch)<<' '<<e.epoch<<' '<<e.next<<' '<<e.records<<' '<<e.bytes<<' '<<e.sum<<'\n'; continue;}
|
|
if(op=="Y") {int id,ch; in>>id>>ch; std::cout<<"Y "<<int(cats[id].erase(ch))<<'\n'; continue;}
|
|
if(op=="U") {int id; in>>id; cats[id].clear(); std::cout<<"U\n"; continue;}
|
|
if(op=="T") {int id; in>>id; std::cout<<"T "<<cats[id].size()<<' '<<cats[id].records()<<' '<<cats[id].bytes()<<' '<<texthex(cats[id].summary()); for(int ch:cats[id].channels()) std::cout<<' '<<ch; std::cout<<'\n'; continue;}
|
|
int id; in>>id; auto& session=*sessions[id]; int code=0;
|
|
if(op=="C") {sensor::Config config; in>>config.block>>config.channel>>config.limit; code=int(session.configure(config));}
|
|
if(op=="A") {std::uint32_t epoch; in>>epoch; code=int(session.start(epoch));}
|
|
if(op=="N") {int reason; in>>reason; hosts[id].notification(reason);}
|
|
if(op=="P") {int quota; in>>quota; code=int(session.pump(quota));}
|
|
if(op=="Z") code=int(session.reset());
|
|
std::cout<<op<<' '<<id<<' '<<code<<'\n';
|
|
dump(id,session,hosts[id]);
|
|
}
|
|
}
|
|
'''
|
|
|
|
def hx(value):
|
|
return bytes(value).hex() or '-'
|
|
|
|
def frame(channel, epoch, sequence, payload):
|
|
if not 1 <= len(payload) <= 32:
|
|
return b''
|
|
data=bytes([165,channel])+epoch.to_bytes(4,'little')+sequence.to_bytes(4,'little')+bytes([len(payload)])+bytes(payload)
|
|
return data+bytes([sum(data[1:])%256])
|
|
|
|
def decode(data):
|
|
if not 13 <= len(data) <= 44 or data[0]!=165 or not 1 <= data[10] <=32 or len(data)!=data[10]+12 or sum(data[1:-1])%256!=data[-1]:
|
|
return None
|
|
return data[1],int.from_bytes(data[2:6],'little'),int.from_bytes(data[6:10],'little'),data[11:-1]
|
|
|
|
class Session:
|
|
def __init__(self):
|
|
self.config=[4,1,0]
|
|
self.state=0
|
|
self.clear()
|
|
self.source=b''; self.out=b''; self.r=[]; self.w=[]; self.events=[]
|
|
def clear(self):
|
|
self.epoch=self.seq=self.stop=self.pending=self.error=0
|
|
self.ib=self.ob=self.records=self.reads=self.writes=self.interrupts=self.discarded=0
|
|
self.buffer=b''; self.packet=b''; self.sent=0; self.eof=False; self.owned=False
|
|
def note(self,reason):
|
|
if self.state==1 and 1<=reason<=31 and not self.pending:
|
|
self.pending=reason
|
|
def fail(self,code,error):
|
|
self.state=4; self.error=error
|
|
self.events+=['MAIN:L:fault:'+str(error),'MAIN:F']; self.owned=False
|
|
return code
|
|
def end(self):
|
|
self.state=3; self.events+=['MAIN:L:end','MAIN:F']; self.owned=False
|
|
def take(self):
|
|
if not self.pending:return False
|
|
self.stop=self.pending; self.pending=0; self.discarded=len(self.buffer)
|
|
self.buffer=b''; self.packet=b''; self.sent=0; self.state=2
|
|
self.events+=['MAIN:L:stop:'+str(self.stop),'MAIN:F'];self.owned=False
|
|
return True
|
|
def pump(self,quota):
|
|
if quota<0 or quota>32:return 1
|
|
if self.state!=1:return 3
|
|
if self.take() or quota==0:return 0
|
|
completed=0
|
|
for guard in range(10000):
|
|
if self.state!=1:return 0
|
|
if self.take():return 0
|
|
if not self.packet and self.eof:self.end();return 0
|
|
if self.config[2] and self.records>=self.config[2]:self.end();return 0
|
|
if completed>=quota:return 0
|
|
writing=bool(self.packet)
|
|
plan=self.w if writing else self.r
|
|
n,error,notice=plan.pop(0) if plan else (999,0,0)
|
|
self.note(notice)
|
|
if writing:
|
|
tail=self.packet[self.sent:]; size=len(tail); self.writes+=1
|
|
if n>=0:n=size+1 if n==998 else size if n==999 else min(size,n)
|
|
if n>0:self.out+=tail[:n]
|
|
self.events.append('MAIN:W:'+str(self.ob)+':'+hx(tail)+':'+str(n)+':'+str(error)+':'+str(notice))
|
|
else:
|
|
size=self.config[0]-len(self.buffer); self.reads+=1
|
|
if n>=0:n=size+1 if n==998 else min(size,max(0,len(self.source)-self.ib),n)
|
|
self.events.append('MAIN:R:'+str(self.ib)+':'+str(size)+':'+str(n)+':'+str(error)+':'+str(notice))
|
|
if n==-1:
|
|
if error==4:self.interrupts+=1;continue
|
|
if error==11:return 0
|
|
return self.fail(4,error)
|
|
if n<0 or n>size or error:return self.fail(5,0)
|
|
if writing:
|
|
if n==0:return self.fail(4,5)
|
|
self.sent+=n;self.ob+=n
|
|
if self.sent==len(self.packet):
|
|
self.packet=b'';self.sent=0;self.buffer=b'';self.records+=1;self.seq+=1;completed+=1
|
|
else:
|
|
if n==0:
|
|
self.eof=True
|
|
if not self.buffer:self.end();return 0
|
|
else:
|
|
self.buffer+=self.source[self.ib:self.ib+n];self.ib+=n
|
|
if self.buffer and (self.eof or len(self.buffer)==self.config[0]):
|
|
self.packet=frame(self.config[1],self.epoch,self.seq,self.buffer);self.sent=0
|
|
raise AssertionError('oracle guard')
|
|
def snap(self,ident):
|
|
values=[self.state,*self.config,self.epoch,self.seq,self.stop,self.pending,self.error,self.ib,self.ob,self.records,self.reads,self.writes,self.interrupts,self.discarded,len(self.buffer),len(self.packet)-self.sent,int(self.owned)]
|
|
names=['channel','block','limit','epoch','sequence','stop','pending','error','input','output','records','reads','writes','interrupts','discarded','buffered','remaining','owned']
|
|
descvalues=[self.config[1],self.config[0],self.config[2],*values[4:]]
|
|
desc=['idle','running','stopped','ended','fault'][self.state]+' '+ ' '.join(k+'='+str(v) for k,v in zip(names,descvalues))
|
|
result=['S '+str(ident)+' '+' '.join(map(str,values))+' '+hx(desc.encode())]
|
|
result+=['V '+str(ident)+' '+e for e in self.events]
|
|
self.events=[]
|
|
result+=['O '+str(ident)+' '+hx(self.out)]
|
|
return result
|
|
|
|
def raw(text):return b'' if text=='-' else bytes.fromhex(text)
|
|
def plan(text):return [] if text=='-' else [tuple(map(int,t.split('/'))) for t in text.split(',')]
|
|
|
|
def oracle(lines):
|
|
sessions=[Session(),Session()]; cats=[{},{}];mail=0;out=[]
|
|
def accept(cat,data):
|
|
f=decode(data)
|
|
if f is None:return 5
|
|
ch,ep,seq,payload=f
|
|
old=cat.get(ch)
|
|
if old is None or old[0]!=ep:
|
|
if seq:return 3
|
|
old=[ep,0,0,0,0]
|
|
if seq!=old[1]:return 3
|
|
cat[ch]=[ep,seq+1,old[2]+1,old[3]+len(payload),old[4]+sum(payload)]
|
|
return 0
|
|
for line in lines:
|
|
p=line.split();op=p[0]
|
|
if op=='H':
|
|
ident=int(p[1]);s=sessions[ident];s.source=raw(p[2]);s.out=b'';s.r=plan(p[3]);s.w=plan(p[4]);out.append('H '+str(ident));continue
|
|
if op=='M':
|
|
action=p[1];reason=int(p[2]);taken=-1
|
|
if action=='P' and 1<=reason<=31 and not mail:mail=reason
|
|
if action=='C':mail=0
|
|
if action=='T':taken=mail;mail=0
|
|
out.append('M '+str(taken)+' '+str(mail));continue
|
|
if op=='Q':
|
|
b,ch,lim=map(int,p[1:]);out.append('Q '+str(0 if 1<=b<=32 and 0<=ch<=255 and 0<=lim<=100000 else 1));continue
|
|
if op=='E':out.append('E '+hx(frame(int(p[1]),int(p[2]),int(p[3]),raw(p[4]))));continue
|
|
if op=='D':
|
|
f=decode(raw(p[1]));f=(77,88,99,b'\x01\x02') if f is None else f
|
|
out.append('D '+str(5 if decode(raw(p[1])) is None else 0)+' '+' '.join(map(str,f[:3]))+' '+hx(f[3]));continue
|
|
if op=='X':
|
|
text='' if p[1]=='-' else p[1]
|
|
valid=len(text)%2==0 and all(c in '0123456789abcdefABCDEF' for c in text)
|
|
out.append('X '+str(0 if valid else 1)+' '+hx(bytes.fromhex(text) if valid else b'\x07\x08'));continue
|
|
if op=='G':out.append('G '+raw(p[1]).hex());continue
|
|
if op in ('K','B'):
|
|
ident=int(p[1]);data=raw(p[2]);cat=cats[ident];code=0
|
|
if op=='K':code=accept(cat,data)
|
|
else:
|
|
nextcat=copy.deepcopy(cat);cursor=0
|
|
while cursor<len(data):
|
|
if len(data)-cursor<11:code=5;break
|
|
length=data[cursor+10]+12
|
|
if length>len(data)-cursor:code=5;break
|
|
code=accept(nextcat,data[cursor:cursor+length])
|
|
if code:break
|
|
cursor+=length
|
|
if not code:cats[ident]=nextcat
|
|
out.append(op+' '+str(code));continue
|
|
if op=='J':
|
|
a,b=map(int,p[1:]);collision=bool(cats[a].keys() & cats[b].keys())
|
|
if not collision:cats[a].update(copy.deepcopy(cats[b]))
|
|
out.append('J '+str(3 if collision else 0));continue
|
|
if op=='L':
|
|
ident,ch=map(int,p[1:]);e=cats[ident].get(ch);code=1 if not 0<=ch<=255 else 3 if e is None else 0
|
|
out.append('L '+str(code)+' '+str(int(e is not None))+' '+' '.join(map(str,e if e is not None else [77,88,99,100,101])));continue
|
|
if op=='Y':
|
|
ident,ch=map(int,p[1:]);code=1 if not 0<=ch<=255 else 3 if ch not in cats[ident] else 0
|
|
if not code:del cats[ident][ch]
|
|
out.append('Y '+str(code));continue
|
|
if op=='U':cats[int(p[1])]={};out.append('U');continue
|
|
if op=='T':
|
|
cat=cats[int(p[1])];summary=';'.join(str(ch)+':'+':'.join(map(str,cat[ch])) for ch in sorted(cat))
|
|
out.append('T '+str(len(cat))+' '+str(sum(e[2] for e in cat.values()))+' '+str(sum(e[3] for e in cat.values()))+' '+hx(summary.encode())+''.join(' '+str(ch) for ch in sorted(cat)));continue
|
|
ident=int(p[1]);s=sessions[ident];code=0
|
|
if op=='C':
|
|
config=list(map(int,p[2:]));b,ch,lim=config
|
|
code=1 if not(1<=b<=32 and 0<=ch<=255 and 0<=lim<=100000) else 2 if s.state==1 else 0
|
|
if not code:s.config=config
|
|
elif op=='A':
|
|
epoch=int(p[2]);code=1 if epoch==0 else 2 if s.state==1 else 0
|
|
if not code:
|
|
s.clear();s.epoch=epoch;s.state=1;s.owned=True;s.events+=['MAIN:A:32','MAIN:L:start:'+str(epoch)]
|
|
elif op=='N':s.note(int(p[2]))
|
|
elif op=='P':code=s.pump(int(p[2]))
|
|
elif op=='Z':
|
|
code=2 if s.state==1 else 0
|
|
if not code:s.clear();s.config=[4,1,0];s.state=0
|
|
out.append(op+' '+str(ident)+' '+str(code));out+=s.snap(ident)
|
|
return ('\n'.join(out)+'\n').encode()
|
|
|
|
def scenes():
|
|
rng=random.Random(11022026002)
|
|
flow=['S 0','S 1','P 0 0','N 0 2','Z 0']
|
|
for reason in [-10,0,1,31,32,2147483647]:flow+=['M P '+str(reason),'M P 4','M T 0','M T 0','M P 3','M C 0']
|
|
for config in [(0,0,0),(33,255,0),(1,-1,0),(32,256,0),(1,1,-1),(32,1,100001)]:flow.append('C 0 '+' '.join(map(str,config)))
|
|
flow+=['A 0 0','H 0 010203040506070809 1/0/0,2/0/0,-1/11/0 -','C 0 4 255 0','A 0 4294967295','C 0 0 0 0','C 0 1 0 0','A 0 0','A 0 1','Z 0','P 0 -1','P 0 33','P 0 0','P 0 1','P 0 1','N 0 31','N 0 1','N 0 0','S 0','P 0 0','N 0 9','P 0 1','Z 0']
|
|
for index in range(48):
|
|
ident=index%2;b=rng.choice([1,2,3,4,7,16,31,32]);ch=rng.choice([0,1,17,255]);limit=rng.choice([0,1,2,100000]);length=rng.choice([0,1,b-1,b,b+1,b*3+2]);source=bytes(rng.randrange(256) for _ in range(length))
|
|
reads='-1/4/0,1/0/0,-1/4/0,2/0/0,-1/11/0' if index%3==0 else '1/0/0,2/0/0'
|
|
writes='1/0/0,-1/4/0,2/0/0,-1/11/0,-1/4/0' if index%3==1 else '1/0/0,2/0/0'
|
|
flow+=['N '+str(ident)+' 7','P '+str(ident)+' 0','Z '+str(ident),'C '+str(ident)+' '+str(b)+' '+str(ch)+' '+str(limit),'H '+str(ident)+' '+hx(source)+' '+reads+' '+writes,'A '+str(ident)+' '+str(index+1)]
|
|
flow+=['P '+str(ident)+' '+str(q) for q in [0,1,2,32,1,32,32]]
|
|
flow+=['S '+str(ident),'N '+str(ident)+' 1','P '+str(ident)+' 0','Z '+str(ident)]
|
|
# Notice inside the backend call, including interrupted and paused transfers.
|
|
for which in ['read','write']:
|
|
for n,error in [(-1,4),(-1,11),(1,0),(999,0)]:
|
|
r=f'{n}/{error}/5' if which=='read' else '-'
|
|
w=f'{n}/{error}/7' if which=='write' else '-'
|
|
flow+=['Z 0','C 0 4 8 0','H 0 000102030405060708 '+r+' '+w,'A 0 53','P 0 32','S 0','P 0 0','Z 0']
|
|
# Failure and malformed backend result paths, followed by restart without reset.
|
|
for which in ['read','write']:
|
|
for n,error in [(-1,5),(-1,0),(-2,0),(998,0),(1,9),(0,0)]:
|
|
r=f'{n}/{error}/0' if which=='read' else '-'
|
|
w=f'{n}/{error}/0' if which=='write' else '-'
|
|
flow+=['Z 0','H 0 1011121314151617 '+r+' '+w,'A 0 9','P 0 32','S 0','P 0 32','C 0 1 0 1','H 0 aa - -','A 0 10','P 0 1','S 0','Z 0','N 0 2','P 0 0','Z 0']
|
|
# Interrupted writes after partial output and stop after partial record.
|
|
flow+=['Z 0','H 0 000102030405060708 - 1/0/0,-1/11/0','A 0 76','P 0 1','N 0 3','P 0 32','Z 0']
|
|
flow+=['Z 0','H 0 010203 - -','C 0 1 0 100000','A 0 4','P 0 32','H 0 0102030405 - -','A 0 5','N 0 7','P 0 0','H 0 aabb - -','A 0 6','P 0 32','Z 0']
|
|
utility=['Q '+' '.join(map(str,c)) for c in [(1,0,0),(32,255,100000),(0,1,0),(33,1,0),(1,-1,0),(1,256,0),(1,1,-1),(1,1,100001)]]
|
|
for size in [0,1,2,31,32,33]:
|
|
data=bytes(rng.randrange(256) for _ in range(size))
|
|
for ch,ep,seq in [(0,0,0),(255,4294967295,4294967295),(23,17,99)]:
|
|
utility+=['E '+str(ch)+' '+str(ep)+' '+str(seq)+' '+hx(data)]
|
|
packet=frame(ch,ep,seq,data)
|
|
utility+=['D '+hx(packet)]
|
|
if packet:
|
|
for i in [0,1,2,6,10,11,len(packet)-1]:
|
|
changed=bytearray(packet);changed[i]^=1;utility+=['D '+hx(changed)]
|
|
utility+=['D '+hx(packet[:-1]),'D '+hx(packet+b'\x00')]
|
|
utility+=['X '+x for x in ['-','00','aBcD','abcdefABCDEF0123456789','f','0z','+1','gg','000']]
|
|
utility+=['G '+hx(bytes(range(256))),'G -']
|
|
cat=['T 0','T 1','J 0 1','J 0 0','L 0 -1','L 0 256','L 0 0','Y 0 0','Y 0 -1','Y 0 256','B 0 -']
|
|
for ch in [255,0,17,1]:
|
|
cat+=['K 0 '+hx(frame(ch,7,1,b'ab')),'T 0','K 0 '+hx(frame(ch,7,0,b'ab')),'K 0 '+hx(frame(ch,7,0,b'ab')),'K 0 '+hx(frame(ch,7,1,b'\x00\xff')),'L 0 '+str(ch),'K 0 '+hx(frame(ch,8,1,b'a')),'K 0 '+hx(frame(ch,8,0,b'x')),'T 0']
|
|
cat+=['J 1 0','T 1','J 0 1','J 0 0','Y 1 17','T 1','L 1 17','U 1','T 1']
|
|
for index in range(32):
|
|
ch=rng.randrange(256);ep=rng.randrange(100);payload=bytes(rng.randrange(256) for _ in range(rng.randrange(1,33)))
|
|
stream=frame(ch,ep,0,payload)+frame(ch,ep,1,payload[::-1])
|
|
cat+=['U 1','B 1 '+hx(stream),'T 1','L 1 '+str(ch),'U 1','B 1 '+hx(stream+b'\x00'),'T 1','B 1 '+hx(stream[:-1]),'T 1','B 1 '+hx(frame(ch,ep,0,payload)+frame(ch,ep,2,payload)),'T 1']
|
|
cat+=['U 0','U 1','K 0 '+hx(frame(0,0,0,b'a')),'K 1 '+hx(frame(255,1,0,b'b')),'J 0 1','T 0','T 1','K 1 00','T 1','Y 0 0','Y 0 255','T 0','U 0','T 0']
|
|
cat+=['B 0 '+hx(frame(19,4,0,b'new')+b'\x00'),'T 0','K 0 '+hx(frame(3,1,0,b'a')[:-1]+b'\x00'),'T 0','B 0 '+hx(frame(3,1,0,b'a')[:-1]+b'\x00'),'T 0']
|
|
smoke=['H 0 41424344454647 - -','A 0 7','P 0 1','N 0 2','P 0 0']
|
|
callback=['H 0 0102030405 1/0/0,-1/11/0 -','A 0 5','P 0 1','N 0 31','N 0 1','P 0 0']
|
|
retry=['H 0 0102030405060708 '+','.join(['-1/4/0']*20+['2/0/0','-1/4/0'])+' 1/0/0,-1/4/0,2/0/0,-1/4/0','A 0 11','P 0 32']
|
|
return [('smoke',smoke),('callback',callback),('retry',retry),('sessions',flow),('codec',utility),('catalogue',cat)]
|
|
|
|
def digest(data):return {'n':len(data),'sha256':hashlib.sha256(data).hexdigest()[:16]}
|
|
|
|
def main():
|
|
candidate=Path(sys.argv[1])
|
|
scratch=Path(tempfile.mkdtemp(prefix='sensor-observe-',dir=Path.cwd()))
|
|
names=['include/sensor.hpp','src/codec.cpp','src/mailbox.cpp','src/session.cpp','src/catalogue.cpp']
|
|
for name in names:
|
|
p=scratch/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes((candidate/name).read_bytes())
|
|
(scratch/'observe.cpp').write_text(OBSERVER)
|
|
# Freeze every input and expected observation before the first child starts.
|
|
prepared=[(name,('\n'.join(lines)+'\n').encode(),oracle(lines)) for name,lines in scenes()]
|
|
reports=[];failures=[]
|
|
argv=['g++','-std=c++17','-O2','-Iinclude','src/codec.cpp','src/mailbox.cpp','src/session.cpp','src/catalogue.cpp','observe.cpp','-o','observe']
|
|
try:
|
|
built=subprocess.run(argv,cwd=scratch,capture_output=True,timeout=25)
|
|
reports.append({'scene':'build','exit':built.returncode,'stdout':digest(built.stdout),'stderr':digest(built.stderr)})
|
|
if built.returncode:failures.append('build')
|
|
else:
|
|
for name,data,expected in prepared:
|
|
try:
|
|
result=subprocess.run(['./observe'],cwd=scratch,input=data,capture_output=True,timeout=12)
|
|
reasons=[]
|
|
if result.returncode:reasons.append('exit')
|
|
if result.stdout!=expected:reasons.append('stdout')
|
|
if result.stderr:reasons.append('stderr')
|
|
reports.append({'scene':name,'exit':result.returncode,'stdin':digest(data),'expected':digest(expected),'stdout':digest(result.stdout),'stderr':digest(result.stderr),'reasons':reasons})
|
|
failures += [name+':'+reason for reason in reasons]
|
|
except subprocess.TimeoutExpired as error:
|
|
reports.append({'scene':name,'timeout':True,'stdout':digest(error.stdout or b''),'stderr':digest(error.stderr or b'')});failures.append(name+':timeout');break
|
|
except subprocess.TimeoutExpired:
|
|
failures.append('build:timeout')
|
|
sys.stderr.write(json.dumps({'checks':reports,'failures':failures},separators=(',',':'))+'\n')
|
|
print('FAIL '+','.join(failures) if failures else 'PASS')
|
|
return int(bool(failures))
|
|
|
|
if __name__=='__main__':raise SystemExit(main())
|