amos 73a5240295 Batches 21 and 18: add 74 calibrated tasks (40 Bos redo, 34 HWE code-gen)
Batch 21 (20261027-A-bos-redo-21) rebuilt 40 of the 53 rejected Bos-qiao
environments: hidden assertions that ran inside the candidate's process
now run in a trusted parent that reads observations from a low-UID
worker; gold bytes are unchanged. 9 were excluded for scope (ROS2,
oversize, security core), 3 are blocked because the source requires an
exec bit the platform does not freeze, 1 lacks requests/bs4 in trace6.

Batch 18 (20261026-C-hwe-codegen-18) adds its last 34 of 134 tasks; the
first 100 landed in 2f32624d.

Claude reviewed 74/74 USABLE against current digests, canonical
families, no ID or source overlap with main, plain public docs. Only the
new task directories are taken; the stale pre-batch-20 copies of
concur-release-capsule and resource-temp-link on the batch 21 branch and
both branches' progress notes stay as main has them. Difficulty is
unmeasured.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 10:23:12 +08:00

494 lines
28 KiB
Python

import copy
import hashlib
import io
import json
import random
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
CODES = ['OK', 'RANGE', 'EXISTS', 'ABSENT', 'BUSY', 'TIMEOUT', 'SYNTAX', 'LIMIT']
LO, HI = -(1 << 63), (1 << 63) - 1
FILES = ['include/eeprom.hpp', 'src/eeprom.cpp', 'src/transfer.cpp', 'src/cli.cpp', 'src/main.cpp']
class Model:
def __init__(self):
self.devices = {}
def snapshot(self, device_id):
if not 0 <= device_id <= 3:
return [1, 0, 0, 0, 0, 0, 0, [], []]
if device_id not in self.devices:
return [3, 0, 0, 0, 0, 0, 0, [], []]
d = self.devices[device_id]
return [0, d['cursor'], d['busy'], d['accepted'], d['committed'], d['polls'], d['start'], d['pending'][:], d['bytes'][:]]
def call(self, verb, device_id, *args):
values = args[0] if verb == 'W' else args[1] if verb == 'T' else []
invalid = not 0 <= device_id <= 3
if verb == 'M': invalid |= not 0 <= args[0] <= 255 or not 0 <= args[1] <= 4
if verb in ('S', 'T'): invalid |= not 0 <= args[0] <= 255
if verb == 'R': invalid |= not 0 <= args[0] <= 256
if verb in ('W', 'T'):
invalid |= len(values) > (8 if verb == 'W' else 256) or any(not 0 <= x <= 255 for x in values)
if verb == 'T': invalid |= len(values) + args[0] > 256 or not 0 <= args[2] <= 5
if invalid: return [1, 0, []]
if verb == 'M':
if device_id in self.devices: return [2, 0, []]
self.devices[device_id] = dict(cursor=0, cycle=args[1], busy=0, accepted=0, committed=0, polls=0, start=0, pending=[], bytes=[args[0]]*256)
return [0, 0, []]
if device_id not in self.devices: return [3, 0, []]
d = self.devices[device_id]
if verb in ('U', 'S', 'W', 'R', 'T') and d['pending']: return [4, 0, []]
if verb == 'U': del self.devices[device_id]
elif verb == 'S': d['cursor'] = args[0]
elif verb == 'W':
if values:
d['pending'] = list(values)
d['start'] = d['cursor']
d['busy'] = d['cycle']
d['accepted'] += 1
elif verb == 'P':
d['polls'] += 1
if d['pending']:
if d['busy']:
d['busy'] -= 1
return [4, 0, []]
length = len(d['pending'])
for i, x in enumerate(d['pending']):
d['bytes'][d['start']//8*8+(d['start']+i)%8] = x
d['cursor'] = (d['start']+length)%256
d['pending'] = []
d['start'] = 0
d['committed'] += 1
return [0, length, []]
elif verb == 'R':
result = [d['bytes'][(d['cursor']+i)%256] for i in range(args[0])]
d['cursor'] = (d['cursor']+args[0])%256
return [0, args[0], result]
elif verb == 'T':
if values:
self.call('S', device_id, args[0])
done = 0
while done < len(values):
length = min(8-(args[0]+done)%8, len(values)-done)
self.call('W', device_id, values[done:done+length])
for attempt in range(args[2]+1):
reply = self.call('P', device_id)
if reply[0] == 0: break
else: return [5, done, []]
done += length
return [0, done, []]
return [0, 0, []]
def byte_text(values):
if not values: return '-'
if any(not 0 <= x <= 255 for x in values): return '['+','.join(map(str,values))+']'
return ''.join('%02x'%x for x in values)
def reply_text(reply):
code = CODES[reply[0]] if 0 <= reply[0] < len(CODES) else 'INVALID(%d)'%reply[0]
return '%s %d %s\n'%(code,reply[1],byte_text(reply[2]))
def snapshot_text(snapshot):
code = CODES[snapshot[0]] if 0 <= snapshot[0] < len(CODES) else 'INVALID(%d)'%snapshot[0]
return code+' '+ ' '.join(map(str,snapshot[1:7]))+' '+byte_text(snapshot[7])+' '+byte_text(snapshot[8])+'\n'
CLI_COVER = set()
CLI_VERBS = set()
CLI_BY_VERB = {}
CLI_NUMERIC = {}
CLI_SYNTAX_VERBS = set()
def cli_model(data, model=None, numeric_tags=None):
if model is None: model = Model()
output = []
lines = data.split(b'\n')
if lines[-1:] == [b'']: lines.pop()
for index,line in enumerate(lines):
reply = [6,0,[]]
snapshot = False
if index >= 4096: reply = [7,0,[]]
elif len(line) <= 2048 and b'\x00' not in line and b'\r' not in line:
fields = re.findall(rb'[^ \t]+',line)
try:
verb = fields[0].decode('ascii')
arity = {'M':4,'U':2,'S':3,'W':3,'P':2,'R':3,'T':5,'I':2}[verb]
if len(fields) != arity: raise ValueError()
numeric = [1,2,3] if verb=='M' else [1,2,3] if verb=='T' else [1,2] if verb in ('S','R') else [1]
numbers = {}
for n in numeric:
if re.fullmatch(rb'-?[0-9]+',fields[n]) is None: raise ValueError()
numbers[n] = int(fields[n])
if not LO <= numbers[n] <= HI: raise ValueError()
values = []
if verb in ('W','T'):
token=fields[-1]
if token != b'-':
if not (2 <= len(token) <= 512 and len(token)%2 == 0 and re.fullmatch(rb'[0-9a-fA-F]+',token)): raise ValueError()
values = list(bytes.fromhex(token.decode('ascii')))
args = (numbers[2],numbers[3]) if verb=='M' else (numbers[2],) if verb in ('S','R') else (values,) if verb=='W' else (numbers[2],values,numbers[3]) if verb=='T' else ()
if verb=='I': reply=model.snapshot(numbers[1]);snapshot=True
else: reply=model.call(verb,numbers[1],*args)
CLI_VERBS.add(verb)
CLI_BY_VERB.setdefault(verb,set()).add(reply[0])
if numeric_tags and index in numeric_tags:
key,value=numeric_tags[index]
CLI_NUMERIC.setdefault(key,[]).append([value,reply[0]])
except (ValueError,KeyError,IndexError,UnicodeError): pass
if reply[0]==6:
initial=re.findall(rb'[^ \t]+',line)
if initial and initial[0] in [b'M',b'U',b'S',b'W',b'P',b'R',b'T',b'I']:CLI_SYNTAX_VERBS.add(initial[0].decode())
CLI_COVER.add(reply[0])
output.append(snapshot_text(reply) if snapshot else reply_text(reply))
return ''.join(output).encode()
OBSERVER = r'''#include "eeprom.hpp"
#include <array>
#include <iomanip>
#include <iostream>
#include <limits>
#include <locale>
#include <memory>
#include <sstream>
#include <string>
using namespace eeprom;
struct Grouping : std::numpunct<char> {
char do_thousands_sep() const override { return '_'; }
std::string do_grouping() const override { return "\3"; }
};
void vector_raw(const std::vector<Number>& values) {
std::cout << ' ' << values.size();
for (Number value : values) std::cout << ' ' << value;
}
void raw_reply(const Reply& reply) {
std::cout << "R " << static_cast<int>(reply.code) << ' ' << reply.completed;
vector_raw(reply.data); std::cout << '\n';
}
void raw_snapshot(const Snapshot& s) {
std::cout << "S " << static_cast<int>(s.code) << ' ' << s.cursor << ' ' << s.busy << ' ' << s.accepted << ' ' << s.committed << ' ' << s.polls << ' ' << s.pending_start;
vector_raw(s.pending); vector_raw(s.bytes); std::cout << '\n';
}
void hex_raw(const std::string& bytes) {
const char* d="0123456789abcdef";
for (unsigned char c : bytes) std::cout << d[c/16] << d[c%16];
}
int main() {
char mode; std::cin >> mode;
if (mode == 'E') {
Bus bus;
std::size_t length;
while (std::cin >> length) {
std::cin.get(); std::string data(length,'\0');
std::cin.read(&data[0], static_cast<std::streamsize>(length));
std::istringstream input(data); std::ostringstream output;
output.imbue(std::locale(std::locale::classic(),new Grouping));
output << std::hex << std::showpos << std::showbase << std::uppercase;
output.width(13); output.fill('@');
auto flags=output.flags(); auto width=output.width(); auto fill=output.fill(); auto locale=output.getloc();
int result=run_cli(bus,input,output);
std::string text=output.str();
std::cout << "C " << result << ' ' << text.size() << ' ';
hex_raw(text);
std::cout << ' ' << (flags==output.flags()) << ' ' << (width==output.width()) << ' ' << (fill==output.fill()) << ' ' << (locale==output.getloc()) << '\n';
for (Number id=0;id<4;++id) raw_snapshot(bus.inspect(id));
}
return 0;
}
if (mode == 'F') {
char kind; int code; Number completed; std::size_t count;
while (std::cin >> kind >> code >> completed >> count) {
std::vector<Number> values(count);
for (Number& n : values) std::cin >> n;
std::ostringstream output;
output.imbue(std::locale(std::locale::classic(),new Grouping));
output << std::hex << std::showpos << std::showbase << std::uppercase;
output.width(13); output.fill('@');
auto flags=output.flags(); auto width=output.width(); auto fill=output.fill(); auto locale=output.getloc();
if (kind=='R') print_reply(output,{static_cast<Code>(code),completed,values});
else {
Snapshot s; s.code=static_cast<Code>(code); s.cursor=completed;
std::cin >> s.busy >> s.accepted >> s.committed >> s.polls >> s.pending_start;
s.pending=values; std::cin >> count; s.bytes.resize(count);
for (Number& n:s.bytes) std::cin >> n;
print_snapshot(output,s);
}
std::string text=output.str();
std::cout << "F " << text.size() << ' '; hex_raw(text);
std::cout << ' ' << (flags==output.flags()) << ' ' << (width==output.width()) << ' ' << (fill==output.fill()) << ' ' << (locale==output.getloc()) << '\n';
}
return 0;
}
std::array<std::unique_ptr<Bus>,2> buses;
buses[0]=std::make_unique<Bus>(); buses[1]=std::make_unique<Bus>();
char verb; int slot; Number id;
while (std::cin >> verb >> slot >> id) {
Bus& bus=*buses[slot];
if (verb=='N') { buses[slot]=std::make_unique<Bus>(); std::cout << "N\n"; continue; }
if (verb=='I') { Snapshot copy=bus.inspect(id); raw_snapshot(copy); if (!copy.bytes.empty()) copy.bytes[0]=222; if (!copy.pending.empty()) copy.pending[0]=111; continue; }
Number first=0,second=0; std::size_t length=0; std::vector<Number> values;
if (verb=='M') { std::cin >> first >> second; raw_reply(bus.mount(id,first,second)); }
else if (verb=='U') raw_reply(bus.unmount(id));
else if (verb=='S') { std::cin >> first; raw_reply(bus.seek(id,first)); }
else if (verb=='P') raw_reply(bus.poll(id));
else if (verb=='R') { std::cin >> first; raw_reply(bus.read(id,first)); }
else {
if (verb=='T') std::cin >> first >> second;
std::cin >> length; values.resize(length);
for (Number& value:values) std::cin >> value;
if (verb=='T') raw_reply(bus.transfer(id,first,values,second));
else raw_reply(bus.page(id,values));
for (Number& value:values) value=222;
}
}
return 0;
}
'''
def raw_reply(reply):
return 'R '+' '.join(map(str,reply[:2]))+' '+str(len(reply[2]))+(' '+' '.join(map(str,reply[2])) if reply[2] else '')+'\n'
def raw_snapshot(snapshot):
def vec(v): return ' '+str(len(v))+(' '+' '.join(map(str,v)) if v else '')
return 'S '+' '.join(map(str,snapshot[:7]))+vec(snapshot[7])+vec(snapshot[8])+'\n'
class NativeCases:
def __init__(self):
self.models=[Model(),Model()]; self.lines=['D']; self.outputs=[]; self.counts={}; self.numeric={}
def call(self,verb,slot,device_id,*args,observe=True):
if verb=='N': self.models[slot]=Model(); reply=None; line=f'N {slot} {device_id}'; out='N\n'
elif verb=='I':
reply=self.models[slot].snapshot(device_id);line=f'I {slot} {device_id}';out=raw_snapshot(reply)
else:
reply=self.models[slot].call(verb,device_id,*args)
prefix=f'{verb} {slot} {device_id}'
if verb in ('M','S','R'): line=prefix+' '+' '.join(map(str,args))
elif verb in ('W','T'):
values=args[0] if verb=='W' else args[1]
line=prefix+(f' {args[0]} {args[2]}' if verb=='T' else '')+' '+str(len(values))+(' '+' '.join(map(str,values)) if values else '')
else: line=prefix
out=raw_reply(reply)
self.lines.append(line);self.outputs.append(out)
self.counts[verb]=self.counts.get(verb,0)+1
if observe and verb not in ('I','N'):
for s in range(2):
for i in range(4): self.call('I',s,i,observe=False)
return reply
def setup(self,cycle=0):
for slot in range(2):
self.call('N',slot,0);self.call('M',slot,0,73+slot,cycle);self.call('M',slot,3,171-slot,cycle)
self.call('T',slot,0,0,[1,2,3,4],5)
def result(self): return ('\n'.join(self.lines)+'\n').encode(),''.join(self.outputs).encode()
def make_cases():
CLI_COVER.clear();CLI_VERBS.clear();CLI_BY_VERB.clear();CLI_NUMERIC.clear();CLI_SYNTAX_VERBS.clear()
n=NativeCases(); n.setup()
# Every public method, both instances and populated error backgrounds.
for device_id in [0,3]:
n.call('U',0,device_id);n.call('M',0,device_id,255,4)
for address in [0,255]: n.call('S',0,device_id,address);n.call('R',0,device_id,1)
for count in [0,256]: n.call('R',0,device_id,count)
n.call('S',0,device_id,7);n.call('W',0,device_id,[0,255,4,5,6,7,8,9])
for _ in range(5):n.call('P',0,device_id)
n.call('W',0,device_id,[]);n.call('P',0,device_id)
n.call('T',0,device_id,0,[0]*256,5);n.call('S',0,device_id,0);n.call('R',0,device_id,256)
n.call('T',0,device_id,255,[255],4);n.call('S',0,device_id,255);n.call('R',0,device_id,1)
n.call('T',0,device_id,255,[],0);n.call('T',0,device_id,0,[],5)
for cycle in [0,4]:
for budget in [0,5]:
n.setup(cycle);n.call('T',0,0,6,[1,2,3,4,5,6,7,8,9],budget)
for _ in range(6):n.call('P',0,0)
n.call('S',0,0,0);n.call('R',0,0,16)
# Numeric parameter domains: success at each endpoint and actual effects above;
# neighbors and native extremes are invoked independently for each argument.
domains={'id':(0,3),'fill':(0,255),'cycle':(0,4),'address':(0,255),'count':(0,256),'budget':(0,5),'byte':(0,255)}
probes={'M':['id','fill','cycle'],'U':['id'],'S':['id','address'],'W':['id','byte'],'P':['id'],'R':['id','count'],'T':['id','address','byte','budget'],'I':['id']}
for verb,parameters in probes.items():
for param_index,param in enumerate(parameters):
low,high=domains[param]
for value in sorted(set([LO,low-1,low,low+1,high-1,high,high+1,HI])):
n.setup()
device_id=0
args={'M':[13,0],'U':[],'S':[11],'W':[[8]],'P':[],'R':[2],'T':[0,[9],5],'I':[]}[verb]
if param=='id':device_id=value
elif verb=='M':args[param_index-1]=value
elif param=='byte':args[0 if verb=='W' else 1]=[value]
elif verb=='T':args[0 if param=='address' else 2]=value
else:args[0]=value
if verb=='M' and 0<=device_id<=3:n.call('U',0,device_id)
result=n.call(verb,0,device_id,*args)
n.numeric.setdefault(verb+'.'+param,[]).append([value,result[0]])
if verb=='M' and result[0]==0:
n.call('W',0,device_id,[0,255])
for _ in range(6):n.call('P',0,device_id)
n.call('S',0,device_id,0);n.call('R',0,device_id,2)
if verb in ('W','T'):
for _ in range(6):n.call('P',0,device_id)
if 0<=device_id<=3:n.call('S',0,device_id,0);n.call('R',0,device_id,256)
# Parameter precedence, absence, pending state, zero count on nonzero bytes.
for pending in [False,True]:
n.setup(4)
if pending:n.call('W',0,0,[31,32])
for verb,args in [('M',(-1,0)),('M',(0,9)),('S',(-1,)),('R',(-1,)),('W',([-1],)),('W',([4]*9,)),('T',(255,[1,2],5)),('T',(0,[1],-1))]:
for device_id in [0,1,LO,HI]:n.call(verb,0,device_id,*args)
for verb,args in [('U',()),('S',(9,)),('R',(0,)),('W',([],)),('T',(0,[],5))]:
for device_id in [0,1]:n.call(verb,0,device_id,*args)
n.call('I',0,LO);n.call('I',0,HI)
n.setup();n.call('T',0,0,0,[7]*257,5);n.call('T',0,0,255,[0],0);n.call('S',0,0,255);n.call('R',0,0,1)
rng=random.Random(120110)
for iteration in range(180):
if iteration%25==0:n.setup(rng.randrange(5))
slot=rng.randrange(2);device_id=rng.choice([0,1,3]);verb=rng.choice(['M','U','S','W','P','R','T'])
args={'M':[rng.randrange(256),rng.randrange(5)],'U':[],'S':[rng.randrange(256)],'W':[[rng.randrange(256) for _ in range(rng.randrange(10))]],'P':[],'R':[rng.randrange(257)],'T':[rng.randrange(256),[rng.randrange(256) for _ in range(rng.randrange(18))],rng.randrange(6)]}[verb]
n.call(verb,slot,device_id,*args)
for verb,maximum in [('W',8),('T',256)]:
for length in [0,maximum,maximum+1]:
n.setup();args=([0]*length,) if verb=='W' else (0,[0]*length,5)
result=n.call(verb,0,0,*args);n.numeric.setdefault(verb+'.length',[]).append([length,result[0]])
for _ in range(6):n.call('P',0,0)
n.call('S',0,0,0);n.call('R',0,0,256)
for key,results in n.numeric.items():
param=key.split('.')[1];low,high=(0,8 if key.startswith('W') else 256) if param=='length' else domains[param]
assert {v for v,c in results if c==0}.issuperset({low,high}),(key,results)
assert all(c==1 for v,c in results if not low<=v<=high),(key,results)
native=n.result()
# CLI all verbs and all reachable error codes in populated states.
cli=[b'M 0 73 0',b'M 3 171 4',b'T 0 0 0 01020304',b'M 0 5 0',b'I 0',b'M 0 -1 0',b'U 1',b'W 3 aabb',b'R 3 0',b'T 3 0 0 -',b'I 3',b'P 3',b'P 3',b'P 3',b'P 3',b'P 3',b'T 3 7 0 0102',b'I 3',b'P 3',b'P 3',b'P 3',b'P 3',b'P 3',b'S 0 0',b'R 0 4',b'W 0 -',b'P 0',b'U 0',b'M 0 255 0',b'W 0 00',b'P 0',b'S 0 0',b'R 0 1']
forms={'M':[b'0',b'9',b'0'],'U':[b'0'],'S':[b'0',b'11'],'W':[b'0',b'00'],'P':[b'0'],'R':[b'0',b'1'],'T':[b'0',b'0',b'5',b'00'],'I':[b'0']}
for verb,args in forms.items():
good=verb.encode()+b' '+b' '.join(args)
for line in [good,b' '.join([verb.encode()]+args[:-1]),good+b' x']:
cli.extend([line,b'I 0',b'I 3'])
for index in range(len(args)):
broken=args[:];broken[index]=b'bad';cli.extend([verb.encode()+b' '+b' '.join(broken),b'I 0',b'I 3'])
cli.extend([b'I 0',b'I 3'])
tags={}
def prepare_cli():
for device_id in [0,3]:
cli.extend([f'P {device_id}'.encode()]*6)
cli.append(f'U {device_id}'.encode())
cli.append(f'M {device_id} {73+device_id} 0'.encode())
cli.append(b'T 0 0 0 01020304')
for verb,parameters in probes.items():
for param_index,param in enumerate(parameters):
low,high=domains[param]
for value in sorted(set([LO,low-1,low,low+1,high-1,high,high+1,HI])):
if param=='byte' and not 0<=value<=255:continue
prepare_cli()
args=forms[verb][:]
if param=='byte':args[-1]=('%02x'%value).encode()
else:args[2 if verb=='T' and param=='budget' else param_index]=str(value).encode()
if verb=='M':
target=value if param=='id' else 0
if 0<=target<=3:cli.append(f'U {target}'.encode())
tags[len(cli)]=(verb+'.'+param,value)
cli.append(verb.encode()+b' '+b' '.join(args));cli.extend([b'I 0',b'I 3'])
if verb=='M':
target=value if param=='id' else 0
if 0<=target<=3 and low<=value<=high:
cli.append(f'W {target} 00ff'.encode())
for _ in range(6):cli.extend([f'P {target}'.encode(),f'I {target}'.encode()])
for verb,maximum in [('W',8),('T',256)]:
for length in [0,maximum]:
prepare_cli();token=b'-' if length==0 else b'00'*length
tags[len(cli)]=(verb+'.length',length)
cli.append((b'W 0 ' if verb=='W' else b'T 0 0 5 ')+token)
cli.extend([b'I 0',b'I 3'])
prepare_cli()
for line in [b'M 1 19 4',b'W 1 0102',b'U 1',b'S 1 3',b'W 1 -',b'R 1 0',b'T 1 0 5 -',b'P 1']:
cli.extend([line,b'I 1',b'I 0',b'I 3'])
cli.extend([b'W 0 '+b'01'*9,b'T 0 255 0 0001'])
cli.extend([b'U 2',b'I 2',b'P 2',b'W 2 00',b'S 2 0',b'R 2 0',b'T 2 0 0 -',b'I 0',b'I 3'])
cli.extend([b'U 2',b'M 2 33 0',b'W 2 00ff010203040506',b'P 2',b'T 2 0 0 '+b'00'*256,b'S 2 0',b'R 2 256',b'I 2'])
cli.extend([b'',b'unknown',b'S +0 0',b'S 0 9223372036854775808',b'S 0 -9223372036854775809',b'W 0 0',b'W 0 gg',b'W 0 '+b'00'*257,b'R 0 00',b'S -0 000',b'R 0 0\r',b'P 0\x00',b' '*2045+b'P 0',b' '*2046+b'P 0',b'I 0',b'I 3'])
cli.extend([b'\tW\t0\tA0fF\t',b'P 0',b'S 0 0',b'R 0 2',b'm 0 1 0',b'I 0',b'I 3'])
cli_data=b'\n'.join(cli) # no final LF
cli_expected=cli_model(cli_data,numeric_tags=tags)
# Mixed physical-line prefix includes successful calls, syntax and API errors.
prefix=[b'M 0 73 0',b'M 3 171 4',b'T 0 0 0 01020304']
mixed=[b'',b'NO',b'R',b'R x 1',b'W 0 '+b'00'*257,b'P 0\x00',b'R 0 -1',b'I 0',b'P 3']
while len(prefix)<4095:prefix.append(mixed[(len(prefix)-3)%len(mixed)])
limit_data=b'\n'.join(prefix+[b'T 0 255 0 00',b'S 0 0',b'I 0'])
limit_expected=cli_model(limit_data)
smoke=b'M 0 90 0\nT 0 0 0 010203\nS 0 0\nR 0 2\nR 0 1\n'
smoke_expected=cli_model(smoke)
# Repeated same-process entry calls, including empty and errors followed by fresh IDs.
entry_sets=[smoke,b'',b'I 0\n',cli_data,b'M 0 17 4\nW 0 00\nI 0\n',b'M 0 99 0\nI 0',b'\n',b'P 0\n',limit_data,b'I 0\n',b'S 0 255\nR 0 1\n']
entry_data=b'E\n';entry_expected=b'';entry_model=Model()
for data in entry_sets:
result=cli_model(data,entry_model);entry_data+=str(len(data)).encode()+b'\n'+data+b'\n'
entry_expected+=b'C 0 '+str(len(result)).encode()+b' '+result.hex().encode()+b' 1 1 1 1\n'
entry_expected+=''.join(raw_snapshot(entry_model.snapshot(i)) for i in range(4)).encode()
formatter_lines=['F'];formatter_expected=[]
for code in list(range(8))+[-(1<<31),-1,99,(1<<31)-1]:
for number_value in [LO,-1,0,1,HI]:
values=[0,255] if number_value==0 else [number_value,31]
formatter_lines.append('R %d %d %d %s'%(code,number_value,len(values),' '.join(map(str,values))))
text=reply_text([code,number_value,values]).encode()
formatter_expected.append('F %d %s 1 1 1 1\n'%(len(text),text.hex()))
s=[code,number_value,HI,LO,-1,1,0,values,[0,255]]
formatter_lines.append('S %d %d %d %s %s 2 0 255'%(code,number_value,len(values),' '.join(map(str,values)),' '.join(map(str,s[2:7]))))
text=snapshot_text(s).encode();formatter_expected.append('F %d %s 1 1 1 1\n'%(len(text),text.hex()))
for field in range(1,7):
for value in [LO,-1,0,1,HI]:
snapshot=[0,7,8,9,10,11,12,[],[]];snapshot[field]=value
formatter_lines.append('S 0 %d 0 %s 0'%(snapshot[1],' '.join(map(str,snapshot[2:7]))))
text=snapshot_text(snapshot).encode();formatter_expected.append('F %d %s 1 1 1 1\n'%(len(text),text.hex()))
expected_codes={'M':{0,1,2},'U':{0,1,3,4},'S':{0,1,3,4},'W':{0,1,3,4},'P':{0,1,3,4},'R':{0,1,3,4},'T':{0,1,3,4,5},'I':{0,1,3}}
assert CLI_BY_VERB==expected_codes,CLI_BY_VERB
for key,results in CLI_NUMERIC.items():
param=key.split('.')[1];low,high=(0,8 if key.startswith('W') else 256) if param=='length' else domains[param]
assert {v for v,c in results if c==0}.issuperset({low,high}),(key,results)
assert all(c==1 for v,c in results if not low<=v<=high),(key,results)
assert set(CLI_NUMERIC)==set(n.numeric),(set(CLI_NUMERIC),set(n.numeric))
assert len(cli)<=4096,len(cli)
assert CLI_COVER==set(range(8)),CLI_COVER
assert CLI_VERBS==set(forms),CLI_VERBS
assert CLI_SYNTAX_VERBS==set(forms),CLI_SYNTAX_VERBS
scenes=[('native','observer',native[0],native[1]),('entry','observer',entry_data,entry_expected),('format','observer',('\n'.join(formatter_lines)+'\n').encode(),''.join(formatter_expected).encode()),('cli','app',cli_data,cli_expected),('limit','app',limit_data,limit_expected),('smoke','app',smoke,smoke_expected),('empty','app',b'',b'')]
facts={'native_commands':len(n.lines)-1,'native_api':n.counts,'numeric_parameters':n.numeric,'cli_syntax_verbs':sorted(CLI_SYNTAX_VERBS),'cli_codes':sorted(CLI_COVER),'cli_codes_by_verb':{k:sorted(v) for k,v in CLI_BY_VERB.items()},'cli_numeric_parameters':CLI_NUMERIC,'cli_verbs':sorted(CLI_VERBS),'cli_physical_lines':len(cli),'entry_calls':len(entry_sets),'scenes':[{ 'name':name,'stdin_bytes':len(data),'expected_bytes':len(expected)} for name,exe,data,expected in scenes]}
return scenes,facts
def signature(data): return [len(data),hashlib.sha256(data).hexdigest()[:16]]
def evaluate(candidate):
scenes,facts=make_cases()
root=Path(tempfile.mkdtemp(prefix='eeprom-',dir=Path.cwd()))
for name in FILES:
target=root/name;target.parent.mkdir(parents=True,exist_ok=True);target.write_bytes((candidate/name).read_bytes())
(root/'observer.cpp').write_text(OBSERVER)
cwd=str(root)
argv=[['g++','-std=c++17','-O0','-I','include','src/eeprom.cpp','src/transfer.cpp','src/cli.cpp','observer.cpp','-o','observer'],['g++','-std=c++17','-O0','-I','include','src/eeprom.cpp','src/transfer.cpp','src/cli.cpp','src/main.cpp','-o','app'],['./observer'],['./app']]
meta={'v':1,'cwd':cwd,'argv':argv,'build':[],'scene':[],'why':[]}
# No trusted file or directory access occurs after this point.
def execute(args,data,seconds):
process=subprocess.Popen(args,cwd=cwd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
timeout=False
try:stdout,stderr=process.communicate(data,timeout=seconds)
except subprocess.TimeoutExpired:
timeout=True;process.kill();stdout,stderr=process.communicate()
return process.pid,process.returncode,timeout,stdout,stderr
builds_ok=True
for index in range(2):
pid,code,timed,out,err=execute(argv[index],b'',25)
meta['build'].append([index,pid,code,timed,signature(b''),signature(b''),signature(out),signature(err)])
if code or timed:builds_ok=False;meta['why'].append('build%d'%index)
if builds_ok:
for name,exe,data,expected in scenes:
index=2 if exe=='observer' else 3
pid,code,timed,out,err=execute(argv[index],data,12)
meta['scene'].append([name,index,pid,code,timed,signature(data),signature(expected),signature(out),signature(err)])
if timed:meta['why'].append(name+':timeout')
if code:meta['why'].append(name+':exit')
if out!=expected:meta['why'].append(name+':stdout')
if err:meta['why'].append(name+':stderr')
encoded=json.dumps(meta,separators=(',',':'))
assert len(encoded.encode())<3900,len(encoded)
sys.stderr.write(encoded+'\n')
print('FAIL' if meta['why'] else 'PASS')
return 1 if meta['why'] else 0
if __name__=='__main__':
sys.exit(evaluate(Path(sys.argv[1])))