amos 2f32624d3b Batch 18 (20261026-C-hwe-codegen-18): add 100 calibrated HWE code-gen tasks; pause production for rollout
100 tasks from the HWE arm-linux code-gen pool (outside the lk/man/pth
range given to batch 22), each with at least one independent
contract-edge defect and plain public docs. Claude reviewed 100/100
USABLE against current digests; no ID or source overlap with main or
batch 22. Difficulty is unmeasured.

progress.md records the wrap-up: all Codex sessions and Qwen schedulers
stopped for the user's rollout, Qwen results so far (57 HARD of 373),
and what is left on each batch branch.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 17:52:13 +08:00

471 lines
21 KiB
Python

import hashlib
import json
from pathlib import Path
import random
import re
import subprocess
import sys
import tempfile
ROOT = Path(__file__).resolve().parent
OBSERVER = r'''#include "record.hpp"
#include <array>
#include <iostream>
#include <memory>
#include <sstream>
#include <string>
using namespace inline_record;
int reuse() {
std::string line;
std::getline(std::cin,line);
const std::size_t calls=std::stoull(line);
for (std::size_t index=0; index<calls; ++index) {
std::getline(std::cin,line);
const std::size_t length=std::stoull(line);
std::string data(length,'\0');
std::cin.read(data.data(),static_cast<std::streamsize>(length));
std::istringstream supplied(data);
std::ostringstream output,diagnostic;
auto* original_input=std::cin.rdbuf(supplied.rdbuf());
auto* original_output=std::cout.rdbuf(output.rdbuf());
auto* original_error=std::cerr.rdbuf(diagnostic.rdbuf());
std::cin.clear(); std::cout.clear(); std::cerr.clear();
const int returned=run_cli();
std::cout.flush(); std::cerr.flush();
std::cin.rdbuf(original_input);
std::cout.rdbuf(original_output);
std::cerr.rdbuf(original_error);
std::cin.clear(); std::cout.clear(); std::cerr.clear();
const std::string out=output.str(),err=diagnostic.str();
std::cout << "C " << returned << ' ' << out.size() << ' ' << err.size() << '\n';
std::cout.write(out.data(),static_cast<std::streamsize>(out.size()));
std::cout.write(err.data(),static_cast<std::streamsize>(err.size()));
}
return 0;
}
int main(int argc,char** argv) {
if (argc==2 && std::string(argv[1])=="reuse") return reuse();
std::array<std::unique_ptr<Store>,2> stores;
stores[0].reset(new Store);
stores[1].reset(new Store);
std::string line;
while (std::getline(std::cin,line)) {
std::istringstream input(line);
int object=0;
std::string verb;
input >> object >> verb;
std::array<std::int64_t,5> a{};
for (auto& value:a) { if (!(input >> value)) break; }
if (verb=="RESET") {
stores[object].reset(new Store);
std::cout << "N\n";
continue;
}
Store& s=*stores[object];
if (verb=="TOTAL") {
const auto t=s.totals();
std::cout << "T " << t.allocations << ' ' << t.frees << ' ' << t.live
<< ' ' << t.bytes << ' ' << t.next << '\n';
continue;
}
Result r;
if (verb=="REPLACE") r=s.replace(a[0],a[1],a[2],a[3]);
else if (verb=="FILL") r=s.fill(a[0],a[1],a[2],a[3]);
else if (verb=="COPY") r=s.copy(a[0],a[1],a[2],a[3],a[4]);
else if (verb=="DROP") r=s.drop(a[0]);
else if (verb=="INFO") r=s.info(a[0]);
else if (verb=="GET") r=s.get(a[0],a[1]);
else if (verb=="BYTE") r=s.byte(a[0],a[1]);
std::cout << "R " << r.error.size() << ' ' << (r.error.empty()?"OK":r.error) << ' ' << r.token
<< ' ' << r.count << ' ' << r.tag << ' ' << r.bytes
<< ' ' << r.checksum << ' ' << r.value << '\n';
}
return 0;
}
'''
class Model:
def __init__(self):
self.records = {}
self.allocations = self.frees = 0
self.next = 1
def wire(self, rec):
token, tag, values = rec
checksum = sum(values) & 0xffffffff
fields = b'REC1' + token.to_bytes(4, 'little')
fields += tag.to_bytes(2, 'little') + len(values).to_bytes(2, 'little')
fields += checksum.to_bytes(4, 'little')
return fields + b''.join((v & 0xffffffff).to_bytes(4, 'little') for v in values)
def call(self, verb, a):
if verb == 'RESET':
self.__init__()
return 'N', [0] * 6
if verb == 'TOTAL':
sizes = [16 + len(r[2]) * 4 for r in self.records.values()]
return 'T', [self.allocations, self.frees, len(sizes), sum(sizes), self.next]
domains = {
'REPLACE': [(0,7,'BAD_ID'),(0,64,'BAD_SIZE'),(0,65535,'BAD_TAG'),(16,272,'BAD_BUDGET')],
'FILL': [(0,7,'BAD_ID'),(0,64,'BAD_START'),(0,64,'BAD_COUNT'),(-1000000,1000000,'BAD_VALUE')],
'COPY': [(0,7,'BAD_ID'),(0,7,'BAD_SOURCE'),(0,64,'BAD_FROM'),(0,64,'BAD_TO'),(0,64,'BAD_COUNT')],
'DROP': [(0,7,'BAD_ID')], 'INFO': [(0,7,'BAD_ID')],
'GET': [(0,7,'BAD_ID'),(0,63,'BAD_INDEX')],
'BYTE': [(0,7,'BAD_ID'),(0,271,'BAD_OFFSET')],
}
zeros = [0] * 6
for value, (low, high, error) in zip(a, domains[verb]):
if not low <= value <= high:
return error, zeros
id = a[0]
if verb == 'REPLACE':
_, n, tag, budget = a
if budget < 16 + 4 * n:
return 'NOMEM', zeros
if id in self.records:
self.frees += 1
self.records[id] = [self.next, tag, [0] * n]
self.allocations += 1
self.next += 1
return 'OK', zeros
if id not in self.records:
return 'ABSENT', zeros
record = self.records[id]
values = record[2]
if verb == 'FILL':
_, start, count, value = a
if start > len(values) or count > len(values) - start:
return 'RANGE', zeros
values[start:start + count] = [value] * count
elif verb == 'COPY':
_, src, start, to, count = a
if src not in self.records:
return 'NO_SOURCE', zeros
source = self.records[src][2]
if start > len(source) or count > len(source) - start:
return 'RANGE', zeros
if to > len(values) or count > len(values) - to:
return 'RANGE', zeros
values[to:to + count] = source[start:start + count]
elif verb == 'DROP':
del self.records[id]
self.frees += 1
elif verb == 'INFO':
return 'OK', [record[0],len(values),record[1],16+4*len(values),sum(values)&0xffffffff,0]
elif verb == 'GET':
if a[1] >= len(values):
return 'RANGE', zeros
return 'OK', [0,0,0,0,0,values[a[1]]]
elif verb == 'BYTE':
wire = self.wire(record)
if a[1] >= len(wire):
return 'RANGE', zeros
return 'OK', [0,0,0,0,0,wire[a[1]]]
return 'OK', zeros
def api_scene():
stores = [Model(), Model()]
lines, outputs = [], []
observing = [False]
def emit(object, verb, *args):
code, fields = stores[object].call(verb, args)
lines.append(' '.join(map(str, (object, verb, *args))))
if code == 'N':
outputs.append('N')
elif code == 'T':
outputs.append('T ' + ' '.join(map(str,fields)))
else:
outputs.append('R ' + str(0 if code == 'OK' else len(code.encode())) + ' ' + code + ' ' + ' '.join(map(str, fields)))
if code not in ('OK','T','N') and not observing[0]:
observe(0); observe(1)
return code
def observe(object):
previous = observing[0]
observing[0] = True
emit(object, 'TOTAL')
for id in range(8):
emit(object, 'INFO', id)
if id in stores[object].records:
n = len(stores[object].records[id][2])
for index in range(n):
emit(object, 'GET', id, index)
for offset in range(16 + 4 * n):
emit(object, 'BYTE', id, offset)
observing[0] = previous
for object in range(2):
emit(object,'TOTAL')
for id in range(8):
emit(object,'REPLACE',id,3,object*100+id,28)
emit(object,'FILL',id,0,3,17+object*19+id)
observe(object)
# Both live objects have the same IDs with different tags, data and tokens.
emit(0,'REPLACE',0,64,0,272)
emit(0,'REPLACE',7,64,65535,272)
for value in (-1000000,1000000,0):
emit(0,'FILL',0,0,64,value)
observe(0)
emit(0,'FILL',0,64,0,1000000)
emit(0,'FILL',7,0,64,-1000000)
emit(0,'COPY',7,0,0,0,64)
emit(0,'COPY',0,7,64,64,0)
emit(0,'FILL',0,0,1,19)
emit(0,'COPY',0,0,0,1,63)
emit(0,'COPY',0,0,1,0,63)
observe(0); observe(1)
defaults = {'REPLACE':[0,3,12,28], 'FILL':[0,0,1,5],
'COPY':[0,7,0,0,1], 'DROP':[0], 'INFO':[0],
'GET':[0,0], 'BYTE':[0,0]}
domains = {'REPLACE':[(0,7),(0,64),(0,65535),(16,272)],
'FILL':[(0,7),(0,64),(0,64),(-1000000,1000000)],
'COPY':[(0,7),(0,7),(0,64),(0,64),(0,64)],
'DROP':[(0,7)], 'INFO':[(0,7)], 'GET':[(0,7),(0,63)],
'BYTE':[(0,7),(0,271)]}
for verb, bounds in domains.items():
for index,(low,high) in enumerate(bounds):
for bad in (low-1,high+1,-2**63,2**63-1):
args=defaults[verb].copy(); args[index]=bad
emit(0,verb,*args)
observe(0); observe(1)
# Multiple simultaneous invalid arguments exercise each validation position.
for first in range(len(bounds)):
args=defaults[verb].copy()
for index in range(first,len(bounds)):
args[index]=bounds[index][1]+1
emit(0,verb,*args); observe(0)
# Successful lower and upper ID endpoints for every read and mutation API.
for id in (0,7):
emit(0,'REPLACE',id,64,65535,272)
emit(0,'FILL',id,0,64,678)
for index in (0,63): emit(0,'GET',id,index)
for offset in (0,271): emit(0,'BYTE',id,offset)
emit(0,'INFO',id)
emit(0,'DROP',id)
observe(0)
emit(0,'DROP',id); observe(0)
for verb,args in [('INFO',[id]),('GET',[id,0]),('BYTE',[id,0]),
('FILL',[id,0,0,0]),('COPY',[id,3,0,0,0])]:
emit(0,verb,*args); observe(0)
for verb,args in [('GET',[id,64]),('BYTE',[id,272]),
('FILL',[id,65,0,0]),('FILL',[id,0,65,0]),
('FILL',[id,0,0,1000001]),('COPY',[id,8,0,0,0]),
('COPY',[id,3,65,0,0]),('COPY',[id,3,0,65,0]),
('COPY',[id,3,0,0,65]),('COPY',[id,id,0,0,0]),
('COPY',[3,id,65,0,0]),('COPY',[id,3,64,0,0])]:
emit(0,verb,*args); observe(0); observe(1)
emit(0,'REPLACE',0,3,65535,28)
emit(0,'FILL',0,0,3,71)
emit(0,'COPY',0,7,0,0,0); observe(0)
for verb,args in [('REPLACE',[0,4,1,31]),('FILL',[0,4,0,0]),
('FILL',[0,3,1,0]),('COPY',[0,3,4,0,0]),
('COPY',[0,3,0,4,0]),('COPY',[0,3,3,0,1]),
('COPY',[0,3,0,3,1]),('GET',[0,3]),('BYTE',[0,28]),
('FILL',[0,64,0,1000001]),('COPY',[0,3,64,0,65])]:
emit(0,verb,*args); observe(0); observe(1)
# The zero-capacity record is a live allocation, including replacement and drop.
for tag in (0,65535):
emit(0,'REPLACE',0,0,tag,16); observe(0); observe(1)
emit(0,'FILL',0,0,0,0)
emit(0,'COPY',0,0,0,0,0)
emit(0,'GET',0,0); observe(0)
emit(0,'BYTE',0,16); observe(0)
emit(0,'DROP',0); observe(0)
rng=random.Random(1001826)
for index in range(120):
object=index%2; id=rng.randrange(8)
n=rng.randrange(65)
emit(object,'REPLACE',id,n,rng.randrange(65536),16+4*n)
emit(object,'FILL',id,0,n,rng.randrange(-1000000,1000001))
if n: emit(object,'GET',id,n-1)
observe(object)
for object in (0,1,0,1):
emit(object,'RESET'); emit(object,'TOTAL')
emit(object,'REPLACE',7,0,65535,16)
observe(object)
return ('\n'.join(lines)+'\n').encode(), ('\n'.join(outputs)+'\n').encode()
def cli_expected(data):
model=Model(); outputs=[]
if not data: return b''
for count,line in enumerate(data.split(b'\n')[:-1] if data.endswith(b'\n') else data.split(b'\n'),1):
if count>4096: outputs.append('ERR LIMIT'); continue
if len(line)>256: outputs.append('ERR SYNTAX'); continue
tokens=re.split(br'[ \t]+',line.strip(b' \t'))
arities={'REPLACE':4,'FILL':4,'COPY':5,'DROP':1,'INFO':1,'GET':2,'BYTE':2,'TOTAL':0}
verb=tokens[0].decode('ascii','replace')
if verb not in arities or len(tokens)!=arities[verb]+1:
outputs.append('ERR SYNTAX'); continue
args=[]; valid=True
for token in tokens[1:]:
if not re.fullmatch(br'-?[0-9]+',token): valid=False; break
value=int(token)
if not -2**63<=value<2**63: valid=False; break
args.append(value)
if not valid: outputs.append('ERR SYNTAX'); continue
code,fields=model.call(verb,args)
if code=='T': outputs.append('TOTAL '+' '.join(map(str,fields)))
elif code!='OK': outputs.append('ERR '+code)
elif verb=='INFO': outputs.append('INFO '+' '.join(map(str,fields[:5])))
elif verb=='GET': outputs.append('ITEM '+str(fields[5]))
elif verb=='BYTE': outputs.append('BYTE '+str(fields[5]))
else: outputs.append('OK')
return ('\n'.join(outputs)+'\n').encode()
def cli_scene():
lines=['REPLACE 0 3 65535 28','REPLACE 7 3 0 28','FILL 0 0 3 -777',
'FILL 7 0 3 0','COPY 7 0 0 0 3','INFO 0','GET 7 2','BYTE 7 27','TOTAL']
defaults={'REPLACE':['7','3','65535','28'], 'FILL':['7','0','3','0'],
'COPY':['7','7','0','0','3'],'DROP':['0'],'INFO':['7'],
'GET':['7','2'],'BYTE':['7','27'],'TOTAL':[]}
def after():
lines.append('TOTAL')
for id in (0,7):
lines.append('INFO '+str(id))
lines.extend('GET '+str(id)+' '+str(i) for i in range(3))
lines.extend('BYTE '+str(id)+' '+str(i) for i in range(28))
for verb,args in defaults.items():
lines.append(' '.join([verb]+args[:-1]) if args else '')
after()
lines.append(' '.join([verb]+args+['0'])); after()
for index in range(len(args)):
altered=args.copy(); altered[index]='x'
lines.append(' '.join([verb]+altered)); after()
for bad in ('+1','1x','1\r','9223372036854775808','-9223372036854775809','-'):
lines.append('GET 7 '+bad); after()
for malformed in ('unknown',' ', 'x'*257, 'TOTAL\r'):
lines.append(malformed); after()
lines += ['TOTAL'+' '*251,'\tINFO\t007\t','FILL 7 -0 01 0000','GET 7 0','TOTAL',
'DROP 0','REPLACE 0 0 0 16','INFO 0','TOTAL']
assert len(lines) <= 4096
return ('\n'.join(lines)+'\n').encode()
def reuse_scene():
prefix=(b'REPLACE 0 2 256 24\nFILL 0 0 2 71\n'
b'REPLACE 7 2 65535 24\nFILL 7 0 2 -19\n')
after=b'TOTAL\n'
for id in (0,7):
after+=('INFO '+str(id)+'\n').encode()
after+=b''.join(('GET %d %d\n'%(id,i)).encode() for i in range(2))
after+=b''.join(('BYTE %d %d\n'%(id,i)).encode() for i in range(24))
# One observer process invokes run_cli repeatedly; all expectations remain in Python.
cases=[b'TOTAL\n'*4095+b'REPLACE 7 2 65535 24\nFILL 7 0 2 1\n',
b'TOTAL\n'+prefix+after,b'',b'REPLACE 7 0 65535 16\nINFO 7\nTOTAL']
defaults={'REPLACE':[7,2,65535,24],'FILL':[7,0,2,0],
'COPY':[7,0,0,0,2],'DROP':[0],'INFO':[7],'GET':[7,1],'BYTE':[7,23]}
domains={'REPLACE':[(0,7),(0,64),(0,65535),(16,272)],
'FILL':[(0,7),(0,64),(0,64),(-1000000,1000000)],
'COPY':[(0,7),(0,7),(0,64),(0,64),(0,64)],
'DROP':[(0,7)],'INFO':[(0,7)],'GET':[(0,7),(0,63)],'BYTE':[(0,7),(0,271)]}
for verb,args in defaults.items():
for position,(low,high) in enumerate(domains[verb]):
bads=[b'+1',b'1x',b'-',b'9223372036854775808',b'-9223372036854775809',
b'-9223372036854775808',b'9223372036854775807',str(low-1).encode(),str(high+1).encode(),
b'1\r',b'1\x00',b'1\x0b',b'1\x0c',b'1\x7f',b'1\x80',b'1\xff']
for bad in bads:
tokens=[str(a).encode() for a in args];tokens[position]=bad
command=verb.encode()+b' '+b' '.join(tokens)+b'\n'
cases.append(prefix+command+after)
invalid_lines=[b'',b'UNKNOWN',b'REPLACE 7 2',b'TOTAL 0',b'GET 7 x',b'x'*257,b'GET 7 1\x00']
prefix_lines=invalid_lines+[b'REPLACE 7 2 7 24',b'FILL 7 0 2 71',b'GET 7 2',
b'REPLACE 8 2 1 24',b'REPLACE 7 3 0 27',b'DROP 0',b'COPY 7 0 0 0 1']
mixed=b''.join(line+b'\n' for line in prefix_lines)
mixed+=b'TOTAL\n'*(4095-len(prefix_lines))
mixed+=b'REPLACE 7 2 65535 24\nFILL 7 0 2 1\nINFO 7\nTOTAL\n'
mixed+=b''.join(line+b'\n' for line in invalid_lines)
cases.append(mixed)
expected=[cli_expected(case) for case in cases]
framed=str(len(cases)).encode()+b'\n'+b''.join(str(len(c)).encode()+b'\n'+c for c in cases)
output=b''.join(('C 0 %d 0\n'%len(e)).encode()+e for e in expected)
return framed,output,expected
def reuse_reasons(actual,expected):
reasons=[];position=0
for output in expected:
end=actual.find(b'\n',position)
if end<0: return ['stdout_bytes']
header=actual[position:end].split(b' ')
try:
if len(header)!=4 or header[0]!=b'C': return ['stdout_bytes']
returned,size,errsize=map(int,header[1:])
if size<0 or errsize<0 or end+1+size+errsize>len(actual): return ['stdout_bytes']
except ValueError:
return ['stdout_bytes']
position=end+1
observed=actual[position:position+size]
position+=size
diagnostic=actual[position:position+errsize]
position+=errsize
if returned and 'cli_return' not in reasons: reasons.append('cli_return')
if observed!=output and 'stdout_bytes' not in reasons: reasons.append('stdout_bytes')
if diagnostic and 'stderr_nonempty' not in reasons: reasons.append('stderr_nonempty')
if position!=len(actual) and 'stdout_bytes' not in reasons: reasons.append('stdout_bytes')
if not reasons and actual!=b''.join(('C 0 %d 0\n'%len(e)).encode()+e for e in expected):
reasons.append('stdout_bytes')
return reasons
def digest(data):
return [len(data),hashlib.sha256(data).hexdigest()[:16]]
def main():
candidate=Path(sys.argv[1])
# All trusted input and source I/O ends before the first compiler starts.
sources={name:(candidate/name).read_bytes() for name in
('record.hpp','internal.hpp','layout.cpp','store.cpp','cli.cpp')}
scratch=Path(tempfile.mkdtemp(prefix='inline-record-',dir=Path.cwd()))
for name,data in sources.items(): (scratch/name).write_bytes(data)
(scratch/'observer.cpp').write_text(OBSERVER)
api_in,api_out=api_scene()
cli_in=cli_scene()
reuse_in,reuse_out,reuse_expected=reuse_scene()
boundary=b'TOTAL\n'*4095+b'REPLACE 7 0 65535 16\nINFO 7\nTOTAL\n'
scenes=[('api','observer',api_in,api_out),('reuse','observer',reuse_in,reuse_out),('cli','records',cli_in,cli_expected(cli_in)),
('limit','records',boundary,cli_expected(boundary)),('empty','records',b'',b''),
('nolf','records',b'REPLACE 7 0 65535 16\nINFO 7\nTOTAL',cli_expected(b'REPLACE 7 0 65535 16\nINFO 7\nTOTAL')),
('fresh','records',b'TOTAL\nREPLACE 0 1 0 20\nFILL 0 0 1 0\nGET 0 0\nTOTAL\n',cli_expected(b'TOTAL\nREPLACE 0 1 0 20\nFILL 0 0 1 0\nGET 0 0\nTOTAL\n'))]
builds=[['g++','-std=c++17','-O2','-DINLINE_LIBRARY','layout.cpp','store.cpp','cli.cpp','observer.cpp','-o','observer'],
['g++','-std=c++17','-O2','layout.cpp','store.cpp','cli.cpp','-o','records']]
meta={'cwd':'wrapper-owned/inline-record-*','build':[],'scenes':[],'reasons':[]}
def run(argv,data,timeout):
process=subprocess.Popen(argv,cwd=str(scratch),stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
timed=False
try: out,err=process.communicate(data,timeout=timeout)
except subprocess.TimeoutExpired:
timed=True; process.kill(); out,err=process.communicate()
return process.pid,process.returncode,out,err,timed
for argv in builds:
pid,code,out,err,timed=run(argv,b'',20)
meta['build'].append({'pid':pid,'argv':argv,'exit':code,'timeout':timed,'stdin':digest(b''),'stdout':digest(out),'stderr':digest(err)})
if code or timed:
meta['reasons'].append('build'); break
if not meta['reasons']:
for name,exe,data,expected in scenes:
argv=['./'+exe]+(['reuse'] if name=='reuse' else [])
pid,code,out,err,timed=run(argv,data,12)
reasons=[]
if timed: reasons.append('timeout')
if code: reasons.append('exit')
if name=='reuse':
reasons.extend(reuse_reasons(out,reuse_expected))
elif out!=expected: reasons.append('stdout_bytes')
if err: reasons.append('stderr_nonempty')
meta['scenes'].append({'name':name,'pid':pid,'argv':argv,'exit':code,'timeout':timed,
'stdin':digest(data),'expected':digest(expected),'actual':digest(out),
'stderr':digest(err),'reason':reasons})
meta['reasons'].extend(name+':'+r for r in reasons)
payload=json.dumps(meta,separators=(',',':'))
if len(payload.encode())>=3900:
raise RuntimeError('trial metadata exceeds 3899 bytes')
print(payload,file=sys.stderr)
if meta['reasons']:
print('FAIL '+','.join(meta['reasons'])); return 1
print('PASS'); return 0
if __name__=='__main__':
sys.exit(main())