236 lines
17 KiB
Python
236 lines
17 KiB
Python
import copy,errno,hashlib,json,pathlib,random,secrets,subprocess,sys,tempfile
|
|
HEADER='f9222e19f845b678b7f66d3e7d44226326600a857b73116e2e80ef5c83d88327'
|
|
E=errno;U64=(1<<64)-1;S64=int.from_bytes(b'\xa5'*8,'little');S32=int.from_bytes(b'\xa5'*4,'little',signed=True);U32=int.from_bytes(b'\xa5'*4,'little')
|
|
def require(ok,message):
|
|
if not ok:raise AssertionError(message)
|
|
class Plan:
|
|
def holds(self):return (sum(s is not None and s["lease"]is not None for c in self.caches if c is not None for s in c["slots"]),sum(c["writer"]is not None for c in self.caches if c is not None))
|
|
def __init__(self):self.caches=[None]*4;self.readers=[(0,0,0)]*16;self.leases=[(0,0,0,0,0)]*16;self.txs=[(0,0)]*16;self.lines=[];self.expected=[]
|
|
def add(self,op,*a):
|
|
self.lines.append(' '.join(map(str,(op,*a))));seq=len(self.lines);rc=0;out=[];lock=None
|
|
if op=='c':
|
|
i,owner,epoch=a
|
|
if owner==0:out=[0,E.EINVAL]
|
|
else:
|
|
self.caches[i]=dict(owner=owner,epoch=epoch,gen=[0]*8,slots=[None]*8,data=[(0,0,0,0)]*16,next=0,serving=0,serial=1,writer=None,closed=False);out=[1,None]
|
|
self.expected.append((op,seq,out,None,self.holds()));return
|
|
i=a[0];c=self.caches[i];nullout=(op in('r','w','i')and a[-1]or op in('s','b','v')and a[-1])if op in('r','w','i','s','b','v')else 0
|
|
if c is None or nullout:rc=E.EINVAL
|
|
elif c['closed']and op not in('i','q','d'):rc=E.ESHUTDOWN
|
|
reader=None;lease=None;tx=None
|
|
if op in('u','s','b'):
|
|
h=a[1];reader=list(self.readers[h]);field=a[3]if op=='b'else a[2];value=a[4]if op=='b'else a[3]
|
|
if field:reader[field-1]=value
|
|
if not rc:
|
|
if not(reader[0]==c['owner']and 0<=reader[2]<8 and c['slots'][reader[2]]is not None and reader[1]==c['gen'][reader[2]]):rc=E.EINVAL
|
|
slot=c['slots'][reader[2]]if not rc else None
|
|
if op in('v','e'):
|
|
lease=list(self.leases[a[1]]);field=a[3]if op=='v'else a[2];value=a[4]if op=='v'else a[3]
|
|
if field:lease[field-1]=value
|
|
if not rc:
|
|
if not(lease[0]==c['owner']and 0<=lease[4]<8 and c['slots'][lease[4]]is not None and c['slots'][lease[4]].get('lease')==tuple(lease)):rc=E.EINVAL
|
|
slot=c['slots'][lease[4]]if not rc else None
|
|
if op in('p','x','k','a'):
|
|
tx=list(self.txs[a[1]]);field=a[-2];value=a[-1]
|
|
if field:tx[field-1]=value
|
|
if not rc and tuple(tx)!=c['writer']:rc=E.EINVAL
|
|
if not rc:
|
|
if op=='r':
|
|
free=next((j for j,s in enumerate(c['slots'])if s is None),None)
|
|
if free is None:rc=E.ENOSPC
|
|
else:
|
|
c['gen'][free]+=1;c['slots'][free]=dict(seen=c['epoch'],completed=0,lease=None);token=(c['owner'],c['gen'][free],free);self.readers[a[1]]=token;out=list(token)
|
|
elif op=='u':
|
|
if slot['lease']is not None:rc=E.EBUSY
|
|
else:c['slots'][reader[2]]=None
|
|
elif op=='s':out=[slot['seen'],slot['completed'],int(slot['lease']is not None)]
|
|
elif op=='b':
|
|
if slot['lease']is not None:rc=E.EBUSY
|
|
else:
|
|
token=(c['owner'],reader[1],c['serial'],c['epoch'],reader[2]);c['serial']+=1;self.leases[a[2]]=token;slot['lease']=token;slot['snapshot']=copy.deepcopy(c['data']);out=list(token);lock='rd'
|
|
elif op=='v':
|
|
if not 0<=a[2]<16:rc=E.EINVAL
|
|
else:
|
|
present,version,value,tag=slot['snapshot'][a[2]];out=[lease[3],present,version,value,tag]
|
|
elif op=='e':slot['seen']=max(slot['seen'],lease[3]);slot['completed']+=1;slot['lease']=None;lock='unlock'
|
|
elif op=='w':
|
|
if c['writer']is not None or any(s and s['lease']is not None for s in c['slots']):rc=E.EDEADLK
|
|
else:
|
|
token=(c['owner'],c['next']);c['next']+=1;c['writer']=token;c['stage']=copy.deepcopy(c['data']);c['dirty']=set();self.txs[a[1]]=token;out=list(token);lock='wr'
|
|
elif op in('p','x'):
|
|
key=a[2]
|
|
if not 0<=key<16:rc=E.EINVAL
|
|
else:c['stage'][key]=(1,0,a[3],a[4])if op=='p'else(0,0,0,0);c['dirty'].add(key)
|
|
elif op=='k':
|
|
if c['epoch']==U64:rc=E.EOVERFLOW
|
|
else:
|
|
c['epoch']+=1
|
|
for key in c['dirty']:
|
|
present,_,value,tag=c['stage'][key];c['stage'][key]=(1,c['epoch'],value,tag)if present else(0,0,0,0)
|
|
c['data']=c['stage'];c['writer']=None;c['serving']+=1;lock='unlock'
|
|
elif op=='a':c['writer']=None;c['serving']+=1;lock='unlock'
|
|
elif op=='i':out=[c['epoch'],c['next'],c['serving'],sum(s is not None for s in c['slots']),sum(s is not None and s['lease']is not None for s in c['slots']),0,int(c['writer']is not None),int(c['closed'])]
|
|
elif op in('z','q','d'):
|
|
busy=any(s is not None for s in c['slots'])or c['writer']is not None
|
|
if busy or(op=='d'and not c['closed']):rc=E.EBUSY
|
|
elif op=='q':c['closed']=True
|
|
elif op=='d':self.caches[i]=None
|
|
else:c.update(epoch=a[1],data=[(0,0,0,0)]*16,next=0,serving=0,serial=1)
|
|
if op in('r','s','b','v','w','i'):
|
|
if rc:
|
|
out={'r':[S64,S64,U32],'s':[S64,S64,S32],'b':[S64,S64,S64,S64,U32],'v':[S64,S32,S64,int.from_bytes(b'\xa5'*8,'little',signed=True),S64],'w':[S64,S64],'i':[S64,S64,S64,U32,U32,U32,S32,S32]}[op]
|
|
fields=[rc,int(bool(rc)),*out]
|
|
else:fields=[rc]
|
|
self.expected.append((op,seq,fields,lock,self.holds()))
|
|
def foreign(self,i,lease,tx,role,value,tag):
|
|
self.lines.append(f'T {i} {lease} {tx} {role} {value} {tag}');self.expected.append(('FOREIGN',None,[role,0,0,E.EINVAL,0 if role==0 else E.EINVAL,E.EINVAL],None,self.holds()))
|
|
def readall(self,i,h,l):
|
|
self.add('b',i,h,l,0,0,0)
|
|
for key in range(16):self.add('v',i,l,key,0,0,0)
|
|
self.add('s',i,h,0,0,0);self.add('e',i,l,0,0);self.add('s',i,h,0,0,0)
|
|
|
|
p=Plan();r=random.Random(81019);owner=r.randrange(1,1<<63);initial=r.randrange(500)
|
|
p.add('c',0,owner,initial);p.add('c',1,owner+1,313);p.add('c',2,0,0)
|
|
for op,args in [('r',(3,0,0)),('s',(3,0,0,0,0)),('b',(3,0,0,0,0,0)),('v',(3,0,0,0,0,0)),('e',(3,0,0,0)),('w',(3,0,0)),('p',(3,0,0,1,2,0,0)),('x',(3,0,0,0,0)),('k',(3,0,0,0)),('a',(3,0,0,0)),('i',(3,0)),('z',(3,0)),('q',(3,)),('d',(3,))]:p.add(op,*args)
|
|
p.add('d',0);p.add('r',0,0,1);p.add('r',0,0,0);p.add('s',0,0,0,0,1);p.add('b',0,0,0,0,0,1);p.add('z',0,1);p.add('q',0)
|
|
for field,value in[(1,0),(2,0),(3,8),(3,(1<<32)-1)]:
|
|
for op in('u','s'):p.add(op,0,0,field,value,*([0]if op=='s'else[]))
|
|
p.add('b',0,0,0,field,value,0)
|
|
p.add('s',1,0,0,0,0);p.readall(0,0,0);p.add('b',0,0,1,0,0,0);p.add('b',0,0,2,0,0,0);p.add('u',0,0,0,0);p.add('w',0,0,0);p.add('i',0,0)
|
|
for field,value in[(1,0),(2,0),(3,0),(4,initial+1),(5,8)]:p.add('v',0,1,0,field,value,0);p.add('e',0,1,field,value)
|
|
p.add('v',0,1,16,0,0,0);p.add('v',0,1,0,0,0,1);p.foreign(0,1,0,0,0,0);p.add('e',0,0,0,0);p.add('e',0,1,0,0);p.add('v',0,1,0,0,0,0);p.add('e',0,1,0,0)
|
|
p.add('w',0,0,1);p.add('w',0,0,0);p.add('w',0,1,0);p.add('i',0,0)
|
|
for field,value in[(1,0),(2,999)]:
|
|
p.add('p',0,0,0,9,99,field,value);p.add('x',0,0,0,field,value);p.add('k',0,0,field,value);p.add('a',0,0,field,value)
|
|
p.add('p',0,0,16,1,2,0,0);p.add('x',0,0,16,0,0);p.add('p',0,0,0,-9223372036854775808,U64,0,0);p.add('p',0,0,1,9223372036854775807,0,0,0);p.foreign(0,0,0,1,1,2);p.add('i',0,0);p.add('k',0,0,0,0);p.add('k',0,0,0,0);p.readall(0,0,2)
|
|
for trial in range(60):
|
|
p.add('w',0,1,0);p.add('p',0,0,3,999,999,0,0)
|
|
for j in range(r.randrange(0,12)):
|
|
key=r.randrange(16)
|
|
if r.randrange(4)==0:p.add('x',0,1,key,0,0)
|
|
else:p.add('p',0,1,key,r.randrange(-10**12,10**12),r.getrandbits(64),0,0)
|
|
p.add('i',0,0);p.add('a'if trial%4==0 else'k',0,1,0,0);p.readall(0,0,3)
|
|
p.add('u',0,0,0,0);p.add('r',0,1,0);p.add('s',0,0,0,0,0);p.add('b',0,0,0,0,0,0);p.add('u',0,0,0,0);p.readall(0,1,4);p.add('u',0,1,0,0);p.add('z',0,7);p.add('r',0,2,0);p.add('s',0,1,0,0,0);p.readall(0,2,5);p.add('u',0,2,0,0)
|
|
for h in range(8,16):p.add('r',1,h,0)
|
|
p.add('r',1,7,0);p.add('q',1);p.add('z',1,0)
|
|
for h in range(8,16):p.add('u',1,h,0,0)
|
|
p.add('z',1,0);p.add('r',1,8,0);p.add('s',1,9,0,0,0);p.readall(1,8,6);p.add('u',1,8,0,0);p.add('q',1);p.add('q',1);p.add('i',1,0)
|
|
for op,args in [('r',(1,8,0)),('s',(1,8,0,0,0)),('b',(1,8,0,0,0,0)),('v',(1,0,0,0,0,0)),('e',(1,0,0,0)),('w',(1,0,0)),('p',(1,0,0,1,2,0,0)),('x',(1,0,0,0,0)),('k',(1,0,0,0)),('a',(1,0,0,0)),('z',(1,99))]:p.add(op,*args)
|
|
p.add('r',1,8,1);p.add('i',1,1);p.add('d',1);p.add('d',1)
|
|
p.add('c',2,owner+2,U64-1);p.add('w',2,2,0);p.add('p',2,2,3,31,17,0,0);p.add('k',2,2,0,0);p.add('r',2,7,0);p.readall(2,7,7);p.add('u',2,7,0,0);p.add('w',2,3,0);p.add('p',2,3,3,999,999,0,0);p.add('k',2,3,0,0);p.add('i',2,0);p.add('a',2,3,0,0);p.add('r',2,7,0);p.readall(2,7,8);p.add('u',2,7,0,0);p.add('q',2);p.add('d',2);p.add('q',0);p.add('d',0)
|
|
|
|
SCOPES={}
|
|
def observations(proc,nonce):
|
|
require(proc.returncode==0,'candidate observer execution failed')
|
|
rows=[]
|
|
for line in proc.stdout.splitlines():
|
|
if not line.startswith(nonce+' '):continue
|
|
row=line.split()[1:]
|
|
if '@SCOPE'in row:
|
|
start=row.index('@SCOPE');end=row.index('@COUNTS');wire=list(map(int,row[start+1:end]));require(len(wire)>=2 and wire[0]==0 and len(wire)==2+wire[1]*4,'lock scope protocol')
|
|
scopes=[tuple(wire[i:i+4])for i in range(2,len(wire),4)]
|
|
require(len(set((t,a)for t,a,r,w in scopes))==len(scopes)and all(t>0 and a>0 and r+w>0 for t,a,r,w in scopes),'lock scope identities')
|
|
row=row[:start]+row[end+1:];SCOPES[id(row)]=scopes
|
|
rows.append(row)
|
|
return rows
|
|
|
|
def lock_addresses(row,kind):return {a for t,a,r,w in SCOPES.get(id(row),[])if(r if kind=='rd'else w)>0}
|
|
def lock_threads(row,addresses,kind):return {t for t,a,r,w in SCOPES.get(id(row),[])if a in addresses and(r if kind=='rd'else w)>0}
|
|
|
|
|
|
def sequential(rows):
|
|
require(len(rows)==len(p.expected),'sequential observation count')
|
|
counters=[0,0,0];retained={'rd':set(),'wr':set()}
|
|
for i,(row,(op,seq,want,lock,held))in enumerate(zip(rows,p.expected)):
|
|
require(row[0]==op,'operation '+str(i));numbers=list(map(int,row[1:]));actual=numbers[1:-3]if seq is not None else numbers[:-3]
|
|
if seq is not None:require(numbers[0]==seq,'sequence')
|
|
require(len(actual)==len(want)and all(b is None or a==b for a,b in zip(actual,want)),'sequential case '+str(i)+' '+op+' mismatch')
|
|
now=numbers[-3:];require(all(a>=b for a,b in zip(now,counters)),'lock counter regressed')
|
|
if lock:require(now[{'rd':0,'wr':1,'unlock':2}[lock]]>counters[{'rd':0,'wr':1,'unlock':2}[lock]],'missing real rwlock operation')
|
|
scope=SCOPES.get(id(row));require(scope is not None,'missing scope observation')
|
|
for kind,needed in zip(('rd','wr'),held):
|
|
current={(thread,address)for thread,address,read,write in scope if(read if kind=='rd'else write)>0}
|
|
if needed:
|
|
require(current,'real '+kind+' lock not retained across API boundary')
|
|
if lock==kind:retained[kind]=current
|
|
require(retained[kind]<=current,'real '+kind+' lock object/thread changed during lease/transaction')
|
|
else:retained[kind]=set()
|
|
if held==(0,0):require(not scope,'lock leaked beyond lease/transaction')
|
|
counters=now
|
|
|
|
# Concurrency cases are chosen and interpreted exclusively in this root process.
|
|
parallel=[]
|
|
for count in(1,3,5):parallel.append(('F',(owner+10+count,r.randrange(100),count,r.randrange(10**6,10**7),r.getrandbits(60))))
|
|
for trial in range(2):parallel.append(('G',(owner+30+trial,r.randrange(100),r.randrange(10**6,10**7),r.getrandbits(60))))
|
|
for trial,loops in enumerate((32,128,71)):parallel.append(('M',(owner+40+trial,r.randrange(100),loops,r.randrange(10**6,10**7),r.getrandbits(60))))
|
|
|
|
def parallel_check(kind,args,rows):
|
|
by={};raw={}
|
|
for row in rows:
|
|
by.setdefault(row[0],[]).append(list(map(int,row[1:])));raw.setdefault(row[0],[]).append(row)
|
|
if kind=='F':
|
|
_,epoch,count,value,tag=args
|
|
data_locks=lock_addresses(raw['FQUEUE'][0],'rd');require(data_locks,'FIFO initial lease released its real lock')
|
|
for row in raw['FQUEUE']:require(data_locks<=lock_addresses(row,'rd'),'FIFO initial real read lock scope')
|
|
for row in raw['FADMIT']:require(data_locks<=lock_addresses(row,'wr')and len(lock_threads(row,data_locks,'wr'))==1,'FIFO active writer real lock scope')
|
|
require(data_locks<=lock_addresses(raw['FREADER'][0],'rd'),'FIFO late lease real lock scope')
|
|
require(not SCOPES.get(id(raw['FREAD'][0]),[]),'FIFO lock leak')
|
|
require(by.get('FSTART')==[[0,0,0]]and by.get('FBUSY')==[[0,0,E.EBUSY]]and by.get('FENDREAD')==[[0]]and by.get('FCLEAN')==[[0,0,0,0]],'FIFO setup/cleanup')
|
|
for i,row in enumerate(by['FQUEUE']):require(row[:12]==[i,0,0,epoch,i+1,0,2,1,i+1,0,0,0],'FIFO queued state')
|
|
for i,row in enumerate(by['FADMIT']):
|
|
require(row[:12]==[i,0,0,epoch+i,count,i,2,0,count-i-1,1,0,i+1]and row[12:-3]==list(range(i+1)),'FIFO admission order/state')
|
|
require(len(by['FWRITER'])==count,'FIFO writer count')
|
|
for i,row in enumerate(by['FWRITER']):require(row[:7]==[i,0,0,0,0,i,0],'FIFO writer results')
|
|
row=by['FREADER'][0];require(row[:12]==[8,0,0,epoch+count,count,count,2,1,0,0,0,count+1]and row[12:-3]==list(range(count))+[8],'late reader overtook writers')
|
|
row=by['FREAD'][0];want=[8,0,0,0,0,0,epoch+count,epoch+count,1,epoch+count,value+count-1,tag+count-1,epoch+count,1,epoch+count,-(value+count-1),(~(tag+count-1))&U64,epoch+count,1,0];require(row[:-3]==want,'late reader snapshot')
|
|
elif kind=='G':
|
|
_,epoch,value,tag=args
|
|
data_locks=lock_addresses(raw['GREAD'][0],'rd');require(data_locks,'shared first reader released real lock')
|
|
require(data_locks<=lock_addresses(raw['GREAD'][1],'rd')and len(lock_threads(raw['GREAD'][1],data_locks,'rd'))==2,'two real readers must retain same cache lock')
|
|
require(data_locks<=lock_addresses(raw['GONE'][0],'rd')and len(lock_threads(raw['GONE'][0],data_locks,'rd'))==1,'remaining real reader scope')
|
|
require(data_locks<=lock_addresses(raw['GWRITE'][0],'wr'),'shared writer real lock scope')
|
|
require(not SCOPES.get(id(raw['GWRITER'][0]),[]),'shared lock leak')
|
|
require(by.get('GSTART')==[[0,0]]and by.get('GCLEAN')==[[0,0,0,0]],'shared setup/cleanup')
|
|
for i,row in enumerate(by['GREAD']):require(row[:12]==[i,0,0,epoch,0,0,2,i+1,0,0,0,i+1],'readers were serialized')
|
|
require(by['GQUEUE'][0][:12]==[2,0,0,epoch,1,0,2,2,1,0,0,2],'writer did not wait for readers')
|
|
require(by['GONE'][0][:12]==[0,0,0,epoch,1,0,2,1,1,0,0,2],'first reader ended wrong ownership')
|
|
require(by['GWRITE'][0][:12]==[2,0,0,epoch,1,0,2,0,0,1,0,3],'writer entered with live reader')
|
|
for row in by['GDONE']:
|
|
require(row[:7]==[row[0],0,0,0,0,0,epoch]and row[7:-3]==[epoch,0,0,0,0,epoch,0,0,0,0,epoch,1,0],'shared read changed')
|
|
require(by['GWRITER'][0][:7]==[2,0,0,0,0,0,0],'shared writer results')
|
|
else:
|
|
_,epoch,loops,value,tag=args
|
|
require(not SCOPES.get(id(raw['MSTATE'][0]),[]),'stress real lock leak')
|
|
require(by.get('MREGISTER')==[[0,0],[1,0],[2,0]]and by.get('MTHREAD')==[[0,0],[1,0],[2,0],[3,0]]and by.get('MJOIN')==[[0,0],[1,0],[2,0],[3,0]]and by.get('MCLEAN')==[[0,0,0,0,0]],'stress lifecycle')
|
|
all_serials=[];last=[epoch]*3;counts=[0]*4;last_serial=[0]*3
|
|
for row in by['MROW']:
|
|
who,index,serial,*remaining=row;require(index==counts[who]and remaining[:4]==[0]*4,'stress API result');counts[who]+=1
|
|
if who==3:require(serial==index,'writer ticket progression');continue
|
|
all_serials.append(serial);require(serial>last_serial[who],'reader serial regressed');last_serial[who]=serial
|
|
a,b=remaining[4:9],remaining[9:14];seen=a[0];require(epoch<=seen<=epoch+loops and seen>=last[who]and b[0]==seen,'reader epoch regressed/torn');last[who]=seen
|
|
if seen==epoch:require(a==[epoch,0,0,0,0]and b==[epoch,0,0,0,0],'initial snapshot')
|
|
else:
|
|
j=seen-epoch-1;require(a==[seen,1,seen,value+j,tag+j]and b==[seen,1,seen,-(value+j),(~(tag+j))&U64],'torn multi-key transaction')
|
|
require(counts==[loops]*4 and sorted(all_serials)==list(range(1,3*loops+1)),'lease serial uniqueness/count')
|
|
require(by['MSTATUS']==[[i,last[i],loops,0]for i in range(3)],'reader monotonic status')
|
|
require(by['MSTATE'][0][:-3]==[0,epoch+loops,loops,loops,3,0,0,0,0],'stress final publication')
|
|
# Every concurrency run must actually acquire both kinds of rwlock and release them.
|
|
counters=[list(map(int,row[-3:]))for row in rows if row[0]in('FREAD','GWRITER','MSTATE')]
|
|
require(counters and all(v>0 for v in counters[-1]),'missing concurrent real rwlock operations')
|
|
|
|
try:
|
|
scratch=pathlib.Path(tempfile.mkdtemp(prefix='epoch-check-',dir=str(pathlib.Path.cwd())))
|
|
for rel in('epoch_cache.c','epoch_cache.h'):
|
|
source=pathlib.Path(sys.argv[1])/rel;require(source.is_file()and not source.is_symlink(),'missing candidate '+rel);(scratch/rel).write_bytes(source.read_bytes())
|
|
require(hashlib.sha256((scratch/'epoch_cache.h').read_bytes()).hexdigest()==HEADER,'public header changed')
|
|
(scratch/'observer.c').write_bytes((pathlib.Path(__file__).parent/'observer.c').read_bytes())
|
|
command=['cc','-std=c11','-D_POSIX_C_SOURCE=200809L','-Wall','-Wextra','-Werror','-pedantic','-pthread','epoch_cache.c','observer.c','-Wl,--wrap=pthread_rwlock_rdlock','-Wl,--wrap=pthread_rwlock_wrlock','-Wl,--wrap=pthread_rwlock_tryrdlock','-Wl,--wrap=pthread_rwlock_trywrlock','-Wl,--wrap=pthread_rwlock_timedrdlock','-Wl,--wrap=pthread_rwlock_timedwrlock','-Wl,--wrap=pthread_rwlock_unlock','-o','observer']
|
|
built=subprocess.run(command,cwd=scratch,capture_output=True,timeout=30);require(built.returncode==0,'compile failed')
|
|
# From this point on root only reads pipes. All source/input files were written before compilation.
|
|
nonce=secrets.token_hex(16);proc=subprocess.run([str(scratch/'observer'),nonce],input='\n'.join(p.lines)+'\n',cwd=scratch,capture_output=True,text=True,timeout=20);sequential(observations(proc,nonce))
|
|
for kind,args in parallel:
|
|
nonce=secrets.token_hex(16);proc=subprocess.run([str(scratch/'observer'),nonce],input=' '.join(map(str,(kind,*args)))+'\n',cwd=scratch,capture_output=True,text=True,timeout=15);parallel_check(kind,args,observations(proc,nonce))
|
|
print('PASS')
|
|
except Exception as error:
|
|
print('FAIL:',type(error).__name__,str(error)[:1200]);sys.exit(1)
|