forked from longtao.wu/droboticflow
37 lines
2.4 KiB
Python
37 lines
2.4 KiB
Python
"""Strict comparison of isolated query-serving measurements; no production IO."""
|
|
|
|
def compare(base, candidate):
|
|
problems=[]
|
|
if base.get('fixture_digest')!=candidate.get('fixture_digest') or not base.get('fixture_digest'):
|
|
problems.append('fixture identity mismatch')
|
|
if not base.get('metrics') or not candidate.get('metrics'):
|
|
problems.append('missing request measurements')
|
|
if candidate.get('actors')!=81 or candidate.get('errors')!=0 or candidate.get('mongo_write_commands')!=0:
|
|
problems.append('concurrency, request errors or business writes failed')
|
|
counters=candidate.get('query_serving_statistics',{})
|
|
if not all(counters.get(k,0)>0 for k in ('report_hits','search_hits','typesense_reads','cache_hits')):
|
|
problems.append('accelerated paths not proven active')
|
|
for key,m in candidate.get('metrics',{}).items():
|
|
limit=2000 if '.search' in key else 500 if key.endswith('.panel') or key=='admin.production' else None
|
|
if limit and m.get('p95_ms',float('inf'))>limit:problems.append('latency exceeded: '+key)
|
|
for row in candidate.get('cold_samples',[]):
|
|
if (row['Key'].endswith('.panel') or row['Key']=='admin.production') and (row['Code']!=200 or row['MS']>2000):
|
|
problems.append('prewarmed first visit exceeded: '+row['Key'])
|
|
def hashes(report):
|
|
out={}
|
|
for r in report.get('cold_samples',[])+report.get('samples',[]):
|
|
if r.get('Code')==200:out.setdefault((r['Actor'],r['Key']),set()).add(r['Hash'])
|
|
return out
|
|
old,new=hashes(base),hashes(candidate)
|
|
overlap=old.keys() & new.keys()
|
|
if not overlap:problems.append('no comparable successful results')
|
|
for key in overlap:
|
|
if old[key]!=new[key]:problems.append('business result mismatch: '+':'.join(key))
|
|
missing=sorted(':'.join(k) for k in new.keys()-old.keys())
|
|
# Baseline timeouts cannot demonstrate equivalence; functional fixtures must
|
|
# cover those scopes separately. Keep them visible rather than marking equal.
|
|
return {'passed':not problems and not missing,'problems':sorted(set(problems)),
|
|
'comparableActorEndpoints':len(overlap),'baselineUnavailableComparisons':missing,
|
|
'cpuRatio':candidate.get('process_cpu_seconds',0)/max(base.get('process_cpu_seconds',0),.00001),
|
|
'retainedHeapRatio':candidate.get('go_heap_after_gc',0)/max(base.get('go_heap_after_gc',0),1)}
|