droboticflow/scripts/lib/query_serving_evidence.py

41 lines
2.8 KiB
Python

"""Strict comparison of isolated query-serving measurements; no production IO."""
def compare(base, candidate):
problems=[]
if base.get('fixture_digest')!=candidate.get('fixture_digest') or not base.get('fixture_digest'):
problems.append('fixture identity mismatch')
if any(r.get('mongo_clock_samples',0)<2 or r.get('mongo_clock_max_jump_ms',float('inf'))>500 for r in (base,candidate)):
problems.append('Mongo clock evidence missing or unstable')
if not base.get('metrics') or not candidate.get('metrics'):
problems.append('missing request measurements')
if candidate.get('actors')!=81 or candidate.get('errors')!=0 or candidate.get('mongo_write_commands')!=0:
problems.append('concurrency, request errors or business writes failed')
counters=candidate.get('query_serving_statistics',{})
if not all(counters.get(k,0)>0 for k in ('report_hits','search_hits','typesense_reads','cache_hits')):
problems.append('accelerated paths not proven active')
if candidate.get('scheduled_seconds',0)>=180 and candidate.get('output_prepared_queries',0)<=0:
problems.append('generation-change prewarming not proven active')
for key,m in candidate.get('metrics',{}).items():
limit=2000 if '.search' in key else 500 if key.endswith('.panel') or key=='admin.production' else None
if limit and m.get('p95_ms',float('inf'))>limit:problems.append('latency exceeded: '+key)
for row in candidate.get('cold_samples',[]):
if (row['Key'].endswith('.panel') or row['Key']=='admin.production') and (row['Code']!=200 or row['MS']>2000):
problems.append('prewarmed first visit exceeded: '+row['Key'])
def hashes(report):
out={}
for r in report.get('cold_samples',[])+report.get('samples',[])+report.get('reference_samples',[]):
if r.get('Code')==200:out.setdefault((r['Actor'],r['Key']),set()).add(r['Hash'])
return out
old,new=hashes(base),hashes(candidate)
overlap=old.keys() & new.keys()
if not overlap:problems.append('no comparable successful results')
for key in overlap:
if old[key]!=new[key]:problems.append('business result mismatch: '+':'.join(key))
missing=sorted(':'.join(k) for k in new.keys()-old.keys())
# Baseline timeouts cannot demonstrate equivalence; functional fixtures must
# cover those scopes separately. Keep them visible rather than marking equal.
return {'passed':not problems and not missing,'problems':sorted(set(problems)),
'comparableActorEndpoints':len(overlap),'baselineUnavailableComparisons':missing,
'cpuRatio':candidate.get('process_cpu_seconds',0)/max(base.get('process_cpu_seconds',0),.00001),
'retainedHeapRatio':candidate.get('go_heap_after_gc',0)/max(base.get('go_heap_after_gc',0),1)}