droboticflow-new/scripts/test_query_serving_performance.py

92 lines
6.4 KiB
Python

#!/usr/bin/env python3
"""Compare the same synthetic API fixture before/after query serving."""
import argparse
import fcntl
import datetime as dt
import json
from pathlib import Path
import subprocess
import threading
import time
import uuid
from test_query_serving import ROOT, PRIVATE, environment, start, run_command
from lib.query_serving_evidence import compare
from lib.release_checks import write_log, source_fingerprint
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--start',action='store_true')
parser.add_argument('--production-shape',action='store_true')
parser.add_argument('--rounds',type=int,default=1)
parser.add_argument('--warmup',type=int,default=5)
parser.add_argument('--duration',type=int,default=30)
parser.add_argument('--database',help='reuse this script\'s existing synthetic fixture')
args=parser.parse_args()
if not 1<=args.rounds<=3 or not 1<=args.duration<=1800 or not 1<=args.warmup<=300:parser.error('bounded rounds/times required')
if args.database and (not args.database.startswith('poll_pool_read_mix_query_') or not args.database.replace('_','').isalnum()):parser.error('synthetic database only')
env={k:v for k,v in environment().items() if not k.startswith('POLL_POOL_')}
dependency_lock=(PRIVATE/'runner.lock').open('w')
try:fcntl.flock(dependency_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
except BlockingIOError:
print('Isolated query dependencies already in use; comparison not started.');return 2
if args.start:start(env)
folder=ROOT/'local-data/diagnostics'/('query-serving-'+dt.datetime.now(dt.timezone.utc).strftime('%Y%m%dT%H%M%SZ'))
folder.mkdir(parents=True)
database=args.database or 'poll_pool_read_mix_query_'+uuid.uuid4().hex[:16]
env.update(POLL_POOL_LOAD='1',POLL_POOL_LOCAL_MONGO_URI='mongodb://127.0.0.1:27019/?replicaSet=query-rs&directConnection=true',POLL_POOL_FIXTURE_DATABASE=database,
POLL_POOL_KEEP_FIXTURE='1',POLL_POOL_QUERY_SERVING_PREPARE='1',POLL_POOL_QUERY_PANELS='1',POLL_POOL_READMODELS='1',POLL_POOL_SEARCH='1',POLL_POOL_SEARCH_INDEXES='1',POLL_POOL_PAGE_SHAPE='1',POLL_POOL_ADMIN_TASKS='1' if args.production_shape else '0',POLL_POOL_AUTHORIZATION_STATE='initialized',
POLL_POOL_WARMUP_SECONDS=str(args.warmup),POLL_POOL_DURATION_SECONDS=str(args.duration),ANNOTATION_SOURCE_VALIDATION_MODE='advisory')
if args.production_shape:
env.update(POLL_POOL_PRODUCTION_SHAPE='1', POLL_POOL_RUN_PAYLOAD_KIB='80',
POLL_POOL_TASK_PAYLOAD_KIB='16', POLL_POOL_AVAILABLE_PER_STAGE='215',
POLL_POOL_AUTHOR_HISTORY='100', GOMAXPROCS='2')
before=source_fingerprint(ROOT)
report={'database':database,'inputDigest':before,'shape':'production' if args.production_shape else 'small','rounds':[],'resourceSamples':[],'passed':False,'limits':['synthetic loopback API harness, no media or production writes','process resource sampling includes builder and handler; not Kubernetes Pod limits']}
argv=['go','test','./internal/api','-run','^TestPollPoolReadMixLocal$','-count=1','-timeout=45m','-v']
def run(name,updates):
settings={**env,**updates,'POLL_POOL_OUTPUT':str(folder/(name+'.json'))}
active['phase']=name
started=dt.datetime.now(dt.timezone.utc).isoformat()
result=run_command(argv,settings,args.warmup+args.duration+900)
proof=write_log(folder,argv,result.stdout+'\n'+result.stderr,'passed' if result.returncode==0 else 'failed')
row={'phase':name,'startedAt':started,'endedAt':dt.datetime.now(dt.timezone.utc).isoformat(),'exitCode':result.returncode,**proof,'report':settings['POLL_POOL_OUTPUT']}
report['rounds'].append(row)
(folder/'comparison.json').write_text(json.dumps(report,indent=2)+'\n')
print(json.dumps(row),flush=True)
return result.returncode==0
active={'phase':'prepare'}
if not args.database:
if not run('prepare',{'POLL_POOL_FIXTURE_ONLY':'1'}):return 1
stopped=threading.Event()
def resources():
while not stopped.is_set():
try:
p=subprocess.run(['docker','stats','--no-stream','--format','{{json .}}','fanyu-query-local-mongo-1','fanyu-query-local-valkey-1','fanyu-query-local-typesense-1'],capture_output=True,text=True,timeout=8)
if p.returncode==0:report['resourceSamples'].append({'phase':active['phase'],'at':dt.datetime.now(dt.timezone.utc).isoformat(),'containers':[json.loads(line) for line in p.stdout.splitlines()]})
except (OSError,ValueError,subprocess.TimeoutExpired):pass
(folder/'progress.json').write_text(json.dumps({'phase':active['phase'],'at':dt.datetime.now(dt.timezone.utc).isoformat(),'samples':len(report['resourceSamples'])})+'\n')
stopped.wait(2)
thread=threading.Thread(target=resources,daemon=True);thread.start()
try:
for i in range(args.rounds):
baseline='baseline' if i==0 else 'baseline-'+str(i+1)
run(baseline,{'POLL_POOL_REUSE_FIXTURE':'1','POLL_POOL_ALLOW_BASELINE_FAILURE':'1'})
if not run('candidate-'+str(i+1),{'POLL_POOL_REUSE_FIXTURE':'1','POLL_POOL_QUERY_SERVING':'1','POLL_POOL_MAX_SEARCH_P95_MS':'2000'}):break
finally:
stopped.set();thread.join(timeout=10)
report['inputsChangedDuringRun']=source_fingerprint(ROOT)!=before
candidates=[r for r in report['rounds'] if r['phase'].startswith('candidate-')]
report['comparisons']=[]
try:
for i,row in enumerate(candidates):
basefile='baseline.json' if i==0 else 'baseline-'+str(i+1)+'.json'
report['comparisons'].append(compare(json.loads((folder/basefile).read_text()),json.loads(Path(row['report']).read_text())))
except (OSError,ValueError,KeyError):report['comparisons'].append({'passed':False,'problems':['missing/invalid comparison report']})
report['resourceReviewRequired']=len(report['comparisons'])==3 and any(all(r.get(key,0)>1.1 for r in report['comparisons']) for key in ('cpuRatio','retainedHeapRatio'))
report['passed']=not report['resourceReviewRequired'] and len(candidates)==args.rounds and all(r['exitCode']==0 for r in candidates) and not report['inputsChangedDuringRun'] and all(r['passed'] for r in report['comparisons'])
(folder/'comparison.json').write_text(json.dumps(report,indent=2)+'\n')
print('Comparison: '+str(folder/'comparison.json'),flush=True)
return 0 if report['passed'] else 1
if __name__=='__main__':raise SystemExit(main())