droboticflow/scripts/test_query_serving_performance.py

111 lines
8.1 KiB
Python

#!/usr/bin/env python3
"""Compare the same synthetic API fixture before/after query serving."""
import argparse
import fcntl
import datetime as dt
import json
from pathlib import Path
import subprocess
import threading
import time
import uuid
from test_query_serving import ROOT, PRIVATE, environment, start, run_command
from lib.query_serving_evidence import compare
from lib.release_checks import write_log, source_fingerprint
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--start',action='store_true')
parser.add_argument('--production-shape',action='store_true')
parser.add_argument('--rounds',type=int,default=1)
parser.add_argument('--warmup',type=int,default=5)
parser.add_argument('--duration',type=int,default=30)
parser.add_argument('--database',help='reuse this script\'s existing synthetic fixture')
parser.add_argument('--native-mongo',action='store_true',help='use task-owned native Mongo on loopback 27019; never starts/stops another service')
args=parser.parse_args()
native_pid=None
if args.native_mongo:
if args.start:parser.error('start the native replica set and optional Docker components separately')
native_pid=int((PRIVATE/'native/mongod.pid').read_text().strip())
command=subprocess.run(['ps','-p',str(native_pid),'-o','command='],capture_output=True,text=True,check=True).stdout
if 'local-data/query-serving-local/native/' not in command or '--port 27019' not in command:parser.error('task-owned native Mongo process required')
if not 1<=args.rounds<=3 or not 1<=args.duration<=1800 or not 1<=args.warmup<=300:parser.error('bounded rounds/times required')
if args.database and (not args.database.startswith('poll_pool_read_mix_query_') or not args.database.replace('_','').isalnum()):parser.error('synthetic database only')
env={k:v for k,v in environment().items() if not k.startswith('POLL_POOL_')}
dependency_lock=(PRIVATE/'runner.lock').open('w')
try:fcntl.flock(dependency_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
except BlockingIOError:
print('Isolated query dependencies already in use; comparison not started.');return 2
if args.start:start(env)
folder=ROOT/'local-data/diagnostics'/('query-serving-'+dt.datetime.now(dt.timezone.utc).strftime('%Y%m%dT%H%M%SZ'))
folder.mkdir(parents=True)
database=args.database or 'poll_pool_read_mix_query_'+uuid.uuid4().hex[:16]
env.update(POLL_POOL_LOAD='1',POLL_POOL_LOCAL_MONGO_URI='mongodb://127.0.0.1:27019/?replicaSet=query-rs&directConnection=true',POLL_POOL_FIXTURE_DATABASE=database,
POLL_POOL_KEEP_FIXTURE='1',POLL_POOL_QUERY_SERVING_PREPARE='1',POLL_POOL_QUERY_PANELS='1',POLL_POOL_READMODELS='1',POLL_POOL_SEARCH='1',POLL_POOL_SEARCH_INDEXES='1',POLL_POOL_PAGE_SHAPE='1',POLL_POOL_ADMIN_TASKS='1' if args.production_shape else '0',POLL_POOL_AUTHORIZATION_STATE='initialized',
POLL_POOL_WARMUP_SECONDS=str(args.warmup),POLL_POOL_DURATION_SECONDS=str(args.duration),ANNOTATION_SOURCE_VALIDATION_MODE='advisory')
if args.production_shape:
env.update(POLL_POOL_PRODUCTION_SHAPE='1', POLL_POOL_RUN_PAYLOAD_KIB='80',
POLL_POOL_TASK_PAYLOAD_KIB='16', POLL_POOL_AVAILABLE_PER_STAGE='215',
POLL_POOL_AUTHOR_HISTORY='100', GOMAXPROCS='2')
before=source_fingerprint(ROOT)
report={'database':database,'inputDigest':before,'mongoEnvironment':'native-uncapped' if native_pid else 'docker-2cpu-4g','shape':'production' if args.production_shape else 'small','rounds':[],'resourceSamples':[],'passed':False,'limits':['synthetic loopback API harness, no media or production writes','process resource sampling includes builder and handler; not Kubernetes Pod limits']}
if native_pid:report['limits'].append('native Mongo has fixed 0.5GiB WiredTiger cache but no container CPU/RSS cap; not comparable with earlier Docker timings')
argv=['go','test','./internal/api','-run','^TestPollPoolReadMixLocal$','-count=1','-timeout=45m','-v']
def run(name,updates):
settings={**env,**updates,'POLL_POOL_OUTPUT':str(folder/(name+'.json'))}
active['phase']=name
started=dt.datetime.now(dt.timezone.utc).isoformat()
result=run_command(argv,settings,args.warmup+args.duration+900)
proof=write_log(folder,argv,result.stdout+'\n'+result.stderr,'passed' if result.returncode==0 else 'failed')
row={'phase':name,'startedAt':started,'endedAt':dt.datetime.now(dt.timezone.utc).isoformat(),'exitCode':result.returncode,**proof,'report':settings['POLL_POOL_OUTPUT']}
report['rounds'].append(row)
(folder/'comparison.json').write_text(json.dumps(report,indent=2)+'\n')
print(json.dumps(row),flush=True)
return result.returncode==0
active={'phase':'prepare'}
if not args.database:
# Synthetic setup is not measured. Avoid a separate majority journal
# barrier for every seed upsert; measured phases reconnect with the
# normal URI/write concern, including background publication writes.
setup={'POLL_POOL_FIXTURE_ONLY':'1','POLL_POOL_LOCAL_MONGO_URI':env['POLL_POOL_LOCAL_MONGO_URI']+'&w=1'}
if not run('prepare',setup):return 1
stopped=threading.Event()
def resources():
while not stopped.is_set():
try:
containers=['fanyu-query-local-valkey-1','fanyu-query-local-typesense-1']
if not native_pid:containers.insert(0,'fanyu-query-local-mongo-1')
p=subprocess.run(['docker','stats','--no-stream','--format','{{json .}}',*containers],capture_output=True,text=True,timeout=8)
if p.returncode==0:
row={'phase':active['phase'],'at':dt.datetime.now(dt.timezone.utc).isoformat(),'containers':[json.loads(line) for line in p.stdout.splitlines()]}
if native_pid:
process=subprocess.run(['ps','-p',str(native_pid),'-o','%cpu=,rss=,vsz=,time='],capture_output=True,text=True,timeout=2,check=True)
row['nativeMongo']=dict(zip(('cpuPercentLifetime','rssKiB','virtualKiB','cpuTime'),process.stdout.split()))
report['resourceSamples'].append(row)
except (OSError,ValueError,subprocess.TimeoutExpired):pass
(folder/'progress.json').write_text(json.dumps({'phase':active['phase'],'at':dt.datetime.now(dt.timezone.utc).isoformat(),'samples':len(report['resourceSamples'])})+'\n')
stopped.wait(2)
thread=threading.Thread(target=resources,daemon=True);thread.start()
try:
for i in range(args.rounds):
baseline='baseline' if i==0 else 'baseline-'+str(i+1)
run(baseline,{'POLL_POOL_REUSE_FIXTURE':'1','POLL_POOL_ALLOW_BASELINE_FAILURE':'1'})
if not run('candidate-'+str(i+1),{'POLL_POOL_REUSE_FIXTURE':'1','POLL_POOL_QUERY_SERVING':'1','POLL_POOL_MAX_SEARCH_P95_MS':'2000'}):break
finally:
stopped.set();thread.join(timeout=10)
report['inputsChangedDuringRun']=source_fingerprint(ROOT)!=before
candidates=[r for r in report['rounds'] if r['phase'].startswith('candidate-')]
report['comparisons']=[]
try:
for i,row in enumerate(candidates):
basefile='baseline.json' if i==0 else 'baseline-'+str(i+1)+'.json'
report['comparisons'].append(compare(json.loads((folder/basefile).read_text()),json.loads(Path(row['report']).read_text())))
except (OSError,ValueError,KeyError):report['comparisons'].append({'passed':False,'problems':['missing/invalid comparison report']})
report['resourceReviewRequired']=len(report['comparisons'])==3 and any(all(r.get(key,0)>1.1 for r in report['comparisons']) for key in ('cpuRatio','retainedHeapRatio'))
report['passed']=not report['resourceReviewRequired'] and len(candidates)==args.rounds and all(r['exitCode']==0 for r in candidates) and not report['inputsChangedDuringRun'] and all(r['passed'] for r in report['comparisons'])
(folder/'comparison.json').write_text(json.dumps(report,indent=2)+'\n')
print('Comparison: '+str(folder/'comparison.json'),flush=True)
return 0 if report['passed'] else 1
if __name__=='__main__':raise SystemExit(main())