111 lines
8.1 KiB
Python
111 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare the same synthetic API fixture before/after query serving."""
|
|
import argparse
|
|
import fcntl
|
|
import datetime as dt
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from test_query_serving import ROOT, PRIVATE, environment, start, run_command
|
|
from lib.query_serving_evidence import compare
|
|
from lib.release_checks import write_log, source_fingerprint
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser()
|
|
parser.add_argument('--start',action='store_true')
|
|
parser.add_argument('--production-shape',action='store_true')
|
|
parser.add_argument('--rounds',type=int,default=1)
|
|
parser.add_argument('--warmup',type=int,default=5)
|
|
parser.add_argument('--duration',type=int,default=30)
|
|
parser.add_argument('--database',help='reuse this script\'s existing synthetic fixture')
|
|
parser.add_argument('--native-mongo',action='store_true',help='use task-owned native Mongo on loopback 27019; never starts/stops another service')
|
|
args=parser.parse_args()
|
|
native_pid=None
|
|
if args.native_mongo:
|
|
if args.start:parser.error('start the native replica set and optional Docker components separately')
|
|
native_pid=int((PRIVATE/'native/mongod.pid').read_text().strip())
|
|
command=subprocess.run(['ps','-p',str(native_pid),'-o','command='],capture_output=True,text=True,check=True).stdout
|
|
if 'local-data/query-serving-local/native/' not in command or '--port 27019' not in command:parser.error('task-owned native Mongo process required')
|
|
if not 1<=args.rounds<=3 or not 1<=args.duration<=1800 or not 1<=args.warmup<=300:parser.error('bounded rounds/times required')
|
|
if args.database and (not args.database.startswith('poll_pool_read_mix_query_') or not args.database.replace('_','').isalnum()):parser.error('synthetic database only')
|
|
env={k:v for k,v in environment().items() if not k.startswith('POLL_POOL_')}
|
|
dependency_lock=(PRIVATE/'runner.lock').open('w')
|
|
try:fcntl.flock(dependency_lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
print('Isolated query dependencies already in use; comparison not started.');return 2
|
|
if args.start:start(env)
|
|
folder=ROOT/'local-data/diagnostics'/('query-serving-'+dt.datetime.now(dt.timezone.utc).strftime('%Y%m%dT%H%M%SZ'))
|
|
folder.mkdir(parents=True)
|
|
database=args.database or 'poll_pool_read_mix_query_'+uuid.uuid4().hex[:16]
|
|
env.update(POLL_POOL_LOAD='1',POLL_POOL_LOCAL_MONGO_URI='mongodb://127.0.0.1:27019/?replicaSet=query-rs&directConnection=true',POLL_POOL_FIXTURE_DATABASE=database,
|
|
POLL_POOL_KEEP_FIXTURE='1',POLL_POOL_QUERY_SERVING_PREPARE='1',POLL_POOL_QUERY_PANELS='1',POLL_POOL_READMODELS='1',POLL_POOL_SEARCH='1',POLL_POOL_SEARCH_INDEXES='1',POLL_POOL_PAGE_SHAPE='1',POLL_POOL_ADMIN_TASKS='1' if args.production_shape else '0',POLL_POOL_AUTHORIZATION_STATE='initialized',
|
|
POLL_POOL_WARMUP_SECONDS=str(args.warmup),POLL_POOL_DURATION_SECONDS=str(args.duration),ANNOTATION_SOURCE_VALIDATION_MODE='advisory')
|
|
if args.production_shape:
|
|
env.update(POLL_POOL_PRODUCTION_SHAPE='1', POLL_POOL_RUN_PAYLOAD_KIB='80',
|
|
POLL_POOL_TASK_PAYLOAD_KIB='16', POLL_POOL_AVAILABLE_PER_STAGE='215',
|
|
POLL_POOL_AUTHOR_HISTORY='100', GOMAXPROCS='2')
|
|
before=source_fingerprint(ROOT)
|
|
report={'database':database,'inputDigest':before,'mongoEnvironment':'native-uncapped' if native_pid else 'docker-2cpu-4g','shape':'production' if args.production_shape else 'small','rounds':[],'resourceSamples':[],'passed':False,'limits':['synthetic loopback API harness, no media or production writes','process resource sampling includes builder and handler; not Kubernetes Pod limits']}
|
|
if native_pid:report['limits'].append('native Mongo has fixed 0.5GiB WiredTiger cache but no container CPU/RSS cap; not comparable with earlier Docker timings')
|
|
argv=['go','test','./internal/api','-run','^TestPollPoolReadMixLocal$','-count=1','-timeout=45m','-v']
|
|
def run(name,updates):
|
|
settings={**env,**updates,'POLL_POOL_OUTPUT':str(folder/(name+'.json'))}
|
|
active['phase']=name
|
|
started=dt.datetime.now(dt.timezone.utc).isoformat()
|
|
result=run_command(argv,settings,args.warmup+args.duration+900)
|
|
proof=write_log(folder,argv,result.stdout+'\n'+result.stderr,'passed' if result.returncode==0 else 'failed')
|
|
row={'phase':name,'startedAt':started,'endedAt':dt.datetime.now(dt.timezone.utc).isoformat(),'exitCode':result.returncode,**proof,'report':settings['POLL_POOL_OUTPUT']}
|
|
report['rounds'].append(row)
|
|
(folder/'comparison.json').write_text(json.dumps(report,indent=2)+'\n')
|
|
print(json.dumps(row),flush=True)
|
|
return result.returncode==0
|
|
active={'phase':'prepare'}
|
|
if not args.database:
|
|
# Synthetic setup is not measured. Avoid a separate majority journal
|
|
# barrier for every seed upsert; measured phases reconnect with the
|
|
# normal URI/write concern, including background publication writes.
|
|
setup={'POLL_POOL_FIXTURE_ONLY':'1','POLL_POOL_LOCAL_MONGO_URI':env['POLL_POOL_LOCAL_MONGO_URI']+'&w=1'}
|
|
if not run('prepare',setup):return 1
|
|
stopped=threading.Event()
|
|
def resources():
|
|
while not stopped.is_set():
|
|
try:
|
|
containers=['fanyu-query-local-valkey-1','fanyu-query-local-typesense-1']
|
|
if not native_pid:containers.insert(0,'fanyu-query-local-mongo-1')
|
|
p=subprocess.run(['docker','stats','--no-stream','--format','{{json .}}',*containers],capture_output=True,text=True,timeout=8)
|
|
if p.returncode==0:
|
|
row={'phase':active['phase'],'at':dt.datetime.now(dt.timezone.utc).isoformat(),'containers':[json.loads(line) for line in p.stdout.splitlines()]}
|
|
if native_pid:
|
|
process=subprocess.run(['ps','-p',str(native_pid),'-o','%cpu=,rss=,vsz=,time='],capture_output=True,text=True,timeout=2,check=True)
|
|
row['nativeMongo']=dict(zip(('cpuPercentLifetime','rssKiB','virtualKiB','cpuTime'),process.stdout.split()))
|
|
report['resourceSamples'].append(row)
|
|
except (OSError,ValueError,subprocess.TimeoutExpired):pass
|
|
(folder/'progress.json').write_text(json.dumps({'phase':active['phase'],'at':dt.datetime.now(dt.timezone.utc).isoformat(),'samples':len(report['resourceSamples'])})+'\n')
|
|
stopped.wait(2)
|
|
thread=threading.Thread(target=resources,daemon=True);thread.start()
|
|
try:
|
|
for i in range(args.rounds):
|
|
baseline='baseline' if i==0 else 'baseline-'+str(i+1)
|
|
run(baseline,{'POLL_POOL_REUSE_FIXTURE':'1','POLL_POOL_ALLOW_BASELINE_FAILURE':'1'})
|
|
if not run('candidate-'+str(i+1),{'POLL_POOL_REUSE_FIXTURE':'1','POLL_POOL_QUERY_SERVING':'1','POLL_POOL_MAX_SEARCH_P95_MS':'2000'}):break
|
|
finally:
|
|
stopped.set();thread.join(timeout=10)
|
|
report['inputsChangedDuringRun']=source_fingerprint(ROOT)!=before
|
|
candidates=[r for r in report['rounds'] if r['phase'].startswith('candidate-')]
|
|
report['comparisons']=[]
|
|
try:
|
|
for i,row in enumerate(candidates):
|
|
basefile='baseline.json' if i==0 else 'baseline-'+str(i+1)+'.json'
|
|
report['comparisons'].append(compare(json.loads((folder/basefile).read_text()),json.loads(Path(row['report']).read_text())))
|
|
except (OSError,ValueError,KeyError):report['comparisons'].append({'passed':False,'problems':['missing/invalid comparison report']})
|
|
report['resourceReviewRequired']=len(report['comparisons'])==3 and any(all(r.get(key,0)>1.1 for r in report['comparisons']) for key in ('cpuRatio','retainedHeapRatio'))
|
|
report['passed']=not report['resourceReviewRequired'] and len(candidates)==args.rounds and all(r['exitCode']==0 for r in candidates) and not report['inputsChangedDuringRun'] and all(r['passed'] for r in report['comparisons'])
|
|
(folder/'comparison.json').write_text(json.dumps(report,indent=2)+'\n')
|
|
print('Comparison: '+str(folder/'comparison.json'),flush=True)
|
|
return 0 if report['passed'] else 1
|
|
|
|
if __name__=='__main__':raise SystemExit(main())
|