droboticflow/tests/release/test_query_serving_evidence.py

43 lines
2.5 KiB
Python

import sys
import unittest
from pathlib import Path
sys.path.insert(0,str(Path(__file__).resolve().parents[2]/'scripts'))
from lib.query_serving_evidence import compare
class QueryEvidenceTests(unittest.TestCase):
def fixture(self):
sample={'Actor':'reviewer','Key':'acceptance.all.search','Code':200,'MS':20,'Hash':'business-id-claim-counts'}
return {'mongo_clock_samples':100, 'mongo_clock_max_jump_ms':1, 'fixture_digest':'same','actors':81,'errors':0,'mongo_write_commands':0,'metrics':{'acceptance.all.search':{'p95_ms':20}},'samples':[sample], 'cold_samples':[], 'query_serving_statistics':dict(report_hits=1,search_hits=1,typesense_reads=1,cache_hits=1)}
def test_requires_real_measurements_and_active_components(self):
a=self.fixture();b=self.fixture();self.assertTrue(compare(a,b)['passed'])
b['query_serving_statistics']={};self.assertFalse(compare(a,b)['passed'])
self.assertFalse(compare({}, {})['passed'])
def test_results_and_latency_are_both_mandatory(self):
a=self.fixture();b=self.fixture();b['samples'][0]['Hash']='lost-task'
self.assertFalse(compare(a,b)['passed'])
b=self.fixture();b['metrics']['acceptance.all.search']['p95_ms']=2001
self.assertFalse(compare(a,b)['passed'])
def test_business_write_and_cold_panel_failure_are_not_success(self):
a=self.fixture();b=self.fixture();b['mongo_write_commands']=1
self.assertFalse(compare(a,b)['passed'])
b=self.fixture();b['cold_samples']=[{'Actor':'admin','Key':'admin.production','Code':202,'MS':1,'Hash':'empty'}]
self.assertFalse(compare(a,b)['passed'])
def test_baseline_timeout_keeps_equivalence_pending(self):
a=self.fixture();b=self.fixture();a['samples'][0]['Code']=500
r=compare(a,b);self.assertFalse(r['passed']);self.assertTrue(r['baselineUnavailableComparisons'])
def test_unstable_clock_cannot_be_accepted(self):
a=self.fixture();b=self.fixture();a['mongo_clock_max_jump_ms']=18000
self.assertFalse(compare(a,b)['passed'])
a=self.fixture();del a['mongo_clock_samples']
self.assertFalse(compare(a,b)['passed'])
def test_serial_reference_closes_gap_without_changing_latency(self):
a=self.fixture();b=self.fixture();sample=dict(a['samples'][0]);a['samples'][0]['Code']=500
a['reference_samples']=[sample]
self.assertTrue(compare(a,b)['passed'])
b['metrics']['acceptance.all.search']['p95_ms']=2001
self.assertFalse(compare(a,b)['passed'])
if __name__=='__main__':unittest.main()