forked from longtao.wu/droboticflow
43 lines
3.0 KiB
Python
43 lines
3.0 KiB
Python
import datetime as dt
|
|
import unittest
|
|
from query_shadow_health import check_clocks, retry_probe, ProbeUnavailable, parse_memory, persistence_failed
|
|
|
|
class HealthProbeTests(unittest.TestCase):
|
|
def test_bounded_retry_has_diagnostics_and_never_retries_bad_result(self):
|
|
events=[];calls=[]
|
|
def work():
|
|
calls.append(1)
|
|
if len(calls)<3: raise ProbeUnavailable('fixed reason')
|
|
return {'healthy':False}
|
|
self.assertEqual(retry_probe(work,'test',diagnostic=events.append,sleep=lambda _:None),{'healthy':False})
|
|
self.assertEqual(len(calls),3);self.assertEqual([e['attempt'] for e in events],[1,2])
|
|
with self.assertRaises(ProbeUnavailable):retry_probe(lambda:(_ for _ in ()).throw(ProbeUnavailable()),'test',diagnostic=events.append,sleep=lambda _:None)
|
|
with self.assertRaises(ValueError):retry_probe(lambda:None,'test',attempts=4)
|
|
def test_clock_offset_and_rtt_are_distinguished_and_bounded(self):
|
|
now=dt.datetime.now(dt.timezone.utc);events=[]
|
|
pods=[dict(metadata=dict(name='api',uid='uid'),spec=dict(nodeName='node'))]
|
|
values=iter([str(now.timestamp()+2),str(now.timestamp())]);ticks=iter([0,.1,1,3.1,4,4.1])
|
|
# first offset2s fails; second RTT2.1 fails; final sample passes.
|
|
def call(*args,**kwargs):
|
|
n=sum(e.get('event')=='clock_probe' for e in events)
|
|
return str(now.timestamp()+2) if n==0 else str(now.timestamp())
|
|
rows=check_clocks(call,'ns',pods,diagnostic=events.append,utcnow=lambda:now,monotonic=lambda:next(ticks),sleep=lambda _:None)
|
|
samples=[e for e in events if e.get('event')=='clock_probe']
|
|
self.assertFalse(samples[0]['offsetPassed']);self.assertTrue(samples[0]['rttPassed'])
|
|
self.assertTrue(samples[1]['offsetPassed']);self.assertFalse(samples[1]['rttPassed'])
|
|
self.assertTrue(rows[0]['offsetPassed']);self.assertEqual(len(samples),3)
|
|
def test_clock_transport_failure_records_reason(self):
|
|
events=[];pods=[dict(metadata=dict(name='api',uid='uid'),spec=dict(nodeName='node'))]
|
|
with self.assertRaises(ProbeUnavailable):check_clocks(lambda *a,**kw:(_ for _ in ()).throw(RuntimeError()),'ns',pods,diagnostic=events.append,sleep=lambda _:None)
|
|
self.assertEqual(sum(e.get('event')=='clock_probe' for e in events),3)
|
|
self.assertTrue(all(e.get('available') is False for e in events if e.get('event')=='clock_probe'))
|
|
def test_memory_and_durability_errors_remain_hard_signals(self):
|
|
self.assertFalse(parse_memory('100','1000','oom 0\noom_kill 0')['pressure'])
|
|
self.assertTrue(parse_memory('990','1000','oom 0\noom_kill 0')['pressure'])
|
|
self.assertTrue(parse_memory('100','1000','oom 0\noom_kill 1')['pressure'])
|
|
with self.assertRaises(ValueError):parse_memory('100','max','')
|
|
self.assertTrue(persistence_failed('WiredTiger WT_PANIC'));self.assertTrue(persistence_failed('journal write failed'))
|
|
self.assertFalse(persistence_failed('checkpoint completed'))
|
|
|
|
if __name__=='__main__':unittest.main()
|