forked from longtao.wu/droboticflow
52 lines
3.0 KiB
Python
52 lines
3.0 KiB
Python
"""Bounded, diagnosed read-only probes. Never retries returned unhealthy states."""
|
|
import datetime as dt
|
|
import json
|
|
import math
|
|
import subprocess
|
|
import time
|
|
|
|
class ProbeUnavailable(RuntimeError):
|
|
pass
|
|
|
|
def emit(value):
|
|
print(json.dumps(value, sort_keys=True), flush=True)
|
|
|
|
def retry_probe(probe, phase, attempts=3, delay=.5, diagnostic=emit, sleep=time.sleep):
|
|
if not 1 <= attempts <= 3 or not 0 <= delay <= 1: raise ValueError('invalid retry budget')
|
|
for index in range(attempts):
|
|
try: return probe()
|
|
except (ProbeUnavailable, subprocess.TimeoutExpired, OSError) as error:
|
|
diagnostic(dict(event='probe_retry', phase=phase, attempt=index+1, maxAttempts=attempts, failureType=type(error).__name__))
|
|
if index+1 == attempts: raise ProbeUnavailable('bounded probe exhausted') from None
|
|
sleep(delay*(index+1))
|
|
|
|
def check_clocks(call, namespace, pods, container='api', attempts=3, diagnostic=emit,
|
|
utcnow=lambda: dt.datetime.now(dt.timezone.utc), monotonic=time.monotonic, sleep=time.sleep):
|
|
results=[]
|
|
for pod in pods:
|
|
def measure():
|
|
start=utcnow(); tick=monotonic()
|
|
try:
|
|
text=call('-n', namespace, 'exec', pod['metadata']['name'], '-c', container, '--', 'date', '-u', '+%s.%N', timeout=5)
|
|
remote=float(text.strip())
|
|
except (ValueError, RuntimeError, subprocess.TimeoutExpired, OSError) as error:
|
|
diagnostic(dict(event='clock_probe', node=pod['spec']['nodeName'], podUID=pod['metadata']['uid'], available=False, failureType=type(error).__name__))
|
|
raise ProbeUnavailable('clock transport unavailable') from None
|
|
end=utcnow();rtt=monotonic()-tick;offset=remote-(start+(end-start)/2).timestamp()
|
|
value=dict(event='clock_probe',node=pod['spec']['nodeName'],podUID=pod['metadata']['uid'],available=True,offsetSeconds=offset,rttSeconds=rtt,offsetPassed=math.isfinite(offset) and abs(offset)<=1,rttPassed=0<=rtt<=2)
|
|
diagnostic(value)
|
|
if not value['offsetPassed'] or not value['rttPassed']: raise ProbeUnavailable('clock gate failed')
|
|
return value
|
|
results.append(retry_probe(measure,'clock',attempts=attempts,diagnostic=diagnostic,sleep=sleep))
|
|
return results
|
|
|
|
def parse_memory(current, maximum, events):
|
|
used=int(current.strip());limit=int(maximum.strip())
|
|
counters={line.split()[0]:int(line.split()[1]) for line in events.splitlines() if len(line.split())==2}
|
|
if used<0 or limit<=0 or not {'oom','oom_kill'} <= counters.keys(): raise ValueError('memory evidence incomplete')
|
|
return dict(usedBytes=used,limitBytes=limit,oom=counters['oom'],oomKill=counters['oom_kill'],pressure=used>=.98*limit or counters['oom']>0 or counters['oom_kill']>0)
|
|
|
|
def persistence_failed(text):
|
|
lower=text.lower()
|
|
return any(token in lower for token in ('wt_panic','failed to fsync','input/output error','no space left on device','journal write failed','fatal assertion'))
|