droboticflow/scripts/local/query_shadow_health.py
d-robotics ad1d91a9fc
Some checks are pending
web-ci / checks (push) Waiting to run
web-ci / pool-search-mongo (push) Waiting to run
web-ci / work-availability (push) Blocked by required conditions
web-ci / query-serving (push) Blocked by required conditions
🛠️ Allow guarded shadow initialization during maintenance windows
2026-11-14 18:30:51 -08:00

52 lines
3.0 KiB
Python

"""Bounded, diagnosed read-only probes. Never retries returned unhealthy states."""
import datetime as dt
import json
import math
import subprocess
import time
class ProbeUnavailable(RuntimeError):
pass
def emit(value):
print(json.dumps(value, sort_keys=True), flush=True)
def retry_probe(probe, phase, attempts=3, delay=.5, diagnostic=emit, sleep=time.sleep):
if not 1 <= attempts <= 3 or not 0 <= delay <= 1: raise ValueError('invalid retry budget')
for index in range(attempts):
try: return probe()
except (ProbeUnavailable, subprocess.TimeoutExpired, OSError) as error:
diagnostic(dict(event='probe_retry', phase=phase, attempt=index+1, maxAttempts=attempts, failureType=type(error).__name__))
if index+1 == attempts: raise ProbeUnavailable('bounded probe exhausted') from None
sleep(delay*(index+1))
def check_clocks(call, namespace, pods, container='api', attempts=3, diagnostic=emit,
utcnow=lambda: dt.datetime.now(dt.timezone.utc), monotonic=time.monotonic, sleep=time.sleep):
results=[]
for pod in pods:
def measure():
start=utcnow(); tick=monotonic()
try:
text=call('-n', namespace, 'exec', pod['metadata']['name'], '-c', container, '--', 'date', '-u', '+%s.%N', timeout=5)
remote=float(text.strip())
except (ValueError, RuntimeError, subprocess.TimeoutExpired, OSError) as error:
diagnostic(dict(event='clock_probe', node=pod['spec']['nodeName'], podUID=pod['metadata']['uid'], available=False, failureType=type(error).__name__))
raise ProbeUnavailable('clock transport unavailable') from None
end=utcnow();rtt=monotonic()-tick;offset=remote-(start+(end-start)/2).timestamp()
value=dict(event='clock_probe',node=pod['spec']['nodeName'],podUID=pod['metadata']['uid'],available=True,offsetSeconds=offset,rttSeconds=rtt,offsetPassed=math.isfinite(offset) and abs(offset)<=1,rttPassed=0<=rtt<=2)
diagnostic(value)
if not value['offsetPassed'] or not value['rttPassed']: raise ProbeUnavailable('clock gate failed')
return value
results.append(retry_probe(measure,'clock',attempts=attempts,diagnostic=diagnostic,sleep=sleep))
return results
def parse_memory(current, maximum, events):
used=int(current.strip());limit=int(maximum.strip())
counters={line.split()[0]:int(line.split()[1]) for line in events.splitlines() if len(line.split())==2}
if used<0 or limit<=0 or not {'oom','oom_kill'} <= counters.keys(): raise ValueError('memory evidence incomplete')
return dict(usedBytes=used,limitBytes=limit,oom=counters['oom'],oomKill=counters['oom_kill'],pressure=used>=.98*limit or counters['oom']>0 or counters['oom_kill']>0)
def persistence_failed(text):
lower=text.lower()
return any(token in lower for token in ('wt_panic','failed to fsync','input/output error','no space left on device','journal write failed','fatal assertion'))