droboticflow/scripts/local/test_query_shadow_observer.py

146 lines
9.8 KiB
Python

import importlib.util
import json
from pathlib import Path
from types import SimpleNamespace
import unittest
import tempfile
import datetime as dt
spec = importlib.util.spec_from_file_location('shadow_observer', Path(__file__).with_name('query-shadow-observer.py'))
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
class ObserverTests(unittest.TestCase):
def test_optional_trace_timeout_does_not_pause_successful_work(self):
# Replay the production warning through the actual observation call,
# alongside the authoritative five-operation health feed.
now = dt.datetime.now(dt.timezone.utc)
with tempfile.TemporaryDirectory() as folder:
path = Path(folder) / 'health.json'
path.write_text(json.dumps(dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance',
unexpectedErrors=False, latencyRegressed=False, bindingSHA256='a' * 64, metricKind='http-slow-error-tail')))
b = self.bindings()
b.update(mongoPodUID='query', workHealthBindingSHA256='a' * 64)
o = m.Observer(SimpleNamespace(kubeconfig='unused', work_health=str(path)), b)
o.verify = lambda: None
o.readiness = lambda: True
o.get = lambda *a: dict(metadata=dict(uid='query'))
o.host_filesystem = lambda: dict(availableBytes=200 << 30)
warning = '2026/11/17 10:40:18 [review-trace] list summary unavailable: connection read: i/o timeout'
for log, expected in ((warning, False), ('unknown backend failure: i/o timeout', True),
(warning + '\nsave failed: i/o timeout', True),
(warning + '\npanic: unexpected fault', True),
(warning + '\nsubmit failed', True),
(warning + '\nwrite conflict retry exhausted', True),
('prefix ' + warning, True)):
with self.subTest(log=log):
def call(*a, **kw):
if 'df' in a: return 'Filesystem 1B Used Available Capacity Mounted\nloop 100 10 90 10% /lab/data'
if 'pods' in a:
return json.dumps(dict(items=[] if '--field-selector=spec.nodeName=10.189.16.4' in a
else [dict(metadata=dict(name='api'))]))
if 'logs' in a: return log
self.fail('unexpected probe')
o.call = call
self.assertEqual(o.observation(50, now.isoformat())['workErrors'], expected)
feed = json.loads(path.read_text())
feed['unexpectedErrors'] = True
path.write_text(json.dumps(feed))
log = warning
self.assertTrue(o.observation(50, now.isoformat())['workErrors'])
def test_mixed_writer_versions_are_bound_individually(self):
b = self.bindings()
b.update(sourceNamespaceUID='authority-uid', shadowNamespaceUID='shadow-uid', writerCommit='old')
b['writers'] = [dict(name='writer-' + str(i), uid=str(i), image='image-' + str(i), replicas=1,
commit='new' if i < 2 else 'old') for i in range(5)]
deployments = {w['name']: dict(metadata=dict(uid=w['uid']),
spec=dict(template=dict(spec=dict(containers=[dict(image=w['image'], env=[
dict(name='DROBOTICFLOW_BUILD_COMMIT', value=w['commit']),
dict(name='QUERY_SOURCE_TRACKING', value='1')])]))),
status=dict(availableReplicas=1)) for w in b['writers']}
o = m.Observer(SimpleNamespace(kubeconfig='unused'), b)
o.get = lambda ns, kind, name: dict(metadata=dict(uid=ns + '-uid')) if kind == 'namespace' else deployments[name]
o.verify()
for i, writer in enumerate(b['writers']):
with self.subTest(writer=i):
env = deployments[writer['name']]['spec']['template']['spec']['containers'][0]['env']
env[0]['value'] = 'unexpected'
with self.assertRaises(ValueError): o.verify()
env[0]['value'] = writer['commit']
for writer in b['writers']:
writer.pop('commit')
deployments[writer['name']]['spec']['template']['spec']['containers'][0]['env'][0]['value'] = 'old'
o.verify() # Existing single-version ConfigMaps remain compatible.
b['writerCommit'] = ''
with self.assertRaises(ValueError): o.verify()
def bindings(self):
return dict(sourceNamespace='authority', shadowNamespace='shadow', sourceDatabase='source', targetDatabase='query',
baselineCPU=85, writers=[{} for _ in range(5)], controllers=[dict(name='query-replica', uid='p'), dict(name='query-read-builder', uid='b')])
def test_cpu_and_identity(self):
self.assertAlmostEqual(m.cpu_percent(dict(sourceProcess='s', uptimeMillis=1000, cpuMicroseconds=0),
dict(sourceProcess='s', uptimeMillis=2000, cpuMicroseconds=2000000)), 50)
with self.assertRaises(ValueError):
m.cpu_percent(dict(sourceProcess='s', uptimeMillis=2, cpuMicroseconds=2), dict(sourceProcess='x', uptimeMillis=3, cpuMicroseconds=3))
def test_stop_scope_and_uid_cas(self):
b = self.bindings(); m.check_bindings(b)
observer = m.Observer(SimpleNamespace(kubeconfig='unused', stop_on_failure=True), b); calls = []
observer.get = lambda ns, kind, name: dict(metadata=dict(uid='p' if name=='query-replica' else 'b', resourceVersion='7'), spec=dict(replicas=1))
observer.call = lambda *args, **kwargs: calls.append(args) or ''
observer.stop_shadow()
self.assertEqual(len(calls), 2)
for args in calls:
self.assertEqual(args[:4], ('-n', 'shadow', 'patch', 'deployment'))
payload = json.loads(args[-1]); self.assertEqual(payload[0]['op'], 'test'); self.assertEqual(payload[-1]['value'], 0)
observer.get = lambda *args: dict(metadata=dict(uid='replaced', resourceVersion='8'), spec=dict(replicas=1))
with self.assertRaises(ValueError): observer.stop_shadow()
self.assertEqual(len(calls), 2)
b['controllers'][0]['name'] = 'droboticflow-api'
with self.assertRaises(ValueError): m.check_bindings(b)
def test_distinct_samples_and_thresholds(self):
o = m.Observer(SimpleNamespace(kubeconfig='unused'), self.bindings())
good = dict(cpu=90, ready=True, writersCovered=True, workErrors=False, protectedIOFull=0, targetUsedFraction=.1, hostFreeBytes=120<<30)
self.assertFalse(o.unsafe(good)); self.assertTrue(o.unsafe(good))
good['cpu']=85; self.assertFalse(o.unsafe(good));good['targetUsedFraction']=.7;self.assertTrue(o.unsafe(good))
def test_business_health_missing_stale_and_real_errors(self):
now = dt.datetime.now(dt.timezone.utc)
with tempfile.TemporaryDirectory() as folder:
path = Path(folder) / 'health.json'
with self.assertRaises(FileNotFoundError): m.business_health(path, now)
feed = dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=False, latencyRegressed=False)
path.write_text(json.dumps(feed)); self.assertFalse(m.business_health(path, now))
feed['latencyRegressed'] = True
path.write_text(json.dumps(feed)); self.assertTrue(m.business_health(path, now))
feed['latencyRegressed'] = 'false'
path.write_text(json.dumps(feed))
with self.assertRaises(ValueError): m.business_health(path, now)
feed['latencyRegressed'] = False; feed['at'] = (now - dt.timedelta(seconds=46)).isoformat()
path.write_text(json.dumps(feed))
with self.assertRaises(ValueError): m.business_health(path, now)
feed['at'] = now.isoformat(); feed['coverage'] = 'readiness-only'
path.write_text(json.dumps(feed))
with self.assertRaises(ValueError): m.business_health(path, now)
def test_business_health_exact_binding_required(self):
now = dt.datetime.now(dt.timezone.utc)
with tempfile.TemporaryDirectory() as folder:
path = Path(folder) / 'health.json'
feed = dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=False, latencyRegressed=False, bindingSHA256='a'*64, metricKind='http-slow-error-tail')
path.write_text(json.dumps(feed))
self.assertFalse(m.business_health(path, now, 'a'*64))
with self.assertRaises(ValueError): m.business_health(path, now, 'b'*64)
feed['metricKind'] = 'manual-constant'; path.write_text(json.dumps(feed))
with self.assertRaises(ValueError): m.business_health(path, now, 'a'*64)
def test_maintenance_cpu_is_observed_but_resource_failures_stop(self):
b=self.bindings();b['maintenanceInitialization']=True;b['baselineCPU']=0;b['maintenanceUntil']=(dt.datetime.now(dt.timezone.utc)+dt.timedelta(hours=1)).isoformat()
m.check_bindings(b);o=m.Observer(SimpleNamespace(kubeconfig='unused'),b)
good=dict(cpu=100,ready=True,writersCovered=True,workErrors=False,protectedIOFull=0,targetUsedFraction=.1,hostFreeBytes=120<<30,safetySignalsVersion=1,sourceAvailable=True,memoryPressure=False,persistenceFailure=False)
for cpu in [95,100,110]:
good['cpu']=cpu;self.assertFalse(o.unsafe(good))
for field,value in [('memoryPressure',True),('persistenceFailure',True),('sourceAvailable',False),('workErrors',True),('safetySignalsVersion',0)]:
bad=dict(good);bad[field]=value;self.assertTrue(o.unsafe(bad))
def test_unknown_io_is_not_zero(self):
with self.assertRaises(ValueError): m.full_io_pressure('no evidence')
self.assertEqual(m.full_io_pressure('some avg10=2.0\nfull avg10=1.25 avg60=0'), 1.25)
if __name__ == '__main__': unittest.main()