forked from longtao.wu/droboticflow
146 lines
9.8 KiB
Python
146 lines
9.8 KiB
Python
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
import tempfile
|
|
import datetime as dt
|
|
|
|
spec = importlib.util.spec_from_file_location('shadow_observer', Path(__file__).with_name('query-shadow-observer.py'))
|
|
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
|
|
|
|
class ObserverTests(unittest.TestCase):
|
|
def test_optional_trace_timeout_does_not_pause_successful_work(self):
|
|
# Replay the production warning through the actual observation call,
|
|
# alongside the authoritative five-operation health feed.
|
|
now = dt.datetime.now(dt.timezone.utc)
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
path = Path(folder) / 'health.json'
|
|
path.write_text(json.dumps(dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance',
|
|
unexpectedErrors=False, latencyRegressed=False, bindingSHA256='a' * 64, metricKind='http-slow-error-tail')))
|
|
b = self.bindings()
|
|
b.update(mongoPodUID='query', workHealthBindingSHA256='a' * 64)
|
|
o = m.Observer(SimpleNamespace(kubeconfig='unused', work_health=str(path)), b)
|
|
o.verify = lambda: None
|
|
o.readiness = lambda: True
|
|
o.get = lambda *a: dict(metadata=dict(uid='query'))
|
|
o.host_filesystem = lambda: dict(availableBytes=200 << 30)
|
|
warning = '2026/11/17 10:40:18 [review-trace] list summary unavailable: connection read: i/o timeout'
|
|
for log, expected in ((warning, False), ('unknown backend failure: i/o timeout', True),
|
|
(warning + '\nsave failed: i/o timeout', True),
|
|
(warning + '\npanic: unexpected fault', True),
|
|
(warning + '\nsubmit failed', True),
|
|
(warning + '\nwrite conflict retry exhausted', True),
|
|
('prefix ' + warning, True)):
|
|
with self.subTest(log=log):
|
|
def call(*a, **kw):
|
|
if 'df' in a: return 'Filesystem 1B Used Available Capacity Mounted\nloop 100 10 90 10% /lab/data'
|
|
if 'pods' in a:
|
|
return json.dumps(dict(items=[] if '--field-selector=spec.nodeName=10.189.16.4' in a
|
|
else [dict(metadata=dict(name='api'))]))
|
|
if 'logs' in a: return log
|
|
self.fail('unexpected probe')
|
|
o.call = call
|
|
self.assertEqual(o.observation(50, now.isoformat())['workErrors'], expected)
|
|
feed = json.loads(path.read_text())
|
|
feed['unexpectedErrors'] = True
|
|
path.write_text(json.dumps(feed))
|
|
log = warning
|
|
self.assertTrue(o.observation(50, now.isoformat())['workErrors'])
|
|
|
|
def test_mixed_writer_versions_are_bound_individually(self):
|
|
b = self.bindings()
|
|
b.update(sourceNamespaceUID='authority-uid', shadowNamespaceUID='shadow-uid', writerCommit='old')
|
|
b['writers'] = [dict(name='writer-' + str(i), uid=str(i), image='image-' + str(i), replicas=1,
|
|
commit='new' if i < 2 else 'old') for i in range(5)]
|
|
deployments = {w['name']: dict(metadata=dict(uid=w['uid']),
|
|
spec=dict(template=dict(spec=dict(containers=[dict(image=w['image'], env=[
|
|
dict(name='DROBOTICFLOW_BUILD_COMMIT', value=w['commit']),
|
|
dict(name='QUERY_SOURCE_TRACKING', value='1')])]))),
|
|
status=dict(availableReplicas=1)) for w in b['writers']}
|
|
o = m.Observer(SimpleNamespace(kubeconfig='unused'), b)
|
|
o.get = lambda ns, kind, name: dict(metadata=dict(uid=ns + '-uid')) if kind == 'namespace' else deployments[name]
|
|
o.verify()
|
|
for i, writer in enumerate(b['writers']):
|
|
with self.subTest(writer=i):
|
|
env = deployments[writer['name']]['spec']['template']['spec']['containers'][0]['env']
|
|
env[0]['value'] = 'unexpected'
|
|
with self.assertRaises(ValueError): o.verify()
|
|
env[0]['value'] = writer['commit']
|
|
for writer in b['writers']:
|
|
writer.pop('commit')
|
|
deployments[writer['name']]['spec']['template']['spec']['containers'][0]['env'][0]['value'] = 'old'
|
|
o.verify() # Existing single-version ConfigMaps remain compatible.
|
|
b['writerCommit'] = ''
|
|
with self.assertRaises(ValueError): o.verify()
|
|
|
|
def bindings(self):
|
|
return dict(sourceNamespace='authority', shadowNamespace='shadow', sourceDatabase='source', targetDatabase='query',
|
|
baselineCPU=85, writers=[{} for _ in range(5)], controllers=[dict(name='query-replica', uid='p'), dict(name='query-read-builder', uid='b')])
|
|
def test_cpu_and_identity(self):
|
|
self.assertAlmostEqual(m.cpu_percent(dict(sourceProcess='s', uptimeMillis=1000, cpuMicroseconds=0),
|
|
dict(sourceProcess='s', uptimeMillis=2000, cpuMicroseconds=2000000)), 50)
|
|
with self.assertRaises(ValueError):
|
|
m.cpu_percent(dict(sourceProcess='s', uptimeMillis=2, cpuMicroseconds=2), dict(sourceProcess='x', uptimeMillis=3, cpuMicroseconds=3))
|
|
def test_stop_scope_and_uid_cas(self):
|
|
b = self.bindings(); m.check_bindings(b)
|
|
observer = m.Observer(SimpleNamespace(kubeconfig='unused', stop_on_failure=True), b); calls = []
|
|
observer.get = lambda ns, kind, name: dict(metadata=dict(uid='p' if name=='query-replica' else 'b', resourceVersion='7'), spec=dict(replicas=1))
|
|
observer.call = lambda *args, **kwargs: calls.append(args) or ''
|
|
observer.stop_shadow()
|
|
self.assertEqual(len(calls), 2)
|
|
for args in calls:
|
|
self.assertEqual(args[:4], ('-n', 'shadow', 'patch', 'deployment'))
|
|
payload = json.loads(args[-1]); self.assertEqual(payload[0]['op'], 'test'); self.assertEqual(payload[-1]['value'], 0)
|
|
observer.get = lambda *args: dict(metadata=dict(uid='replaced', resourceVersion='8'), spec=dict(replicas=1))
|
|
with self.assertRaises(ValueError): observer.stop_shadow()
|
|
self.assertEqual(len(calls), 2)
|
|
b['controllers'][0]['name'] = 'droboticflow-api'
|
|
with self.assertRaises(ValueError): m.check_bindings(b)
|
|
def test_distinct_samples_and_thresholds(self):
|
|
o = m.Observer(SimpleNamespace(kubeconfig='unused'), self.bindings())
|
|
good = dict(cpu=90, ready=True, writersCovered=True, workErrors=False, protectedIOFull=0, targetUsedFraction=.1, hostFreeBytes=120<<30)
|
|
self.assertFalse(o.unsafe(good)); self.assertTrue(o.unsafe(good))
|
|
good['cpu']=85; self.assertFalse(o.unsafe(good));good['targetUsedFraction']=.7;self.assertTrue(o.unsafe(good))
|
|
def test_business_health_missing_stale_and_real_errors(self):
|
|
now = dt.datetime.now(dt.timezone.utc)
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
path = Path(folder) / 'health.json'
|
|
with self.assertRaises(FileNotFoundError): m.business_health(path, now)
|
|
feed = dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=False, latencyRegressed=False)
|
|
path.write_text(json.dumps(feed)); self.assertFalse(m.business_health(path, now))
|
|
feed['latencyRegressed'] = True
|
|
path.write_text(json.dumps(feed)); self.assertTrue(m.business_health(path, now))
|
|
feed['latencyRegressed'] = 'false'
|
|
path.write_text(json.dumps(feed))
|
|
with self.assertRaises(ValueError): m.business_health(path, now)
|
|
feed['latencyRegressed'] = False; feed['at'] = (now - dt.timedelta(seconds=46)).isoformat()
|
|
path.write_text(json.dumps(feed))
|
|
with self.assertRaises(ValueError): m.business_health(path, now)
|
|
feed['at'] = now.isoformat(); feed['coverage'] = 'readiness-only'
|
|
path.write_text(json.dumps(feed))
|
|
with self.assertRaises(ValueError): m.business_health(path, now)
|
|
def test_business_health_exact_binding_required(self):
|
|
now = dt.datetime.now(dt.timezone.utc)
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
path = Path(folder) / 'health.json'
|
|
feed = dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=False, latencyRegressed=False, bindingSHA256='a'*64, metricKind='http-slow-error-tail')
|
|
path.write_text(json.dumps(feed))
|
|
self.assertFalse(m.business_health(path, now, 'a'*64))
|
|
with self.assertRaises(ValueError): m.business_health(path, now, 'b'*64)
|
|
feed['metricKind'] = 'manual-constant'; path.write_text(json.dumps(feed))
|
|
with self.assertRaises(ValueError): m.business_health(path, now, 'a'*64)
|
|
def test_maintenance_cpu_is_observed_but_resource_failures_stop(self):
|
|
b=self.bindings();b['maintenanceInitialization']=True;b['baselineCPU']=0;b['maintenanceUntil']=(dt.datetime.now(dt.timezone.utc)+dt.timedelta(hours=1)).isoformat()
|
|
m.check_bindings(b);o=m.Observer(SimpleNamespace(kubeconfig='unused'),b)
|
|
good=dict(cpu=100,ready=True,writersCovered=True,workErrors=False,protectedIOFull=0,targetUsedFraction=.1,hostFreeBytes=120<<30,safetySignalsVersion=1,sourceAvailable=True,memoryPressure=False,persistenceFailure=False)
|
|
for cpu in [95,100,110]:
|
|
good['cpu']=cpu;self.assertFalse(o.unsafe(good))
|
|
for field,value in [('memoryPressure',True),('persistenceFailure',True),('sourceAvailable',False),('workErrors',True),('safetySignalsVersion',0)]:
|
|
bad=dict(good);bad[field]=value;self.assertTrue(o.unsafe(bad))
|
|
def test_unknown_io_is_not_zero(self):
|
|
with self.assertRaises(ValueError): m.full_io_pressure('no evidence')
|
|
self.assertEqual(m.full_io_pressure('some avg10=2.0\nfull avg10=1.25 avg60=0'), 1.25)
|
|
|
|
if __name__ == '__main__': unittest.main()
|