droboticflow/scripts/local/test_query_shadow_monitor.py
2026-11-14 20:59:54 -08:00

138 lines
8.1 KiB
Python

import datetime as dt
import hashlib
import importlib.util
import io
import json
import os
from pathlib import Path
from types import SimpleNamespace
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location('monitor', Path(__file__).with_name('query-shadow-monitor.py'))
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
UTC = dt.timezone.utc
class MonitorTests(unittest.TestCase):
def make(self):
b = dict(sourceNamespace='authority', shadowNamespace='shadow', sourceDatabase='source', targetDatabase='query',
servingNamespace='serving', controllers=[dict(name='query-replica', uid='p'), dict(name='query-read-builder', uid='b')],
sourceEndpointSHA256=hashlib.sha256(b'private-source:27017').hexdigest())
return m.ClusterObserver(SimpleNamespace(kubeconfig='unused'), b, {})
def test_clock_resolution_and_backwards_step(self):
start = dt.datetime(2026, 11, 15, 5, 0, 0, 400000, tzinfo=UTC)
low, high = m.date_interval('Sun, 15 Nov 2026 05:00:00 GMT', start, start + dt.timedelta(seconds=.1), .1)
self.assertAlmostEqual(low, -.5); self.assertAlmostEqual(high, .6)
for header, end, elapsed in [(None, start, .1), ('bad', start, .1),
('Sun, 15 Nov 2026 05:00:00 GMT', start - dt.timedelta(seconds=15), .1),
('Sun, 15 Nov 2026 05:00:00 GMT', start + dt.timedelta(seconds=3), 3)]:
with self.assertRaises(m.ProbeUnavailable): m.date_interval(header, start, end, elapsed)
def test_http_clock_all_pods_bounded_without_exec(self):
pods = [dict(metadata=dict(uid='pod' + str(i)), status=dict(phase='Running', podIP='10.0.0.' + str(i))) for i in range(1, 5)]
now = dt.datetime(2026, 11, 15, 5, 0, 0, 400000, tzinfo=UTC)
dates = [now + dt.timedelta(seconds=x) for x in (0, .05, .6, .65)] * 4
response = SimpleNamespace(status=200, headers={'Date': 'Sun, 15 Nov 2026 05:00:00 GMT'})
class Context:
def __enter__(self): return response
def __exit__(self, *args): pass
calls = []
def opener(request, timeout):
calls.append((request.get_method(), request.full_url, timeout)); return Context()
m.check_http_clocks(pods, 3001, opener=opener, utcnow=iter(dates).__next__,
monotonic=iter([0, .05, .6, .65] * 4).__next__, sleep=lambda _: None, diagnostic=lambda _: None)
self.assertEqual(len(calls), 8)
self.assertTrue(all(method == 'HEAD' and url.endswith(':3001/') and timeout == 2 for method, url, timeout in calls))
def test_clock_failure_is_diagnosed_and_retried_three_times(self):
pod = dict(metadata=dict(uid='pod'), status=dict(phase='Running', podIP='10.0.0.1'))
diagnostics = []
with self.assertRaises(m.ProbeUnavailable):
m.check_http_clocks([pod], 3001, opener=lambda *a, **k: (_ for _ in ()).throw(OSError('credential must not print')),
sleep=lambda _: None, diagnostic=diagnostics.append)
self.assertEqual(len([x for x in diagnostics if x.get('event') == 'probe_retry']), 3)
self.assertNotIn('credential', json.dumps(diagnostics))
def test_source_and_target_credentials_are_separated_no_relay(self):
o = self.make(); calls = []
o.run_binary = lambda args, env, data=None: calls.append((args, env, data)) or '{}'
o.secret = lambda *a: self.fail('Secret API must never be used')
with patch.dict(os.environ, dict(SHADOW_MONITOR_SOURCE_URI='mongodb://user:password@private-source:27017/source',
SHADOW_MONITOR_TARGET_URI='mongodb://target:secret@query:27129/query',
MONGO_URI='untrusted', QUERY_READ_MONGO_URI='untrusted')):
o.source_sample(); o.publish(dict(ready=True))
self.assertIn('private-source', calls[0][1]['MONGO_URI'])
self.assertEqual(calls[0][1]['QUERY_READ_MONGO_URI'], '')
self.assertNotIn('MONGO_URI', calls[1][1])
self.assertTrue(all(not any(k.startswith('SHADOW_MONITOR_') for k in c[1]) for c in calls))
with patch.dict(os.environ, dict(SHADOW_MONITOR_SOURCE_URI='mongodb://wrong:27017/source')):
with self.assertRaises(ValueError): o.source_sample()
def test_host_space_uses_parent_mount_not_node_proxy(self):
o = self.make(); calls = []
o.call = lambda *a, **k: calls.append(a) or '30000000 4096\n'
self.assertEqual(o.host_filesystem()['availableBytes'], 30000000 * 4096)
self.assertEqual(calls[0][-1], '/lab')
self.assertNotIn('proxy', str(calls))
o.call = lambda *a: 'unknown'
with self.assertRaises(m.ProbeUnavailable): o.host_filesystem()
def test_stop_only_scale_uid_rv_cas_and_restart_does_not_resume(self):
o = self.make(); calls = []
o.get = lambda ns, kind, name: dict(metadata=dict(uid='p' if kind.endswith('query-replica') else 'b', resourceVersion='7'), spec=dict(replicas=1))
o.call = lambda *a, **k: calls.append(a) or ''
o.stop_shadow()
self.assertEqual(len(calls), 2)
for call in calls:
self.assertIn('--subresource=scale', call)
payload = json.loads(call[-1])
self.assertEqual([(x['op'], x['path']) for x in payload], [('test', '/metadata/uid'), ('test', '/metadata/resourceVersion'), ('replace', '/spec/replicas')])
self.assertEqual(payload[-1]['value'], 0)
other = self.make()
self.assertEqual(other.high, 0) # Constructor performs no Kubernetes/database writes.
other.get = lambda *a: dict(metadata=dict(uid='replaced'), spec=dict(replicas=1))
with self.assertRaises(ValueError): other.stop_shadow()
def test_parallel_business_logs_require_complete_coverage(self):
b = dict(sourceNamespace='authority', sourceNamespaceUID='ns', apiDeploymentUID='api', apiImage='image', apiReplicas=4,
apiPodUIDs=['p'+str(i) for i in range(4)], podRestarts={'p'+str(i): 0 for i in range(4)},
tailLimitsMs=dict(save=5000, slice=10000, submit=15000, review=5000, acceptance=15000))
pods = [dict(metadata=dict(uid='p'+str(i), name='api-'+str(i)), status=dict(phase='Running', containerStatuses=[dict(ready=True, restartCount=0)])) for i in range(4)]
deployment = dict(metadata=dict(uid='api'), spec=dict(template=dict(spec=dict(containers=[dict(name='api', image='image')]))), status=dict(updatedReplicas=4, availableReplicas=4))
logs = []
def call(k, *a):
if 'namespace' in a: return json.dumps(dict(metadata=dict(uid='ns')))
if 'deployment' in a: return json.dumps(deployment)
if 'pods' in a: return json.dumps(dict(items=pods))
logs.append(a[a.index('logs')+1]); return ''
result = m.work.collect([], b, caller=call, log_workers=4)
self.assertEqual(set(logs), {'api-'+str(i) for i in range(4)})
self.assertEqual(result['monitoredPods'], 4)
def denied(k, *a):
if 'api-3' in a: raise m.ProbeUnavailable('denied')
return call(k, *a)
with self.assertRaises(m.ProbeUnavailable): m.work.collect([], b, caller=denied, log_workers=4)
def test_stop_failure_still_attempts_other_controller(self):
o = self.make(); calls = []
def get(ns, kind, name):
calls.append(kind)
if kind.endswith('query-replica'): raise OSError('private error')
return dict(metadata=dict(uid='b', resourceVersion='8'), spec=dict(replicas=0))
o.get = get
with patch.object(m, 'emit') as diagnostic:
with self.assertRaises(ValueError): o.stop_shadow()
self.assertEqual(len(calls), 2)
self.assertNotIn('private error', str(diagnostic.call_args))
def test_scale_omits_zero_replicas(self):
o = self.make()
o.get = lambda ns, kind, name: dict(metadata=dict(uid='p' if kind.endswith('query-replica') else 'b'), spec={})
o.call = lambda *a, **k: self.fail('already stopped Scale must not be patched')
o.stop_shadow()
if __name__ == '__main__': unittest.main()