import datetime as dt import hashlib import importlib.util import io import json import os from pathlib import Path from types import SimpleNamespace import unittest from unittest.mock import patch spec = importlib.util.spec_from_file_location('monitor', Path(__file__).with_name('query-shadow-monitor.py')) m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m) UTC = dt.timezone.utc class MonitorTests(unittest.TestCase): def make(self): b = dict(sourceNamespace='authority', shadowNamespace='shadow', sourceDatabase='source', targetDatabase='query', servingNamespace='serving', controllers=[dict(name='query-replica', uid='p'), dict(name='query-read-builder', uid='b')], sourceEndpointSHA256=hashlib.sha256(b'private-source:27017').hexdigest()) return m.ClusterObserver(SimpleNamespace(kubeconfig='unused'), b, {}) def test_clock_resolution_and_backwards_step(self): start = dt.datetime(2026, 11, 15, 5, 0, 0, 400000, tzinfo=UTC) low, high = m.date_interval('Sun, 15 Nov 2026 05:00:00 GMT', start, start + dt.timedelta(seconds=.1), .1) self.assertAlmostEqual(low, -.5); self.assertAlmostEqual(high, .6) for header, end, elapsed in [(None, start, .1), ('bad', start, .1), ('Sun, 15 Nov 2026 05:00:00 GMT', start - dt.timedelta(seconds=15), .1), ('Sun, 15 Nov 2026 05:00:00 GMT', start + dt.timedelta(seconds=3), 3)]: with self.assertRaises(m.ProbeUnavailable): m.date_interval(header, start, end, elapsed) def test_http_clock_all_pods_bounded_without_exec(self): pods = [dict(metadata=dict(uid='pod' + str(i)), status=dict(phase='Running', podIP='10.0.0.' + str(i))) for i in range(1, 5)] now = dt.datetime(2026, 11, 15, 5, 0, 0, 400000, tzinfo=UTC) dates = [now + dt.timedelta(seconds=x) for x in (0, .05, .6, .65)] * 4 response = SimpleNamespace(status=200, headers={'Date': 'Sun, 15 Nov 2026 05:00:00 GMT'}) class Context: def __enter__(self): return response def __exit__(self, *args): pass calls = [] def opener(request, timeout): calls.append((request.get_method(), request.full_url, timeout)); return Context() m.check_http_clocks(pods, 3001, opener=opener, utcnow=iter(dates).__next__, monotonic=iter([0, .05, .6, .65] * 4).__next__, sleep=lambda _: None, diagnostic=lambda _: None) self.assertEqual(len(calls), 8) self.assertTrue(all(method == 'HEAD' and url.endswith(':3001/') and timeout == 2 for method, url, timeout in calls)) def test_clock_failure_is_diagnosed_and_retried_three_times(self): pod = dict(metadata=dict(uid='pod'), status=dict(phase='Running', podIP='10.0.0.1')) diagnostics = [] with self.assertRaises(m.ProbeUnavailable): m.check_http_clocks([pod], 3001, opener=lambda *a, **k: (_ for _ in ()).throw(OSError('credential must not print')), sleep=lambda _: None, diagnostic=diagnostics.append) self.assertEqual(len([x for x in diagnostics if x.get('event') == 'probe_retry']), 3) self.assertNotIn('credential', json.dumps(diagnostics)) def test_source_and_target_credentials_are_separated_no_relay(self): o = self.make(); calls = [] o.run_binary = lambda args, env, data=None: calls.append((args, env, data)) or '{}' o.secret = lambda *a: self.fail('Secret API must never be used') with patch.dict(os.environ, dict(SHADOW_MONITOR_SOURCE_URI='mongodb://user:password@private-source:27017/source', SHADOW_MONITOR_TARGET_URI='mongodb://target:secret@query:27129/query', MONGO_URI='untrusted', QUERY_READ_MONGO_URI='untrusted')): o.source_sample(); o.publish(dict(ready=True)) self.assertIn('private-source', calls[0][1]['MONGO_URI']) self.assertEqual(calls[0][1]['QUERY_READ_MONGO_URI'], '') self.assertNotIn('MONGO_URI', calls[1][1]) self.assertTrue(all(not any(k.startswith('SHADOW_MONITOR_') for k in c[1]) for c in calls)) with patch.dict(os.environ, dict(SHADOW_MONITOR_SOURCE_URI='mongodb://wrong:27017/source')): with self.assertRaises(ValueError): o.source_sample() def test_host_space_uses_parent_mount_not_node_proxy(self): o = self.make(); calls = [] o.call = lambda *a, **k: calls.append(a) or '30000000 4096\n' self.assertEqual(o.host_filesystem()['availableBytes'], 30000000 * 4096) self.assertEqual(calls[0][-1], '/lab') self.assertNotIn('proxy', str(calls)) o.call = lambda *a: 'unknown' with self.assertRaises(m.ProbeUnavailable): o.host_filesystem() def test_stop_only_scale_uid_rv_cas_and_restart_does_not_resume(self): o = self.make(); calls = [] o.get = lambda ns, kind, name: dict(metadata=dict(uid='p' if kind.endswith('query-replica') else 'b', resourceVersion='7'), spec=dict(replicas=1)) o.call = lambda *a, **k: calls.append(a) or '' o.stop_shadow() self.assertEqual(len(calls), 2) for call in calls: self.assertIn('--subresource=scale', call) payload = json.loads(call[-1]) self.assertEqual([(x['op'], x['path']) for x in payload], [('test', '/metadata/uid'), ('test', '/metadata/resourceVersion'), ('replace', '/spec/replicas')]) self.assertEqual(payload[-1]['value'], 0) other = self.make() self.assertEqual(other.high, 0) # Constructor performs no Kubernetes/database writes. other.get = lambda *a: dict(metadata=dict(uid='replaced'), spec=dict(replicas=1)) with self.assertRaises(ValueError): other.stop_shadow() def test_parallel_business_logs_require_complete_coverage(self): b = dict(sourceNamespace='authority', sourceNamespaceUID='ns', apiDeploymentUID='api', apiImage='image', apiReplicas=4, apiPodUIDs=['p'+str(i) for i in range(4)], podRestarts={'p'+str(i): 0 for i in range(4)}, tailLimitsMs=dict(save=5000, slice=10000, submit=15000, review=5000, acceptance=15000)) pods = [dict(metadata=dict(uid='p'+str(i), name='api-'+str(i)), status=dict(phase='Running', containerStatuses=[dict(ready=True, restartCount=0)])) for i in range(4)] deployment = dict(metadata=dict(uid='api'), spec=dict(template=dict(spec=dict(containers=[dict(name='api', image='image')]))), status=dict(updatedReplicas=4, availableReplicas=4)) logs = [] def call(k, *a): if 'namespace' in a: return json.dumps(dict(metadata=dict(uid='ns'))) if 'deployment' in a: return json.dumps(deployment) if 'pods' in a: return json.dumps(dict(items=pods)) logs.append(a[a.index('logs')+1]); return '' result = m.work.collect([], b, caller=call, log_workers=4) self.assertEqual(set(logs), {'api-'+str(i) for i in range(4)}) self.assertEqual(result['monitoredPods'], 4) def denied(k, *a): if 'api-3' in a: raise m.ProbeUnavailable('denied') return call(k, *a) with self.assertRaises(m.ProbeUnavailable): m.work.collect([], b, caller=denied, log_workers=4) def test_stop_failure_still_attempts_other_controller(self): o = self.make(); calls = [] def get(ns, kind, name): calls.append(kind) if kind.endswith('query-replica'): raise OSError('private error') return dict(metadata=dict(uid='b', resourceVersion='8'), spec=dict(replicas=0)) o.get = get with patch.object(m, 'emit') as diagnostic: with self.assertRaises(ValueError): o.stop_shadow() self.assertEqual(len(calls), 2) self.assertNotIn('private error', str(diagnostic.call_args)) def test_scale_omits_zero_replicas(self): o = self.make() o.get = lambda ns, kind, name: dict(metadata=dict(uid='p' if kind.endswith('query-replica') else 'b'), spec={}) o.call = lambda *a, **k: self.fail('already stopped Scale must not be patched') o.stop_shadow() if __name__ == '__main__': unittest.main()