import importlib.util import json from pathlib import Path from types import SimpleNamespace import unittest import tempfile import datetime as dt spec = importlib.util.spec_from_file_location('shadow_observer', Path(__file__).with_name('query-shadow-observer.py')) m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m) class ObserverTests(unittest.TestCase): def test_optional_trace_timeout_does_not_pause_successful_work(self): # Replay the production warning through the actual observation call, # alongside the authoritative five-operation health feed. now = dt.datetime.now(dt.timezone.utc) with tempfile.TemporaryDirectory() as folder: path = Path(folder) / 'health.json' path.write_text(json.dumps(dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=False, latencyRegressed=False, bindingSHA256='a' * 64, metricKind='http-slow-error-tail'))) b = self.bindings() b.update(mongoPodUID='query', workHealthBindingSHA256='a' * 64) o = m.Observer(SimpleNamespace(kubeconfig='unused', work_health=str(path)), b) o.verify = lambda: None o.readiness = lambda: True o.get = lambda *a: dict(metadata=dict(uid='query')) o.host_filesystem = lambda: dict(availableBytes=200 << 30) warning = '2026/11/17 10:40:18 [review-trace] list summary unavailable: connection read: i/o timeout' for log, expected in ((warning, False), ('unknown backend failure: i/o timeout', True), (warning + '\nsave failed: i/o timeout', True), (warning + '\npanic: unexpected fault', True), (warning + '\nsubmit failed', True), (warning + '\nwrite conflict retry exhausted', True), ('prefix ' + warning, True)): with self.subTest(log=log): def call(*a, **kw): if 'df' in a: return 'Filesystem 1B Used Available Capacity Mounted\nloop 100 10 90 10% /lab/data' if 'pods' in a: return json.dumps(dict(items=[] if '--field-selector=spec.nodeName=10.189.16.4' in a else [dict(metadata=dict(name='api'))])) if 'logs' in a: return log self.fail('unexpected probe') o.call = call self.assertEqual(o.observation(50, now.isoformat())['workErrors'], expected) feed = json.loads(path.read_text()) feed['unexpectedErrors'] = True path.write_text(json.dumps(feed)) log = warning self.assertTrue(o.observation(50, now.isoformat())['workErrors']) def test_mixed_writer_versions_are_bound_individually(self): b = self.bindings() b.update(sourceNamespaceUID='authority-uid', shadowNamespaceUID='shadow-uid', writerCommit='old') b['writers'] = [dict(name='writer-' + str(i), uid=str(i), image='image-' + str(i), replicas=1, commit='new' if i < 2 else 'old') for i in range(5)] deployments = {w['name']: dict(metadata=dict(uid=w['uid']), spec=dict(template=dict(spec=dict(containers=[dict(image=w['image'], env=[ dict(name='DROBOTICFLOW_BUILD_COMMIT', value=w['commit']), dict(name='QUERY_SOURCE_TRACKING', value='1')])]))), status=dict(availableReplicas=1)) for w in b['writers']} o = m.Observer(SimpleNamespace(kubeconfig='unused'), b) o.get = lambda ns, kind, name: dict(metadata=dict(uid=ns + '-uid')) if kind == 'namespace' else deployments[name] o.verify() for i, writer in enumerate(b['writers']): with self.subTest(writer=i): env = deployments[writer['name']]['spec']['template']['spec']['containers'][0]['env'] env[0]['value'] = 'unexpected' with self.assertRaises(ValueError): o.verify() env[0]['value'] = writer['commit'] for writer in b['writers']: writer.pop('commit') deployments[writer['name']]['spec']['template']['spec']['containers'][0]['env'][0]['value'] = 'old' o.verify() # Existing single-version ConfigMaps remain compatible. b['writerCommit'] = '' with self.assertRaises(ValueError): o.verify() def bindings(self): return dict(sourceNamespace='authority', shadowNamespace='shadow', sourceDatabase='source', targetDatabase='query', baselineCPU=85, writers=[{} for _ in range(5)], controllers=[dict(name='query-replica', uid='p'), dict(name='query-read-builder', uid='b')]) def test_cpu_and_identity(self): self.assertAlmostEqual(m.cpu_percent(dict(sourceProcess='s', uptimeMillis=1000, cpuMicroseconds=0), dict(sourceProcess='s', uptimeMillis=2000, cpuMicroseconds=2000000)), 50) with self.assertRaises(ValueError): m.cpu_percent(dict(sourceProcess='s', uptimeMillis=2, cpuMicroseconds=2), dict(sourceProcess='x', uptimeMillis=3, cpuMicroseconds=3)) def test_stop_scope_and_uid_cas(self): b = self.bindings(); m.check_bindings(b) observer = m.Observer(SimpleNamespace(kubeconfig='unused', stop_on_failure=True), b); calls = [] observer.get = lambda ns, kind, name: dict(metadata=dict(uid='p' if name=='query-replica' else 'b', resourceVersion='7'), spec=dict(replicas=1)) observer.call = lambda *args, **kwargs: calls.append(args) or '' observer.stop_shadow() self.assertEqual(len(calls), 2) for args in calls: self.assertEqual(args[:4], ('-n', 'shadow', 'patch', 'deployment')) payload = json.loads(args[-1]); self.assertEqual(payload[0]['op'], 'test'); self.assertEqual(payload[-1]['value'], 0) observer.get = lambda *args: dict(metadata=dict(uid='replaced', resourceVersion='8'), spec=dict(replicas=1)) with self.assertRaises(ValueError): observer.stop_shadow() self.assertEqual(len(calls), 2) b['controllers'][0]['name'] = 'droboticflow-api' with self.assertRaises(ValueError): m.check_bindings(b) def test_distinct_samples_and_thresholds(self): o = m.Observer(SimpleNamespace(kubeconfig='unused'), self.bindings()) good = dict(cpu=90, ready=True, writersCovered=True, workErrors=False, protectedIOFull=0, targetUsedFraction=.1, hostFreeBytes=120<<30) self.assertFalse(o.unsafe(good)); self.assertTrue(o.unsafe(good)) good['cpu']=85; self.assertFalse(o.unsafe(good));good['targetUsedFraction']=.7;self.assertTrue(o.unsafe(good)) def test_business_health_missing_stale_and_real_errors(self): now = dt.datetime.now(dt.timezone.utc) with tempfile.TemporaryDirectory() as folder: path = Path(folder) / 'health.json' with self.assertRaises(FileNotFoundError): m.business_health(path, now) feed = dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=False, latencyRegressed=False) path.write_text(json.dumps(feed)); self.assertFalse(m.business_health(path, now)) feed['latencyRegressed'] = True path.write_text(json.dumps(feed)); self.assertTrue(m.business_health(path, now)) feed['latencyRegressed'] = 'false' path.write_text(json.dumps(feed)) with self.assertRaises(ValueError): m.business_health(path, now) feed['latencyRegressed'] = False; feed['at'] = (now - dt.timedelta(seconds=46)).isoformat() path.write_text(json.dumps(feed)) with self.assertRaises(ValueError): m.business_health(path, now) feed['at'] = now.isoformat(); feed['coverage'] = 'readiness-only' path.write_text(json.dumps(feed)) with self.assertRaises(ValueError): m.business_health(path, now) def test_business_health_exact_binding_required(self): now = dt.datetime.now(dt.timezone.utc) with tempfile.TemporaryDirectory() as folder: path = Path(folder) / 'health.json' feed = dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance', unexpectedErrors=False, latencyRegressed=False, bindingSHA256='a'*64, metricKind='http-slow-error-tail') path.write_text(json.dumps(feed)) self.assertFalse(m.business_health(path, now, 'a'*64)) with self.assertRaises(ValueError): m.business_health(path, now, 'b'*64) feed['metricKind'] = 'manual-constant'; path.write_text(json.dumps(feed)) with self.assertRaises(ValueError): m.business_health(path, now, 'a'*64) def test_maintenance_cpu_is_observed_but_resource_failures_stop(self): b=self.bindings();b['maintenanceInitialization']=True;b['baselineCPU']=0;b['maintenanceUntil']=(dt.datetime.now(dt.timezone.utc)+dt.timedelta(hours=1)).isoformat() m.check_bindings(b);o=m.Observer(SimpleNamespace(kubeconfig='unused'),b) good=dict(cpu=100,ready=True,writersCovered=True,workErrors=False,protectedIOFull=0,targetUsedFraction=.1,hostFreeBytes=120<<30,safetySignalsVersion=1,sourceAvailable=True,memoryPressure=False,persistenceFailure=False) for cpu in [95,100,110]: good['cpu']=cpu;self.assertFalse(o.unsafe(good)) for field,value in [('memoryPressure',True),('persistenceFailure',True),('sourceAvailable',False),('workErrors',True),('safetySignalsVersion',0)]: bad=dict(good);bad[field]=value;self.assertTrue(o.unsafe(bad)) def test_unknown_io_is_not_zero(self): with self.assertRaises(ValueError): m.full_io_pressure('no evidence') self.assertEqual(m.full_io_pressure('some avg10=2.0\nfull avg10=1.25 avg60=0'), 1.25) if __name__ == '__main__': unittest.main()