#!/usr/bin/env python3 """Trusted operator-side observer. No kubeconfig or authority credentials enter builders. Every run writes private health observations in the derived query database. Only --stop-on-failure permits cluster configuration mutations: exact UID-tested scale-to-zero of the TWO shadow controllers in the hash-bound plan. Never changes authority services. Do not run against production before approving the final startup/stop scope. """ import argparse import base64 import contextlib import datetime as dt import hashlib import json import os import math import re from pathlib import Path import socket import subprocess import threading import time import urllib.parse import urllib.request from query_shadow_health import check_clocks, retry_probe, ProbeUnavailable, parse_memory, persistence_failed def check_bindings(b): if b['sourceNamespace'] == b['shadowNamespace'] or len(b['writers']) != 5: raise ValueError('invalid namespace or writer scope') if {x['name'] for x in b['controllers']} != {'query-replica', 'query-read-builder'}: raise ValueError('stop scope must contain only shadow controllers') if b['sourceDatabase'] == b['targetDatabase'] or (not b.get('maintenanceInitialization') and not 0 < b['baselineCPU'] <= 90): raise ValueError('invalid source identity or baseline') def cpu_percent(a, z): if z['sourceProcess'] != a['sourceProcess']: raise ValueError('source identity changed') elapsed = z['uptimeMillis'] - a['uptimeMillis'] used = z['cpuMicroseconds'] - a['cpuMicroseconds'] if elapsed <= 0 or used < 0: raise ValueError('nonmonotonic source counters') return used / 1000 / elapsed / 4 * 100 def full_io_pressure(text): for line in text.splitlines(): if line.startswith('full '): return float(next(x.split('=', 1)[1] for x in line.split() if x.startswith('avg10='))) raise ValueError('protected IO evidence missing') def api_business_log_failure(text): """Keep hard failures; optional GET audit-label enrichment is not a write. Match only the API's exact, standalone log template. Unknown transport failures and the structured save/slice/submit/review feed remain blocking. """ optional_trace = re.compile(r'^(?:\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)? )?' r'\[review-trace\] list summary unavailable: [^\r\n]*$') for line in text.splitlines(): lower = line.lower() if any(token in lower for token in ('panic:', 'fatal error:', 'write conflict retry exhausted', 'submit failed', 'save failed')): return True if 'i/o timeout' in lower and not optional_trace.fullmatch(line): return True return False def business_health(path, now=None, binding_sha256=None): feed = json.loads(Path(path).read_text()) if binding_sha256 is not None and (feed.get('bindingSHA256') != binding_sha256 or feed.get('metricKind') != 'http-slow-error-tail'): raise ValueError('business-health binding changed') at = dt.datetime.fromisoformat(feed['at'].replace('Z', '+00:00')) age = ((now or dt.datetime.now(dt.timezone.utc)) - at).total_seconds() if (not 0 <= age <= 45 or feed.get('coverage') != 'save-slice-submit-review-acceptance' or type(feed.get('unexpectedErrors')) is not bool or type(feed.get('latencyRegressed')) is not bool): raise ValueError('business-health evidence stale or incomplete') return feed['unexpectedErrors'] or feed['latencyRegressed'] class Observer: def __init__(self, args, bindings): self.args, self.b = args, bindings self.k = ['kubectl', '--kubeconfig', args.kubeconfig] self.high = self.high_io = 0 def call(self, *args, data=None, timeout=15): p = subprocess.run(self.k + list(args), input=data, capture_output=True, text=True, timeout=timeout) if p.returncode: raise ProbeUnavailable('bounded cluster operation unavailable') return p.stdout def get(self, ns, kind, name): return json.loads(self.call('-n', ns, 'get', kind, name, '-o', 'json')) def verify(self): for key in ('source', 'shadow'): n = self.get(self.b[key + 'Namespace'], 'namespace', self.b[key + 'Namespace']) if n['metadata']['uid'] != self.b[key + 'NamespaceUID']: raise ValueError('namespace identity changed') for w in self.b['writers']: d = self.get(self.b['sourceNamespace'], 'deployment', w['name']) env = {x['name']: x.get('value') for x in d['spec']['template']['spec']['containers'][0].get('env', [])} # Independently released writers may legitimately have different # commits. Bind each one explicitly; retain legacy uniform bindings. commit = w.get('commit', self.b.get('writerCommit')) if (not isinstance(commit, str) or not commit or d['metadata']['uid'] != w['uid'] or env.get('DROBOTICFLOW_BUILD_COMMIT') != commit or env.get('QUERY_SOURCE_TRACKING') != '1' or d['spec']['template']['spec']['containers'][0]['image'] != w['image'] or d.get('status', {}).get('availableReplicas', 0) != w['replicas']): raise ValueError('writer coverage or health changed') for flag in ('QUERY_REPLICA_REPORTS', 'QUERY_REPLICA_COUNTS', 'QUERY_REPLICA_POOLS', 'QUERY_SERVING_READS', 'QUERY_PUBLISHED_RESULTS'): if env.get(flag, '0') not in ('', '0', 'false'): raise ValueError('production query entry changed') def secret(self, name, key): return base64.b64decode(self.get(self.b['shadowNamespace'], 'secret', name)['data'][key]).decode() def target_environment(self, uri): env = {k: v for k, v in os.environ.items() if not k.startswith(('MONGO_', 'QUERY_'))} env.update(QUERY_READ_MONGO_URI=uri, QUERY_READ_MONGO_DB=self.b['targetDatabase'], QUERY_SERVING_NAMESPACE=self.b['servingNamespace']) return env def run_binary(self, args, env, data=None): if hashlib.sha256(Path(self.args.binary).read_bytes()).hexdigest() != self.args.binary_sha256: raise ValueError('observer binary changed') p = subprocess.run([self.args.binary] + args, input=data, env=env, capture_output=True, text=True, timeout=10) if p.returncode: raise ProbeUnavailable('bounded observation command unavailable') return p.stdout @contextlib.contextmanager def source_relay(self, uri): u = urllib.parse.urlsplit(uri) if u.scheme != 'mongodb' or not u.hostname or ',' in u.netloc: raise ValueError('single reviewed source endpoint required') endpoint = u.hostname + ':' + str(u.port or 27017) if hashlib.sha256(endpoint.encode()).hexdigest() != self.b['sourceEndpointSHA256']: raise ValueError('source endpoint changed') # No secret arguments or stderr: credentials remain in local child env. pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items'] pod = next(x['metadata']['name'] for x in pods if not x['metadata'].get('deletionTimestamp') and x['status'].get('phase') == 'Running' and all(c['ready'] for c in x['status'].get('containerStatuses', []))) listener = socket.socket(); listener.bind(('127.0.0.1', 0)); listener.listen(8); listener.settimeout(.2) stop = threading.Event(); children = [] relay = 'exec 3<>/dev/tcp/"$1"/"$2" || exit 1; cat <&3 & reader=$!; cat >&3; kill "$reader" 2>/dev/null || true; wait "$reader" 2>/dev/null || true' def transfer(conn): p = subprocess.Popen(self.k + ['-n', self.b['sourceNamespace'], 'exec', '-i', pod, '--', 'bash', '-c', relay, 'shadow-readonly-metrics', u.hostname, str(u.port or 27017)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) children.append(p) def upstream(): try: while not stop.is_set(): chunk = conn.recv(65536) if not chunk: break p.stdin.write(chunk); p.stdin.flush() except (OSError, ValueError): pass finally: with contextlib.suppress(OSError): p.stdin.close() threading.Thread(target=upstream, daemon=True).start() try: while not stop.is_set(): chunk = p.stdout.read1(65536) if not chunk: break conn.sendall(chunk) except (OSError, ValueError): pass finally: conn.close(); p.terminate() def accept(): while not stop.is_set(): try: conn, _ = listener.accept() except socket.timeout: continue except OSError: return threading.Thread(target=transfer, args=(conn,), daemon=True).start() thread = threading.Thread(target=accept, daemon=True); thread.start() params = urllib.parse.parse_qs(u.query); params['directConnection'] = ['true'] credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else '' local = urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(listener.getsockname()[1]), u.path, urllib.parse.urlencode(params, doseq=True), '')) # A TLS hostname is not rewritten/bypassed. This initial deployment uses # its reviewed non-TLS private endpoint; other topologies need a new plan. if params.get('tls', params.get('ssl', ['false']))[0].lower() == 'true': listener.close(); stop.set(); raise ValueError('TLS relay needs separately verified server-name handling') try: yield local finally: stop.set(); listener.close(); thread.join(timeout=1) for child in children: if child.poll() is None: child.terminate() for child in children: try: child.wait(timeout=2) except subprocess.TimeoutExpired: child.kill() @contextlib.contextmanager def target_forward(self, uri): with socket.socket() as listener: listener.bind(('127.0.0.1', 0)); port = listener.getsockname()[1] process = subprocess.Popen(self.k + ['-n', self.b['shadowNamespace'], 'port-forward', '--address=127.0.0.1', 'service/query-mongo', str(port) + ':27129'], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) try: for _ in range(40): if process.poll() is not None: raise RuntimeError('query tunnel unavailable') try: with socket.create_connection(('127.0.0.1', port), timeout=.1): break except OSError: time.sleep(.1) else: raise RuntimeError('query tunnel deadline') u = urllib.parse.urlsplit(uri) credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else '' yield urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(port), u.path, u.query, '')) finally: process.terminate() try: process.wait(timeout=2) except subprocess.TimeoutExpired: process.kill() def source_sample(self): with self.source_relay(self.secret('query-shadow-source-connection', 'uri')) as uri: env = self.target_environment(''); env.update(MONGO_URI=uri, MONGO_DB=self.b['sourceDatabase']) return json.loads(self.run_binary(['--sample-source'], env)) def readiness(self): try: with urllib.request.urlopen(self.b['readyURL'], timeout=3) as response: return response.status == 200 and json.load(response).get('ready') is True except urllib.error.HTTPError: return False except (OSError, ValueError) as error: raise ProbeUnavailable('readiness transport unavailable') from None def host_filesystem(self): return json.loads(self.call('get', '--raw', '/api/v1/nodes/10.189.16.4/proxy/stats/summary'))['node']['fs'] def verify_clocks(self, pods): return check_clocks(self.call, self.b['sourceNamespace'], pods) def observation(self, cpu, sampled_at, source=None): self.verify() ready = retry_probe(self.readiness, 'readiness') ns = self.b['shadowNamespace']; pod = self.get(ns, 'pod', 'query-mongo-0') if pod['metadata']['uid'] != self.b['mongoPodUID']: raise ValueError('query Mongo rebuilt; recovery verification required') fs = self.host_filesystem() text = self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'df', '-P', '-B1', '/lab/data') fields = text.splitlines()[-1].split(); used = int(fields[2]) / int(fields[1]) protected = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '--field-selector=spec.nodeName=10.189.16.4', '-o', 'json'))['items'] io = 0. for p in protected: if p['status'].get('phase') != 'Running': continue uid = p['metadata']['uid'].replace('-', '_') # Current production workloads are Burstable; refuse unknown cgroups. path = '/hostcg/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod' + uid + '.slice/io.pressure' io = max(io, full_io_pressure(self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'cat', path))) # A trusted business-health feed is required; readiness/log absence is # not evidence that saves/submits meet their latency/error baseline. work_errors = business_health(self.args.work_health, binding_sha256=self.b.get('workHealthBindingSHA256')) pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items'] for p in pods: if p['metadata'].get('deletionTimestamp'): continue logs = self.call('-n', self.b['sourceNamespace'], 'logs', p['metadata']['name'], '--since=45s', '--tail=2000').lower() work_errors |= api_business_log_failure(logs) result = dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True, workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes']) if self.b.get('maintenanceInitialization'): if not source or source.get('memorySupported') is not True or type(source.get('residentMiB')) not in (int,float) or not math.isfinite(source['residentMiB']) or source['residentMiB'] <= 0 or type(self.b.get('sourceMemoryLimitMiB')) not in (int,float) or not math.isfinite(self.b['sourceMemoryLimitMiB']) or self.b['sourceMemoryLimitMiB'] <= 0: raise ProbeUnavailable('source memory evidence incomplete') group='/hostcg/kubepods.slice/kubepods-pod'+pod['metadata']['uid'].replace('-','_')+'.slice/' used_mem=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.current') limit_mem=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.max') events=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.events') memory=parse_memory(used_mem,limit_mem,events) node=json.loads(self.call('get','node','10.189.16.4','-o','json')) pressure = [c['status'] for c in node['status']['conditions'] if c['type']=='MemoryPressure'] if len(pressure)!=1 or pressure[0] not in ('True','False'): raise ProbeUnavailable('node memory evidence incomplete') node_pressure = pressure[0]=='True' mongo_logs=self.call('-n',ns,'logs','query-mongo-0','-c','mongo','--since=45s','--tail=2000') if len(mongo_logs.splitlines())>=2000: raise ProbeUnavailable('persistence log window truncated') result.update(safetySignalsVersion=1,sourceAvailable=True, memoryPressure=memory['pressure'] or node_pressure or source['residentMiB']>=.98*self.b['sourceMemoryLimitMiB'], persistenceFailure=persistence_failed(mongo_logs)) return result def publish(self, observation): with self.target_forward(self.secret('query-shadow-query-connections', 'builder-uri')) as uri: self.run_binary(['--record-observation', '-'], self.target_environment(uri), json.dumps(observation)) def unsafe(self, o): self.high = self.high + 1 if not self.b.get('maintenanceInitialization') and o['cpu'] > self.b['baselineCPU'] + 3 else 0 self.high_io = self.high_io + 1 if o['protectedIOFull'] > 1 else 0 return (not o['ready'] or not o['writersCovered'] or o['workErrors'] or (not self.b.get('maintenanceInitialization') and (o['cpu'] >= 95 or self.high >= 2)) or (self.b.get('maintenanceInitialization') and (not self.b.get('maintenanceUntil') or dt.datetime.now(dt.timezone.utc)>=dt.datetime.fromisoformat(self.b['maintenanceUntil'].replace('Z','+00:00')))) or o.get('memoryPressure',False) or o.get('persistenceFailure',False) or (self.b.get('maintenanceInitialization') and (o.get('safetySignalsVersion')!=1 or not o.get('sourceAvailable'))) or self.high_io >= 2 or o['protectedIOFull'] > 5 or o['targetUsedFraction'] >= .7 or o['hostFreeBytes'] < (100 << 30)) def stop_shadow(self): for c in self.b['controllers']: d = self.get(self.b['shadowNamespace'], 'deployment', c['name']) if d['metadata']['uid'] != c['uid']: raise ValueError('stop controller replaced') if d['spec']['replicas'] == 0: continue patch = [{'op': 'test', 'path': '/metadata/uid', 'value': c['uid']}, {'op': 'test', 'path': '/metadata/resourceVersion', 'value': d['metadata']['resourceVersion']}, {'op': 'replace', 'path': '/spec/replicas', 'value': 0}] self.call('-n', self.b['shadowNamespace'], 'patch', 'deployment', c['name'], '--type=json', '-p', json.dumps(patch)) def run(self): try: self.verify() pods=json.loads(self.call('-n',self.b['sourceNamespace'],'get','pods','-l','app=droboticflow-api','-o','json'))['items'] self.verify_clocks(pods) prior = retry_probe(self.source_sample,'initial-source') except Exception: if self.args.stop_on_failure: self.stop_shadow() raise RuntimeError('initial source observation unavailable') from None while True: try: for _ in range(3): time.sleep(10) if not retry_probe(self.readiness, 'readiness'): raise ValueError('production readiness unavailable') current = retry_probe(self.source_sample,'source-sample'); cpu = cpu_percent(prior, current); prior = current sampled_at=current['sampledAt'];age=(dt.datetime.now(dt.timezone.utc)-dt.datetime.fromisoformat(sampled_at.replace('Z','+00:00'))).total_seconds() if not 0<=age<=45: raise ValueError('source observation stale') o = retry_probe(lambda: self.observation(cpu,sampled_at,current),'health-observation'); retry_probe(lambda: self.publish(o),'health-publish') unsafe=self.unsafe(o) print(json.dumps({'at':o['at'],'cpu':cpu,'ready':o['ready'],'unsafe':unsafe,'cpuObservedOnly':bool(self.b.get('maintenanceInitialization'))}),flush=True) if unsafe: if self.args.stop_on_failure: self.stop_shadow() return except Exception: # No raw exception may print a URI, secret or customer log line. with contextlib.suppress(Exception): self.publish(dict(at=dt.datetime.now(dt.timezone.utc).isoformat(), cpu=0, ready=False, writersCovered=False, workErrors=True, protectedIOFull=0, targetUsedFraction=0, hostFreeBytes=0)) if self.args.stop_on_failure: self.stop_shadow() raise RuntimeError('observer paused; inspect private metadata and verify before resume') from None def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--kubeconfig', required=True); p.add_argument('--bindings', required=True) p.add_argument('--bindings-sha256', required=True); p.add_argument('--binary', required=True) p.add_argument('--binary-sha256', required=True); p.add_argument('--work-health', required=True); p.add_argument('--stop-on-failure', action='store_true') args = p.parse_args(); raw = Path(args.bindings).read_bytes() if hashlib.sha256(raw).hexdigest() != args.bindings_sha256: raise ValueError('bindings changed') b = json.loads(raw); check_bindings(b) if not re.fullmatch(r'[a-f0-9]{64}', b.get('workHealthBindingSHA256', '')): raise ValueError('reviewed business-health binding required') Observer(args, b).run() if __name__ == '__main__': try: main() except KeyboardInterrupt: raise SystemExit(0) except Exception: raise SystemExit('observer stopped; private checkpoint retained; no automatic resume')