droboticflow/scripts/local/query-shadow-observer.py
2026-11-14 20:59:54 -08:00

342 lines
20 KiB
Python

#!/usr/bin/env python3
"""Trusted operator-side observer. No kubeconfig or authority credentials enter builders.
Every run writes private health observations in the derived query database.
Only --stop-on-failure permits cluster configuration mutations: exact UID-tested scale-to-zero of the
TWO shadow controllers in the hash-bound plan. Never changes authority services.
Do not run against production before approving the final startup/stop scope.
"""
import argparse
import base64
import contextlib
import datetime as dt
import hashlib
import json
import os
import math
import re
from pathlib import Path
import socket
import subprocess
import threading
import time
import urllib.parse
import urllib.request
from query_shadow_health import check_clocks, retry_probe, ProbeUnavailable, parse_memory, persistence_failed
def check_bindings(b):
if b['sourceNamespace'] == b['shadowNamespace'] or len(b['writers']) != 5:
raise ValueError('invalid namespace or writer scope')
if {x['name'] for x in b['controllers']} != {'query-replica', 'query-read-builder'}:
raise ValueError('stop scope must contain only shadow controllers')
if b['sourceDatabase'] == b['targetDatabase'] or (not b.get('maintenanceInitialization') and not 0 < b['baselineCPU'] <= 90):
raise ValueError('invalid source identity or baseline')
def cpu_percent(a, z):
if z['sourceProcess'] != a['sourceProcess']:
raise ValueError('source identity changed')
elapsed = z['uptimeMillis'] - a['uptimeMillis']
used = z['cpuMicroseconds'] - a['cpuMicroseconds']
if elapsed <= 0 or used < 0:
raise ValueError('nonmonotonic source counters')
return used / 1000 / elapsed / 4 * 100
def full_io_pressure(text):
for line in text.splitlines():
if line.startswith('full '):
return float(next(x.split('=', 1)[1] for x in line.split() if x.startswith('avg10=')))
raise ValueError('protected IO evidence missing')
def business_health(path, now=None, binding_sha256=None):
feed = json.loads(Path(path).read_text())
if binding_sha256 is not None and (feed.get('bindingSHA256') != binding_sha256 or feed.get('metricKind') != 'http-slow-error-tail'):
raise ValueError('business-health binding changed')
at = dt.datetime.fromisoformat(feed['at'].replace('Z', '+00:00'))
age = ((now or dt.datetime.now(dt.timezone.utc)) - at).total_seconds()
if (not 0 <= age <= 45 or feed.get('coverage') != 'save-slice-submit-review-acceptance'
or type(feed.get('unexpectedErrors')) is not bool or type(feed.get('latencyRegressed')) is not bool):
raise ValueError('business-health evidence stale or incomplete')
return feed['unexpectedErrors'] or feed['latencyRegressed']
class Observer:
def __init__(self, args, bindings):
self.args, self.b = args, bindings
self.k = ['kubectl', '--kubeconfig', args.kubeconfig]
self.high = self.high_io = 0
def call(self, *args, data=None, timeout=15):
p = subprocess.run(self.k + list(args), input=data, capture_output=True,
text=True, timeout=timeout)
if p.returncode:
raise ProbeUnavailable('bounded cluster operation unavailable')
return p.stdout
def get(self, ns, kind, name):
return json.loads(self.call('-n', ns, 'get', kind, name, '-o', 'json'))
def verify(self):
for key in ('source', 'shadow'):
n = self.get(self.b[key + 'Namespace'], 'namespace', self.b[key + 'Namespace'])
if n['metadata']['uid'] != self.b[key + 'NamespaceUID']:
raise ValueError('namespace identity changed')
for w in self.b['writers']:
d = self.get(self.b['sourceNamespace'], 'deployment', w['name'])
env = {x['name']: x.get('value') for x in d['spec']['template']['spec']['containers'][0].get('env', [])}
if (d['metadata']['uid'] != w['uid'] or env.get('DROBOTICFLOW_BUILD_COMMIT') != self.b['writerCommit']
or env.get('QUERY_SOURCE_TRACKING') != '1'
or d['spec']['template']['spec']['containers'][0]['image'] != w['image']
or d.get('status', {}).get('availableReplicas', 0) != w['replicas']):
raise ValueError('writer coverage or health changed')
for flag in ('QUERY_REPLICA_REPORTS', 'QUERY_REPLICA_COUNTS', 'QUERY_REPLICA_POOLS', 'QUERY_SERVING_READS', 'QUERY_PUBLISHED_RESULTS'):
if env.get(flag, '0') not in ('', '0', 'false'):
raise ValueError('production query entry changed')
def secret(self, name, key):
return base64.b64decode(self.get(self.b['shadowNamespace'], 'secret', name)['data'][key]).decode()
def target_environment(self, uri):
env = {k: v for k, v in os.environ.items() if not k.startswith(('MONGO_', 'QUERY_'))}
env.update(QUERY_READ_MONGO_URI=uri, QUERY_READ_MONGO_DB=self.b['targetDatabase'],
QUERY_SERVING_NAMESPACE=self.b['servingNamespace'])
return env
def run_binary(self, args, env, data=None):
if hashlib.sha256(Path(self.args.binary).read_bytes()).hexdigest() != self.args.binary_sha256:
raise ValueError('observer binary changed')
p = subprocess.run([self.args.binary] + args, input=data, env=env, capture_output=True,
text=True, timeout=10)
if p.returncode:
raise ProbeUnavailable('bounded observation command unavailable')
return p.stdout
@contextlib.contextmanager
def source_relay(self, uri):
u = urllib.parse.urlsplit(uri)
if u.scheme != 'mongodb' or not u.hostname or ',' in u.netloc:
raise ValueError('single reviewed source endpoint required')
endpoint = u.hostname + ':' + str(u.port or 27017)
if hashlib.sha256(endpoint.encode()).hexdigest() != self.b['sourceEndpointSHA256']:
raise ValueError('source endpoint changed')
# No secret arguments or stderr: credentials remain in local child env.
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
pod = next(x['metadata']['name'] for x in pods if not x['metadata'].get('deletionTimestamp')
and x['status'].get('phase') == 'Running'
and all(c['ready'] for c in x['status'].get('containerStatuses', [])))
listener = socket.socket(); listener.bind(('127.0.0.1', 0)); listener.listen(8); listener.settimeout(.2)
stop = threading.Event(); children = []
relay = 'exec 3<>/dev/tcp/"$1"/"$2" || exit 1; cat <&3 & reader=$!; cat >&3; kill "$reader" 2>/dev/null || true; wait "$reader" 2>/dev/null || true'
def transfer(conn):
p = subprocess.Popen(self.k + ['-n', self.b['sourceNamespace'], 'exec', '-i', pod, '--', 'bash', '-c', relay,
'shadow-readonly-metrics', u.hostname, str(u.port or 27017)],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
children.append(p)
def upstream():
try:
while not stop.is_set():
chunk = conn.recv(65536)
if not chunk: break
p.stdin.write(chunk); p.stdin.flush()
except (OSError, ValueError): pass
finally:
with contextlib.suppress(OSError): p.stdin.close()
threading.Thread(target=upstream, daemon=True).start()
try:
while not stop.is_set():
chunk = p.stdout.read1(65536)
if not chunk: break
conn.sendall(chunk)
except (OSError, ValueError): pass
finally:
conn.close(); p.terminate()
def accept():
while not stop.is_set():
try: conn, _ = listener.accept()
except socket.timeout: continue
except OSError: return
threading.Thread(target=transfer, args=(conn,), daemon=True).start()
thread = threading.Thread(target=accept, daemon=True); thread.start()
params = urllib.parse.parse_qs(u.query); params['directConnection'] = ['true']
credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else ''
local = urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(listener.getsockname()[1]), u.path,
urllib.parse.urlencode(params, doseq=True), ''))
# A TLS hostname is not rewritten/bypassed. This initial deployment uses
# its reviewed non-TLS private endpoint; other topologies need a new plan.
if params.get('tls', params.get('ssl', ['false']))[0].lower() == 'true':
listener.close(); stop.set(); raise ValueError('TLS relay needs separately verified server-name handling')
try: yield local
finally:
stop.set(); listener.close(); thread.join(timeout=1)
for child in children:
if child.poll() is None: child.terminate()
for child in children:
try: child.wait(timeout=2)
except subprocess.TimeoutExpired: child.kill()
@contextlib.contextmanager
def target_forward(self, uri):
with socket.socket() as listener:
listener.bind(('127.0.0.1', 0)); port = listener.getsockname()[1]
process = subprocess.Popen(self.k + ['-n', self.b['shadowNamespace'], 'port-forward', '--address=127.0.0.1',
'service/query-mongo', str(port) + ':27129'],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
for _ in range(40):
if process.poll() is not None: raise RuntimeError('query tunnel unavailable')
try:
with socket.create_connection(('127.0.0.1', port), timeout=.1): break
except OSError: time.sleep(.1)
else: raise RuntimeError('query tunnel deadline')
u = urllib.parse.urlsplit(uri)
credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else ''
yield urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(port), u.path, u.query, ''))
finally:
process.terminate()
try: process.wait(timeout=2)
except subprocess.TimeoutExpired: process.kill()
def source_sample(self):
with self.source_relay(self.secret('query-shadow-source-connection', 'uri')) as uri:
env = self.target_environment(''); env.update(MONGO_URI=uri, MONGO_DB=self.b['sourceDatabase'])
return json.loads(self.run_binary(['--sample-source'], env))
def readiness(self):
try:
with urllib.request.urlopen(self.b['readyURL'], timeout=3) as response:
return response.status == 200 and json.load(response).get('ready') is True
except urllib.error.HTTPError:
return False
except (OSError, ValueError) as error:
raise ProbeUnavailable('readiness transport unavailable') from None
def host_filesystem(self):
return json.loads(self.call('get', '--raw', '/api/v1/nodes/10.189.16.4/proxy/stats/summary'))['node']['fs']
def verify_clocks(self, pods):
return check_clocks(self.call, self.b['sourceNamespace'], pods)
def observation(self, cpu, sampled_at, source=None):
self.verify()
ready = retry_probe(self.readiness, 'readiness')
ns = self.b['shadowNamespace']; pod = self.get(ns, 'pod', 'query-mongo-0')
if pod['metadata']['uid'] != self.b['mongoPodUID']:
raise ValueError('query Mongo rebuilt; recovery verification required')
fs = self.host_filesystem()
text = self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'df', '-P', '-B1', '/lab/data')
fields = text.splitlines()[-1].split(); used = int(fields[2]) / int(fields[1])
protected = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '--field-selector=spec.nodeName=10.189.16.4', '-o', 'json'))['items']
io = 0.
for p in protected:
if p['status'].get('phase') != 'Running': continue
uid = p['metadata']['uid'].replace('-', '_')
# Current production workloads are Burstable; refuse unknown cgroups.
path = '/hostcg/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod' + uid + '.slice/io.pressure'
io = max(io, full_io_pressure(self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'cat', path)))
# A trusted business-health feed is required; readiness/log absence is
# not evidence that saves/submits meet their latency/error baseline.
work_errors = business_health(self.args.work_health, binding_sha256=self.b.get('workHealthBindingSHA256'))
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
for p in pods:
if p['metadata'].get('deletionTimestamp'): continue
logs = self.call('-n', self.b['sourceNamespace'], 'logs', p['metadata']['name'], '--since=45s', '--tail=2000').lower()
work_errors |= any(s in logs for s in ('panic:', 'i/o timeout', 'write conflict retry exhausted', 'submit failed', 'save failed'))
result = dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True,
workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes'])
if self.b.get('maintenanceInitialization'):
if not source or source.get('memorySupported') is not True or type(source.get('residentMiB')) not in (int,float) or not math.isfinite(source['residentMiB']) or source['residentMiB'] <= 0 or type(self.b.get('sourceMemoryLimitMiB')) not in (int,float) or not math.isfinite(self.b['sourceMemoryLimitMiB']) or self.b['sourceMemoryLimitMiB'] <= 0:
raise ProbeUnavailable('source memory evidence incomplete')
group='/hostcg/kubepods.slice/kubepods-pod'+pod['metadata']['uid'].replace('-','_')+'.slice/'
used_mem=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.current')
limit_mem=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.max')
events=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.events')
memory=parse_memory(used_mem,limit_mem,events)
node=json.loads(self.call('get','node','10.189.16.4','-o','json'))
pressure = [c['status'] for c in node['status']['conditions'] if c['type']=='MemoryPressure']
if len(pressure)!=1 or pressure[0] not in ('True','False'): raise ProbeUnavailable('node memory evidence incomplete')
node_pressure = pressure[0]=='True'
mongo_logs=self.call('-n',ns,'logs','query-mongo-0','-c','mongo','--since=45s','--tail=2000')
if len(mongo_logs.splitlines())>=2000: raise ProbeUnavailable('persistence log window truncated')
result.update(safetySignalsVersion=1,sourceAvailable=True,
memoryPressure=memory['pressure'] or node_pressure or source['residentMiB']>=.98*self.b['sourceMemoryLimitMiB'],
persistenceFailure=persistence_failed(mongo_logs))
return result
def publish(self, observation):
with self.target_forward(self.secret('query-shadow-query-connections', 'builder-uri')) as uri:
self.run_binary(['--record-observation', '-'], self.target_environment(uri), json.dumps(observation))
def unsafe(self, o):
self.high = self.high + 1 if not self.b.get('maintenanceInitialization') and o['cpu'] > self.b['baselineCPU'] + 3 else 0
self.high_io = self.high_io + 1 if o['protectedIOFull'] > 1 else 0
return (not o['ready'] or not o['writersCovered'] or o['workErrors'] or (not self.b.get('maintenanceInitialization') and (o['cpu'] >= 95 or self.high >= 2))
or (self.b.get('maintenanceInitialization') and (not self.b.get('maintenanceUntil') or dt.datetime.now(dt.timezone.utc)>=dt.datetime.fromisoformat(self.b['maintenanceUntil'].replace('Z','+00:00'))))
or o.get('memoryPressure',False) or o.get('persistenceFailure',False) or (self.b.get('maintenanceInitialization') and (o.get('safetySignalsVersion')!=1 or not o.get('sourceAvailable')))
or self.high_io >= 2 or o['protectedIOFull'] > 5 or o['targetUsedFraction'] >= .7 or o['hostFreeBytes'] < (100 << 30))
def stop_shadow(self):
for c in self.b['controllers']:
d = self.get(self.b['shadowNamespace'], 'deployment', c['name'])
if d['metadata']['uid'] != c['uid']: raise ValueError('stop controller replaced')
if d['spec']['replicas'] == 0: continue
patch = [{'op': 'test', 'path': '/metadata/uid', 'value': c['uid']},
{'op': 'test', 'path': '/metadata/resourceVersion', 'value': d['metadata']['resourceVersion']},
{'op': 'replace', 'path': '/spec/replicas', 'value': 0}]
self.call('-n', self.b['shadowNamespace'], 'patch', 'deployment', c['name'], '--type=json', '-p', json.dumps(patch))
def run(self):
try:
self.verify()
pods=json.loads(self.call('-n',self.b['sourceNamespace'],'get','pods','-l','app=droboticflow-api','-o','json'))['items']
self.verify_clocks(pods)
prior = retry_probe(self.source_sample,'initial-source')
except Exception:
if self.args.stop_on_failure: self.stop_shadow()
raise RuntimeError('initial source observation unavailable') from None
while True:
try:
for _ in range(3):
time.sleep(10)
if not retry_probe(self.readiness, 'readiness'):
raise ValueError('production readiness unavailable')
current = retry_probe(self.source_sample,'source-sample'); cpu = cpu_percent(prior, current); prior = current
sampled_at=current['sampledAt'];age=(dt.datetime.now(dt.timezone.utc)-dt.datetime.fromisoformat(sampled_at.replace('Z','+00:00'))).total_seconds()
if not 0<=age<=45: raise ValueError('source observation stale')
o = retry_probe(lambda: self.observation(cpu,sampled_at,current),'health-observation'); retry_probe(lambda: self.publish(o),'health-publish')
unsafe=self.unsafe(o)
print(json.dumps({'at':o['at'],'cpu':cpu,'ready':o['ready'],'unsafe':unsafe,'cpuObservedOnly':bool(self.b.get('maintenanceInitialization'))}),flush=True)
if unsafe:
if self.args.stop_on_failure: self.stop_shadow()
return
except Exception:
# No raw exception may print a URI, secret or customer log line.
with contextlib.suppress(Exception):
self.publish(dict(at=dt.datetime.now(dt.timezone.utc).isoformat(), cpu=0, ready=False, writersCovered=False,
workErrors=True, protectedIOFull=0, targetUsedFraction=0, hostFreeBytes=0))
if self.args.stop_on_failure: self.stop_shadow()
raise RuntimeError('observer paused; inspect private metadata and verify before resume') from None
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--kubeconfig', required=True); p.add_argument('--bindings', required=True)
p.add_argument('--bindings-sha256', required=True); p.add_argument('--binary', required=True)
p.add_argument('--binary-sha256', required=True); p.add_argument('--work-health', required=True); p.add_argument('--stop-on-failure', action='store_true')
args = p.parse_args(); raw = Path(args.bindings).read_bytes()
if hashlib.sha256(raw).hexdigest() != args.bindings_sha256: raise ValueError('bindings changed')
b = json.loads(raw); check_bindings(b)
if not re.fullmatch(r'[a-f0-9]{64}', b.get('workHealthBindingSHA256', '')):
raise ValueError('reviewed business-health binding required')
Observer(args, b).run()
if __name__ == '__main__':
try: main()
except KeyboardInterrupt: raise SystemExit(0)
except Exception: raise SystemExit('observer stopped; private checkpoint retained; no automatic resume')