droboticflow/scripts/local/query-shadow-observer.py

297 lines
17 KiB
Python

#!/usr/bin/env python3
"""Trusted operator-side observer. No kubeconfig or authority credentials enter builders.
Every run writes private health observations in the derived query database.
Only --stop-on-failure permits cluster configuration mutations: exact UID-tested scale-to-zero of the
TWO shadow controllers in the hash-bound plan. Never changes authority services.
Do not run against production before approving the final startup/stop scope.
"""
import argparse
import base64
import contextlib
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import socket
import subprocess
import threading
import time
import urllib.parse
import urllib.request
def check_bindings(b):
if b['sourceNamespace'] == b['shadowNamespace'] or len(b['writers']) != 5:
raise ValueError('invalid namespace or writer scope')
if {x['name'] for x in b['controllers']} != {'query-replica', 'query-read-builder'}:
raise ValueError('stop scope must contain only shadow controllers')
if b['sourceDatabase'] == b['targetDatabase'] or not 0 < b['baselineCPU'] <= 90:
raise ValueError('invalid source identity or baseline')
def cpu_percent(a, z):
if z['sourceProcess'] != a['sourceProcess']:
raise ValueError('source identity changed')
elapsed = z['uptimeMillis'] - a['uptimeMillis']
used = z['cpuMicroseconds'] - a['cpuMicroseconds']
if elapsed <= 0 or used < 0:
raise ValueError('nonmonotonic source counters')
return used / 1000 / elapsed / 4 * 100
def full_io_pressure(text):
for line in text.splitlines():
if line.startswith('full '):
return float(next(x.split('=', 1)[1] for x in line.split() if x.startswith('avg10=')))
raise ValueError('protected IO evidence missing')
def business_health(path, now=None):
feed = json.loads(Path(path).read_text())
at = dt.datetime.fromisoformat(feed['at'].replace('Z', '+00:00'))
age = ((now or dt.datetime.now(dt.timezone.utc)) - at).total_seconds()
if (not 0 <= age <= 45 or feed.get('coverage') != 'save-slice-submit-review-acceptance'
or type(feed.get('unexpectedErrors')) is not bool or type(feed.get('latencyRegressed')) is not bool):
raise ValueError('business-health evidence stale or incomplete')
return feed['unexpectedErrors'] or feed['latencyRegressed']
class Observer:
def __init__(self, args, bindings):
self.args, self.b = args, bindings
self.k = ['kubectl', '--kubeconfig', args.kubeconfig]
self.high = self.high_io = 0
def call(self, *args, data=None, timeout=15):
p = subprocess.run(self.k + list(args), input=data, capture_output=True,
text=True, timeout=timeout)
if p.returncode:
raise RuntimeError('bounded cluster operation unavailable')
return p.stdout
def get(self, ns, kind, name):
return json.loads(self.call('-n', ns, 'get', kind, name, '-o', 'json'))
def verify(self):
for key in ('source', 'shadow'):
n = self.get(self.b[key + 'Namespace'], 'namespace', self.b[key + 'Namespace'])
if n['metadata']['uid'] != self.b[key + 'NamespaceUID']:
raise ValueError('namespace identity changed')
for w in self.b['writers']:
d = self.get(self.b['sourceNamespace'], 'deployment', w['name'])
env = {x['name']: x.get('value') for x in d['spec']['template']['spec']['containers'][0].get('env', [])}
if (d['metadata']['uid'] != w['uid'] or env.get('DROBOTICFLOW_BUILD_COMMIT') != self.b['writerCommit']
or env.get('QUERY_SOURCE_TRACKING') != '1'
or d['spec']['template']['spec']['containers'][0]['image'] != w['image']
or d.get('status', {}).get('availableReplicas', 0) != w['replicas']):
raise ValueError('writer coverage or health changed')
for flag in ('QUERY_REPLICA_REPORTS', 'QUERY_REPLICA_COUNTS', 'QUERY_REPLICA_POOLS', 'QUERY_SERVING_READS', 'QUERY_PUBLISHED_RESULTS'):
if env.get(flag, '0') not in ('', '0', 'false'):
raise ValueError('production query entry changed')
def secret(self, name, key):
return base64.b64decode(self.get(self.b['shadowNamespace'], 'secret', name)['data'][key]).decode()
def target_environment(self, uri):
env = {k: v for k, v in os.environ.items() if not k.startswith(('MONGO_', 'QUERY_'))}
env.update(QUERY_READ_MONGO_URI=uri, QUERY_READ_MONGO_DB=self.b['targetDatabase'],
QUERY_SERVING_NAMESPACE=self.b['servingNamespace'])
return env
def run_binary(self, args, env, data=None):
if hashlib.sha256(Path(self.args.binary).read_bytes()).hexdigest() != self.args.binary_sha256:
raise ValueError('observer binary changed')
p = subprocess.run([self.args.binary] + args, input=data, env=env, capture_output=True,
text=True, timeout=10)
if p.returncode:
raise RuntimeError('bounded observation command unavailable')
return p.stdout
@contextlib.contextmanager
def source_relay(self, uri):
u = urllib.parse.urlsplit(uri)
if u.scheme != 'mongodb' or not u.hostname or ',' in u.netloc:
raise ValueError('single reviewed source endpoint required')
endpoint = u.hostname + ':' + str(u.port or 27017)
if hashlib.sha256(endpoint.encode()).hexdigest() != self.b['sourceEndpointSHA256']:
raise ValueError('source endpoint changed')
# No secret arguments or stderr: credentials remain in local child env.
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
pod = next(x['metadata']['name'] for x in pods if not x['metadata'].get('deletionTimestamp')
and x['status'].get('phase') == 'Running'
and all(c['ready'] for c in x['status'].get('containerStatuses', [])))
listener = socket.socket(); listener.bind(('127.0.0.1', 0)); listener.listen(8); listener.settimeout(.2)
stop = threading.Event(); children = []
relay = 'exec 3<>/dev/tcp/"$1"/"$2" || exit 1; cat <&3 & reader=$!; cat >&3; kill "$reader" 2>/dev/null || true; wait "$reader" 2>/dev/null || true'
def transfer(conn):
p = subprocess.Popen(self.k + ['-n', self.b['sourceNamespace'], 'exec', '-i', pod, '--', 'bash', '-c', relay,
'shadow-readonly-metrics', u.hostname, str(u.port or 27017)],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
children.append(p)
def upstream():
try:
while not stop.is_set():
chunk = conn.recv(65536)
if not chunk: break
p.stdin.write(chunk); p.stdin.flush()
except (OSError, ValueError): pass
finally:
with contextlib.suppress(OSError): p.stdin.close()
threading.Thread(target=upstream, daemon=True).start()
try:
while not stop.is_set():
chunk = p.stdout.read1(65536)
if not chunk: break
conn.sendall(chunk)
except (OSError, ValueError): pass
finally:
conn.close(); p.terminate()
def accept():
while not stop.is_set():
try: conn, _ = listener.accept()
except socket.timeout: continue
except OSError: return
threading.Thread(target=transfer, args=(conn,), daemon=True).start()
thread = threading.Thread(target=accept, daemon=True); thread.start()
params = urllib.parse.parse_qs(u.query); params['directConnection'] = ['true']
credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else ''
local = urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(listener.getsockname()[1]), u.path,
urllib.parse.urlencode(params, doseq=True), ''))
# A TLS hostname is not rewritten/bypassed. This initial deployment uses
# its reviewed non-TLS private endpoint; other topologies need a new plan.
if params.get('tls', params.get('ssl', ['false']))[0].lower() == 'true':
listener.close(); stop.set(); raise ValueError('TLS relay needs separately verified server-name handling')
try: yield local
finally:
stop.set(); listener.close(); thread.join(timeout=1)
for child in children:
if child.poll() is None: child.terminate()
for child in children:
try: child.wait(timeout=2)
except subprocess.TimeoutExpired: child.kill()
@contextlib.contextmanager
def target_forward(self, uri):
with socket.socket() as listener:
listener.bind(('127.0.0.1', 0)); port = listener.getsockname()[1]
process = subprocess.Popen(self.k + ['-n', self.b['shadowNamespace'], 'port-forward', '--address=127.0.0.1',
'service/query-mongo', str(port) + ':27129'],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
for _ in range(40):
if process.poll() is not None: raise RuntimeError('query tunnel unavailable')
try:
with socket.create_connection(('127.0.0.1', port), timeout=.1): break
except OSError: time.sleep(.1)
else: raise RuntimeError('query tunnel deadline')
u = urllib.parse.urlsplit(uri)
credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else ''
yield urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(port), u.path, u.query, ''))
finally:
process.terminate()
try: process.wait(timeout=2)
except subprocess.TimeoutExpired: process.kill()
def source_sample(self):
with self.source_relay(self.secret('query-shadow-source-connection', 'uri')) as uri:
env = self.target_environment(''); env.update(MONGO_URI=uri, MONGO_DB=self.b['sourceDatabase'])
return json.loads(self.run_binary(['--sample-source'], env))
def observation(self, cpu, sampled_at):
self.verify()
with urllib.request.urlopen(self.b['readyURL'], timeout=3) as response:
ready = response.status == 200 and json.load(response).get('ready') is True
ns = self.b['shadowNamespace']; pod = self.get(ns, 'pod', 'query-mongo-0')
if pod['metadata']['uid'] != self.b['mongoPodUID']:
raise ValueError('query Mongo rebuilt; recovery verification required')
fs = json.loads(self.call('get', '--raw', '/api/v1/nodes/10.189.16.4/proxy/stats/summary'))['node']['fs']
text = self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'df', '-P', '-B1', '/lab/data')
fields = text.splitlines()[-1].split(); used = int(fields[2]) / int(fields[1])
protected = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '--field-selector=spec.nodeName=10.189.16.4', '-o', 'json'))['items']
io = 0.
for p in protected:
if p['status'].get('phase') != 'Running': continue
uid = p['metadata']['uid'].replace('-', '_')
# Current production workloads are Burstable; refuse unknown cgroups.
path = '/hostcg/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod' + uid + '.slice/io.pressure'
io = max(io, full_io_pressure(self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'cat', path)))
# A trusted business-health feed is required; readiness/log absence is
# not evidence that saves/submits meet their latency/error baseline.
work_errors = business_health(self.args.work_health)
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
for p in pods:
if p['metadata'].get('deletionTimestamp'): continue
logs = self.call('-n', self.b['sourceNamespace'], 'logs', p['metadata']['name'], '--since=45s', '--tail=2000').lower()
work_errors |= any(s in logs for s in ('panic:', 'i/o timeout', 'write conflict retry exhausted', 'submit failed', 'save failed'))
return dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True,
workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes'])
def publish(self, observation):
with self.target_forward(self.secret('query-shadow-query-connections', 'builder-uri')) as uri:
self.run_binary(['--record-observation', '-'], self.target_environment(uri), json.dumps(observation))
def unsafe(self, o):
self.high = self.high + 1 if o['cpu'] > self.b['baselineCPU'] + 3 else 0
self.high_io = self.high_io + 1 if o['protectedIOFull'] > 1 else 0
return (not o['ready'] or not o['writersCovered'] or o['workErrors'] or o['cpu'] >= 95 or self.high >= 2
or self.high_io >= 2 or o['protectedIOFull'] > 5 or o['targetUsedFraction'] >= .7 or o['hostFreeBytes'] < (100 << 30))
def stop_shadow(self):
for c in self.b['controllers']:
d = self.get(self.b['shadowNamespace'], 'deployment', c['name'])
if d['metadata']['uid'] != c['uid']: raise ValueError('stop controller replaced')
if d['spec']['replicas'] == 0: continue
patch = [{'op': 'test', 'path': '/metadata/uid', 'value': c['uid']},
{'op': 'test', 'path': '/metadata/resourceVersion', 'value': d['metadata']['resourceVersion']},
{'op': 'replace', 'path': '/spec/replicas', 'value': 0}]
self.call('-n', self.b['shadowNamespace'], 'patch', 'deployment', c['name'], '--type=json', '-p', json.dumps(patch))
def run(self):
try: prior = self.source_sample()
except Exception:
if self.args.stop_on_failure: self.stop_shadow()
raise RuntimeError('initial source observation unavailable') from None
while True:
try:
for _ in range(3):
time.sleep(10)
with urllib.request.urlopen(self.b['readyURL'],timeout=3) as response:
if response.status!=200 or json.load(response).get('ready') is not True:
raise ValueError('production readiness unavailable')
current = self.source_sample(); cpu = cpu_percent(prior, current); prior = current
sampled_at=current['sampledAt'];age=(dt.datetime.now(dt.timezone.utc)-dt.datetime.fromisoformat(sampled_at.replace('Z','+00:00'))).total_seconds()
if not 0<=age<=45: raise ValueError('source observation stale')
o = self.observation(cpu,sampled_at); self.publish(o)
unsafe=self.unsafe(o)
print(json.dumps({'at':o['at'],'cpu':cpu,'ready':o['ready'],'unsafe':unsafe}),flush=True)
if unsafe:
if self.args.stop_on_failure: self.stop_shadow()
return
except Exception:
# No raw exception may print a URI, secret or customer log line.
with contextlib.suppress(Exception):
self.publish(dict(at=dt.datetime.now(dt.timezone.utc).isoformat(), cpu=0, ready=False, writersCovered=False,
workErrors=True, protectedIOFull=0, targetUsedFraction=0, hostFreeBytes=0))
if self.args.stop_on_failure: self.stop_shadow()
raise RuntimeError('observer paused; inspect private metadata and verify before resume') from None
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--kubeconfig', required=True); p.add_argument('--bindings', required=True)
p.add_argument('--bindings-sha256', required=True); p.add_argument('--binary', required=True)
p.add_argument('--binary-sha256', required=True); p.add_argument('--work-health', required=True); p.add_argument('--stop-on-failure', action='store_true')
args = p.parse_args(); raw = Path(args.bindings).read_bytes()
if hashlib.sha256(raw).hexdigest() != args.bindings_sha256: raise ValueError('bindings changed')
b = json.loads(raw); check_bindings(b); Observer(args, b).run()
if __name__ == '__main__':
try: main()
except KeyboardInterrupt: raise SystemExit(0)
except Exception: raise SystemExit('observer stopped; private checkpoint retained; no automatic resume')