forked from longtao.wu/droboticflow
297 lines
17 KiB
Python
297 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Trusted operator-side observer. No kubeconfig or authority credentials enter builders.
|
|
|
|
Every run writes private health observations in the derived query database.
|
|
Only --stop-on-failure permits cluster configuration mutations: exact UID-tested scale-to-zero of the
|
|
TWO shadow controllers in the hash-bound plan. Never changes authority services.
|
|
Do not run against production before approving the final startup/stop scope.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import contextlib
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import socket
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
|
|
def check_bindings(b):
|
|
if b['sourceNamespace'] == b['shadowNamespace'] or len(b['writers']) != 5:
|
|
raise ValueError('invalid namespace or writer scope')
|
|
if {x['name'] for x in b['controllers']} != {'query-replica', 'query-read-builder'}:
|
|
raise ValueError('stop scope must contain only shadow controllers')
|
|
if b['sourceDatabase'] == b['targetDatabase'] or not 0 < b['baselineCPU'] <= 90:
|
|
raise ValueError('invalid source identity or baseline')
|
|
|
|
|
|
def cpu_percent(a, z):
|
|
if z['sourceProcess'] != a['sourceProcess']:
|
|
raise ValueError('source identity changed')
|
|
elapsed = z['uptimeMillis'] - a['uptimeMillis']
|
|
used = z['cpuMicroseconds'] - a['cpuMicroseconds']
|
|
if elapsed <= 0 or used < 0:
|
|
raise ValueError('nonmonotonic source counters')
|
|
return used / 1000 / elapsed / 4 * 100
|
|
|
|
|
|
def full_io_pressure(text):
|
|
for line in text.splitlines():
|
|
if line.startswith('full '):
|
|
return float(next(x.split('=', 1)[1] for x in line.split() if x.startswith('avg10=')))
|
|
raise ValueError('protected IO evidence missing')
|
|
|
|
|
|
def business_health(path, now=None):
|
|
feed = json.loads(Path(path).read_text())
|
|
at = dt.datetime.fromisoformat(feed['at'].replace('Z', '+00:00'))
|
|
age = ((now or dt.datetime.now(dt.timezone.utc)) - at).total_seconds()
|
|
if (not 0 <= age <= 45 or feed.get('coverage') != 'save-slice-submit-review-acceptance'
|
|
or type(feed.get('unexpectedErrors')) is not bool or type(feed.get('latencyRegressed')) is not bool):
|
|
raise ValueError('business-health evidence stale or incomplete')
|
|
return feed['unexpectedErrors'] or feed['latencyRegressed']
|
|
|
|
|
|
class Observer:
|
|
def __init__(self, args, bindings):
|
|
self.args, self.b = args, bindings
|
|
self.k = ['kubectl', '--kubeconfig', args.kubeconfig]
|
|
self.high = self.high_io = 0
|
|
|
|
def call(self, *args, data=None, timeout=15):
|
|
p = subprocess.run(self.k + list(args), input=data, capture_output=True,
|
|
text=True, timeout=timeout)
|
|
if p.returncode:
|
|
raise RuntimeError('bounded cluster operation unavailable')
|
|
return p.stdout
|
|
|
|
def get(self, ns, kind, name):
|
|
return json.loads(self.call('-n', ns, 'get', kind, name, '-o', 'json'))
|
|
|
|
def verify(self):
|
|
for key in ('source', 'shadow'):
|
|
n = self.get(self.b[key + 'Namespace'], 'namespace', self.b[key + 'Namespace'])
|
|
if n['metadata']['uid'] != self.b[key + 'NamespaceUID']:
|
|
raise ValueError('namespace identity changed')
|
|
for w in self.b['writers']:
|
|
d = self.get(self.b['sourceNamespace'], 'deployment', w['name'])
|
|
env = {x['name']: x.get('value') for x in d['spec']['template']['spec']['containers'][0].get('env', [])}
|
|
if (d['metadata']['uid'] != w['uid'] or env.get('DROBOTICFLOW_BUILD_COMMIT') != self.b['writerCommit']
|
|
or env.get('QUERY_SOURCE_TRACKING') != '1'
|
|
or d['spec']['template']['spec']['containers'][0]['image'] != w['image']
|
|
or d.get('status', {}).get('availableReplicas', 0) != w['replicas']):
|
|
raise ValueError('writer coverage or health changed')
|
|
for flag in ('QUERY_REPLICA_REPORTS', 'QUERY_REPLICA_COUNTS', 'QUERY_REPLICA_POOLS', 'QUERY_SERVING_READS', 'QUERY_PUBLISHED_RESULTS'):
|
|
if env.get(flag, '0') not in ('', '0', 'false'):
|
|
raise ValueError('production query entry changed')
|
|
|
|
def secret(self, name, key):
|
|
return base64.b64decode(self.get(self.b['shadowNamespace'], 'secret', name)['data'][key]).decode()
|
|
|
|
def target_environment(self, uri):
|
|
env = {k: v for k, v in os.environ.items() if not k.startswith(('MONGO_', 'QUERY_'))}
|
|
env.update(QUERY_READ_MONGO_URI=uri, QUERY_READ_MONGO_DB=self.b['targetDatabase'],
|
|
QUERY_SERVING_NAMESPACE=self.b['servingNamespace'])
|
|
return env
|
|
|
|
def run_binary(self, args, env, data=None):
|
|
if hashlib.sha256(Path(self.args.binary).read_bytes()).hexdigest() != self.args.binary_sha256:
|
|
raise ValueError('observer binary changed')
|
|
p = subprocess.run([self.args.binary] + args, input=data, env=env, capture_output=True,
|
|
text=True, timeout=10)
|
|
if p.returncode:
|
|
raise RuntimeError('bounded observation command unavailable')
|
|
return p.stdout
|
|
|
|
@contextlib.contextmanager
|
|
def source_relay(self, uri):
|
|
u = urllib.parse.urlsplit(uri)
|
|
if u.scheme != 'mongodb' or not u.hostname or ',' in u.netloc:
|
|
raise ValueError('single reviewed source endpoint required')
|
|
endpoint = u.hostname + ':' + str(u.port or 27017)
|
|
if hashlib.sha256(endpoint.encode()).hexdigest() != self.b['sourceEndpointSHA256']:
|
|
raise ValueError('source endpoint changed')
|
|
# No secret arguments or stderr: credentials remain in local child env.
|
|
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
|
|
pod = next(x['metadata']['name'] for x in pods if not x['metadata'].get('deletionTimestamp')
|
|
and x['status'].get('phase') == 'Running'
|
|
and all(c['ready'] for c in x['status'].get('containerStatuses', [])))
|
|
listener = socket.socket(); listener.bind(('127.0.0.1', 0)); listener.listen(8); listener.settimeout(.2)
|
|
stop = threading.Event(); children = []
|
|
relay = 'exec 3<>/dev/tcp/"$1"/"$2" || exit 1; cat <&3 & reader=$!; cat >&3; kill "$reader" 2>/dev/null || true; wait "$reader" 2>/dev/null || true'
|
|
|
|
def transfer(conn):
|
|
p = subprocess.Popen(self.k + ['-n', self.b['sourceNamespace'], 'exec', '-i', pod, '--', 'bash', '-c', relay,
|
|
'shadow-readonly-metrics', u.hostname, str(u.port or 27017)],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
|
children.append(p)
|
|
def upstream():
|
|
try:
|
|
while not stop.is_set():
|
|
chunk = conn.recv(65536)
|
|
if not chunk: break
|
|
p.stdin.write(chunk); p.stdin.flush()
|
|
except (OSError, ValueError): pass
|
|
finally:
|
|
with contextlib.suppress(OSError): p.stdin.close()
|
|
threading.Thread(target=upstream, daemon=True).start()
|
|
try:
|
|
while not stop.is_set():
|
|
chunk = p.stdout.read1(65536)
|
|
if not chunk: break
|
|
conn.sendall(chunk)
|
|
except (OSError, ValueError): pass
|
|
finally:
|
|
conn.close(); p.terminate()
|
|
def accept():
|
|
while not stop.is_set():
|
|
try: conn, _ = listener.accept()
|
|
except socket.timeout: continue
|
|
except OSError: return
|
|
threading.Thread(target=transfer, args=(conn,), daemon=True).start()
|
|
thread = threading.Thread(target=accept, daemon=True); thread.start()
|
|
params = urllib.parse.parse_qs(u.query); params['directConnection'] = ['true']
|
|
credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else ''
|
|
local = urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(listener.getsockname()[1]), u.path,
|
|
urllib.parse.urlencode(params, doseq=True), ''))
|
|
# A TLS hostname is not rewritten/bypassed. This initial deployment uses
|
|
# its reviewed non-TLS private endpoint; other topologies need a new plan.
|
|
if params.get('tls', params.get('ssl', ['false']))[0].lower() == 'true':
|
|
listener.close(); stop.set(); raise ValueError('TLS relay needs separately verified server-name handling')
|
|
try: yield local
|
|
finally:
|
|
stop.set(); listener.close(); thread.join(timeout=1)
|
|
for child in children:
|
|
if child.poll() is None: child.terminate()
|
|
for child in children:
|
|
try: child.wait(timeout=2)
|
|
except subprocess.TimeoutExpired: child.kill()
|
|
|
|
@contextlib.contextmanager
|
|
def target_forward(self, uri):
|
|
with socket.socket() as listener:
|
|
listener.bind(('127.0.0.1', 0)); port = listener.getsockname()[1]
|
|
process = subprocess.Popen(self.k + ['-n', self.b['shadowNamespace'], 'port-forward', '--address=127.0.0.1',
|
|
'service/query-mongo', str(port) + ':27129'],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
try:
|
|
for _ in range(40):
|
|
if process.poll() is not None: raise RuntimeError('query tunnel unavailable')
|
|
try:
|
|
with socket.create_connection(('127.0.0.1', port), timeout=.1): break
|
|
except OSError: time.sleep(.1)
|
|
else: raise RuntimeError('query tunnel deadline')
|
|
u = urllib.parse.urlsplit(uri)
|
|
credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else ''
|
|
yield urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(port), u.path, u.query, ''))
|
|
finally:
|
|
process.terminate()
|
|
try: process.wait(timeout=2)
|
|
except subprocess.TimeoutExpired: process.kill()
|
|
|
|
def source_sample(self):
|
|
with self.source_relay(self.secret('query-shadow-source-connection', 'uri')) as uri:
|
|
env = self.target_environment(''); env.update(MONGO_URI=uri, MONGO_DB=self.b['sourceDatabase'])
|
|
return json.loads(self.run_binary(['--sample-source'], env))
|
|
|
|
def observation(self, cpu, sampled_at):
|
|
self.verify()
|
|
with urllib.request.urlopen(self.b['readyURL'], timeout=3) as response:
|
|
ready = response.status == 200 and json.load(response).get('ready') is True
|
|
ns = self.b['shadowNamespace']; pod = self.get(ns, 'pod', 'query-mongo-0')
|
|
if pod['metadata']['uid'] != self.b['mongoPodUID']:
|
|
raise ValueError('query Mongo rebuilt; recovery verification required')
|
|
fs = json.loads(self.call('get', '--raw', '/api/v1/nodes/10.189.16.4/proxy/stats/summary'))['node']['fs']
|
|
text = self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'df', '-P', '-B1', '/lab/data')
|
|
fields = text.splitlines()[-1].split(); used = int(fields[2]) / int(fields[1])
|
|
protected = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '--field-selector=spec.nodeName=10.189.16.4', '-o', 'json'))['items']
|
|
io = 0.
|
|
for p in protected:
|
|
if p['status'].get('phase') != 'Running': continue
|
|
uid = p['metadata']['uid'].replace('-', '_')
|
|
# Current production workloads are Burstable; refuse unknown cgroups.
|
|
path = '/hostcg/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod' + uid + '.slice/io.pressure'
|
|
io = max(io, full_io_pressure(self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'cat', path)))
|
|
# A trusted business-health feed is required; readiness/log absence is
|
|
# not evidence that saves/submits meet their latency/error baseline.
|
|
work_errors = business_health(self.args.work_health)
|
|
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
|
|
for p in pods:
|
|
if p['metadata'].get('deletionTimestamp'): continue
|
|
logs = self.call('-n', self.b['sourceNamespace'], 'logs', p['metadata']['name'], '--since=45s', '--tail=2000').lower()
|
|
work_errors |= any(s in logs for s in ('panic:', 'i/o timeout', 'write conflict retry exhausted', 'submit failed', 'save failed'))
|
|
return dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True,
|
|
workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes'])
|
|
|
|
def publish(self, observation):
|
|
with self.target_forward(self.secret('query-shadow-query-connections', 'builder-uri')) as uri:
|
|
self.run_binary(['--record-observation', '-'], self.target_environment(uri), json.dumps(observation))
|
|
|
|
def unsafe(self, o):
|
|
self.high = self.high + 1 if o['cpu'] > self.b['baselineCPU'] + 3 else 0
|
|
self.high_io = self.high_io + 1 if o['protectedIOFull'] > 1 else 0
|
|
return (not o['ready'] or not o['writersCovered'] or o['workErrors'] or o['cpu'] >= 95 or self.high >= 2
|
|
or self.high_io >= 2 or o['protectedIOFull'] > 5 or o['targetUsedFraction'] >= .7 or o['hostFreeBytes'] < (100 << 30))
|
|
|
|
def stop_shadow(self):
|
|
for c in self.b['controllers']:
|
|
d = self.get(self.b['shadowNamespace'], 'deployment', c['name'])
|
|
if d['metadata']['uid'] != c['uid']: raise ValueError('stop controller replaced')
|
|
if d['spec']['replicas'] == 0: continue
|
|
patch = [{'op': 'test', 'path': '/metadata/uid', 'value': c['uid']},
|
|
{'op': 'test', 'path': '/metadata/resourceVersion', 'value': d['metadata']['resourceVersion']},
|
|
{'op': 'replace', 'path': '/spec/replicas', 'value': 0}]
|
|
self.call('-n', self.b['shadowNamespace'], 'patch', 'deployment', c['name'], '--type=json', '-p', json.dumps(patch))
|
|
|
|
def run(self):
|
|
try: prior = self.source_sample()
|
|
except Exception:
|
|
if self.args.stop_on_failure: self.stop_shadow()
|
|
raise RuntimeError('initial source observation unavailable') from None
|
|
while True:
|
|
try:
|
|
for _ in range(3):
|
|
time.sleep(10)
|
|
with urllib.request.urlopen(self.b['readyURL'],timeout=3) as response:
|
|
if response.status!=200 or json.load(response).get('ready') is not True:
|
|
raise ValueError('production readiness unavailable')
|
|
current = self.source_sample(); cpu = cpu_percent(prior, current); prior = current
|
|
sampled_at=current['sampledAt'];age=(dt.datetime.now(dt.timezone.utc)-dt.datetime.fromisoformat(sampled_at.replace('Z','+00:00'))).total_seconds()
|
|
if not 0<=age<=45: raise ValueError('source observation stale')
|
|
o = self.observation(cpu,sampled_at); self.publish(o)
|
|
unsafe=self.unsafe(o)
|
|
print(json.dumps({'at':o['at'],'cpu':cpu,'ready':o['ready'],'unsafe':unsafe}),flush=True)
|
|
if unsafe:
|
|
if self.args.stop_on_failure: self.stop_shadow()
|
|
return
|
|
except Exception:
|
|
# No raw exception may print a URI, secret or customer log line.
|
|
with contextlib.suppress(Exception):
|
|
self.publish(dict(at=dt.datetime.now(dt.timezone.utc).isoformat(), cpu=0, ready=False, writersCovered=False,
|
|
workErrors=True, protectedIOFull=0, targetUsedFraction=0, hostFreeBytes=0))
|
|
if self.args.stop_on_failure: self.stop_shadow()
|
|
raise RuntimeError('observer paused; inspect private metadata and verify before resume') from None
|
|
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('--kubeconfig', required=True); p.add_argument('--bindings', required=True)
|
|
p.add_argument('--bindings-sha256', required=True); p.add_argument('--binary', required=True)
|
|
p.add_argument('--binary-sha256', required=True); p.add_argument('--work-health', required=True); p.add_argument('--stop-on-failure', action='store_true')
|
|
args = p.parse_args(); raw = Path(args.bindings).read_bytes()
|
|
if hashlib.sha256(raw).hexdigest() != args.bindings_sha256: raise ValueError('bindings changed')
|
|
b = json.loads(raw); check_bindings(b); Observer(args, b).run()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try: main()
|
|
except KeyboardInterrupt: raise SystemExit(0)
|
|
except Exception: raise SystemExit('observer stopped; private checkpoint retained; no automatic resume')
|