364 lines
21 KiB
Python
364 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
"""Trusted operator-side observer. No kubeconfig or authority credentials enter builders.
|
|
|
|
Every run writes private health observations in the derived query database.
|
|
Only --stop-on-failure permits cluster configuration mutations: exact UID-tested scale-to-zero of the
|
|
TWO shadow controllers in the hash-bound plan. Never changes authority services.
|
|
Do not run against production before approving the final startup/stop scope.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import contextlib
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import math
|
|
import re
|
|
from pathlib import Path
|
|
import socket
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
from query_shadow_health import check_clocks, retry_probe, ProbeUnavailable, parse_memory, persistence_failed
|
|
|
|
|
|
def check_bindings(b):
|
|
if b['sourceNamespace'] == b['shadowNamespace'] or len(b['writers']) != 5:
|
|
raise ValueError('invalid namespace or writer scope')
|
|
if {x['name'] for x in b['controllers']} != {'query-replica', 'query-read-builder'}:
|
|
raise ValueError('stop scope must contain only shadow controllers')
|
|
if b['sourceDatabase'] == b['targetDatabase'] or (not b.get('maintenanceInitialization') and not 0 < b['baselineCPU'] <= 90):
|
|
raise ValueError('invalid source identity or baseline')
|
|
|
|
|
|
def cpu_percent(a, z):
|
|
if z['sourceProcess'] != a['sourceProcess']:
|
|
raise ValueError('source identity changed')
|
|
elapsed = z['uptimeMillis'] - a['uptimeMillis']
|
|
used = z['cpuMicroseconds'] - a['cpuMicroseconds']
|
|
if elapsed <= 0 or used < 0:
|
|
raise ValueError('nonmonotonic source counters')
|
|
return used / 1000 / elapsed / 4 * 100
|
|
|
|
|
|
def full_io_pressure(text):
|
|
for line in text.splitlines():
|
|
if line.startswith('full '):
|
|
return float(next(x.split('=', 1)[1] for x in line.split() if x.startswith('avg10=')))
|
|
raise ValueError('protected IO evidence missing')
|
|
|
|
|
|
def api_business_log_failure(text):
|
|
"""Keep hard failures; optional GET audit-label enrichment is not a write.
|
|
|
|
Match only the API's exact, standalone log template. Unknown transport
|
|
failures and the structured save/slice/submit/review feed remain blocking.
|
|
"""
|
|
optional_trace = re.compile(r'^(?:\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)? )?'
|
|
r'\[review-trace\] list summary unavailable: [^\r\n]*$')
|
|
for line in text.splitlines():
|
|
lower = line.lower()
|
|
if any(token in lower for token in ('panic:', 'fatal error:', 'write conflict retry exhausted',
|
|
'submit failed', 'save failed')):
|
|
return True
|
|
if 'i/o timeout' in lower and not optional_trace.fullmatch(line):
|
|
return True
|
|
return False
|
|
|
|
|
|
def business_health(path, now=None, binding_sha256=None):
|
|
feed = json.loads(Path(path).read_text())
|
|
if binding_sha256 is not None and (feed.get('bindingSHA256') != binding_sha256 or feed.get('metricKind') != 'http-slow-error-tail'):
|
|
raise ValueError('business-health binding changed')
|
|
at = dt.datetime.fromisoformat(feed['at'].replace('Z', '+00:00'))
|
|
age = ((now or dt.datetime.now(dt.timezone.utc)) - at).total_seconds()
|
|
if (not 0 <= age <= 45 or feed.get('coverage') != 'save-slice-submit-review-acceptance'
|
|
or type(feed.get('unexpectedErrors')) is not bool or type(feed.get('latencyRegressed')) is not bool):
|
|
raise ValueError('business-health evidence stale or incomplete')
|
|
return feed['unexpectedErrors'] or feed['latencyRegressed']
|
|
|
|
|
|
class Observer:
|
|
def __init__(self, args, bindings):
|
|
self.args, self.b = args, bindings
|
|
self.k = ['kubectl', '--kubeconfig', args.kubeconfig]
|
|
self.high = self.high_io = 0
|
|
|
|
def call(self, *args, data=None, timeout=15):
|
|
p = subprocess.run(self.k + list(args), input=data, capture_output=True,
|
|
text=True, timeout=timeout)
|
|
if p.returncode:
|
|
raise ProbeUnavailable('bounded cluster operation unavailable')
|
|
return p.stdout
|
|
|
|
def get(self, ns, kind, name):
|
|
return json.loads(self.call('-n', ns, 'get', kind, name, '-o', 'json'))
|
|
|
|
def verify(self):
|
|
for key in ('source', 'shadow'):
|
|
n = self.get(self.b[key + 'Namespace'], 'namespace', self.b[key + 'Namespace'])
|
|
if n['metadata']['uid'] != self.b[key + 'NamespaceUID']:
|
|
raise ValueError('namespace identity changed')
|
|
for w in self.b['writers']:
|
|
d = self.get(self.b['sourceNamespace'], 'deployment', w['name'])
|
|
env = {x['name']: x.get('value') for x in d['spec']['template']['spec']['containers'][0].get('env', [])}
|
|
# Independently released writers may legitimately have different
|
|
# commits. Bind each one explicitly; retain legacy uniform bindings.
|
|
commit = w.get('commit', self.b.get('writerCommit'))
|
|
if (not isinstance(commit, str) or not commit
|
|
or d['metadata']['uid'] != w['uid'] or env.get('DROBOTICFLOW_BUILD_COMMIT') != commit
|
|
or env.get('QUERY_SOURCE_TRACKING') != '1'
|
|
or d['spec']['template']['spec']['containers'][0]['image'] != w['image']
|
|
or d.get('status', {}).get('availableReplicas', 0) != w['replicas']):
|
|
raise ValueError('writer coverage or health changed')
|
|
for flag in ('QUERY_REPLICA_REPORTS', 'QUERY_REPLICA_COUNTS', 'QUERY_REPLICA_POOLS', 'QUERY_SERVING_READS', 'QUERY_PUBLISHED_RESULTS'):
|
|
if env.get(flag, '0') not in ('', '0', 'false'):
|
|
raise ValueError('production query entry changed')
|
|
|
|
def secret(self, name, key):
|
|
return base64.b64decode(self.get(self.b['shadowNamespace'], 'secret', name)['data'][key]).decode()
|
|
|
|
def target_environment(self, uri):
|
|
env = {k: v for k, v in os.environ.items() if not k.startswith(('MONGO_', 'QUERY_'))}
|
|
env.update(QUERY_READ_MONGO_URI=uri, QUERY_READ_MONGO_DB=self.b['targetDatabase'],
|
|
QUERY_SERVING_NAMESPACE=self.b['servingNamespace'])
|
|
return env
|
|
|
|
def run_binary(self, args, env, data=None):
|
|
if hashlib.sha256(Path(self.args.binary).read_bytes()).hexdigest() != self.args.binary_sha256:
|
|
raise ValueError('observer binary changed')
|
|
p = subprocess.run([self.args.binary] + args, input=data, env=env, capture_output=True,
|
|
text=True, timeout=10)
|
|
if p.returncode:
|
|
raise ProbeUnavailable('bounded observation command unavailable')
|
|
return p.stdout
|
|
|
|
@contextlib.contextmanager
|
|
def source_relay(self, uri):
|
|
u = urllib.parse.urlsplit(uri)
|
|
if u.scheme != 'mongodb' or not u.hostname or ',' in u.netloc:
|
|
raise ValueError('single reviewed source endpoint required')
|
|
endpoint = u.hostname + ':' + str(u.port or 27017)
|
|
if hashlib.sha256(endpoint.encode()).hexdigest() != self.b['sourceEndpointSHA256']:
|
|
raise ValueError('source endpoint changed')
|
|
# No secret arguments or stderr: credentials remain in local child env.
|
|
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
|
|
pod = next(x['metadata']['name'] for x in pods if not x['metadata'].get('deletionTimestamp')
|
|
and x['status'].get('phase') == 'Running'
|
|
and all(c['ready'] for c in x['status'].get('containerStatuses', [])))
|
|
listener = socket.socket(); listener.bind(('127.0.0.1', 0)); listener.listen(8); listener.settimeout(.2)
|
|
stop = threading.Event(); children = []
|
|
relay = 'exec 3<>/dev/tcp/"$1"/"$2" || exit 1; cat <&3 & reader=$!; cat >&3; kill "$reader" 2>/dev/null || true; wait "$reader" 2>/dev/null || true'
|
|
|
|
def transfer(conn):
|
|
p = subprocess.Popen(self.k + ['-n', self.b['sourceNamespace'], 'exec', '-i', pod, '--', 'bash', '-c', relay,
|
|
'shadow-readonly-metrics', u.hostname, str(u.port or 27017)],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
|
children.append(p)
|
|
def upstream():
|
|
try:
|
|
while not stop.is_set():
|
|
chunk = conn.recv(65536)
|
|
if not chunk: break
|
|
p.stdin.write(chunk); p.stdin.flush()
|
|
except (OSError, ValueError): pass
|
|
finally:
|
|
with contextlib.suppress(OSError): p.stdin.close()
|
|
threading.Thread(target=upstream, daemon=True).start()
|
|
try:
|
|
while not stop.is_set():
|
|
chunk = p.stdout.read1(65536)
|
|
if not chunk: break
|
|
conn.sendall(chunk)
|
|
except (OSError, ValueError): pass
|
|
finally:
|
|
conn.close(); p.terminate()
|
|
def accept():
|
|
while not stop.is_set():
|
|
try: conn, _ = listener.accept()
|
|
except socket.timeout: continue
|
|
except OSError: return
|
|
threading.Thread(target=transfer, args=(conn,), daemon=True).start()
|
|
thread = threading.Thread(target=accept, daemon=True); thread.start()
|
|
params = urllib.parse.parse_qs(u.query); params['directConnection'] = ['true']
|
|
credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else ''
|
|
local = urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(listener.getsockname()[1]), u.path,
|
|
urllib.parse.urlencode(params, doseq=True), ''))
|
|
# A TLS hostname is not rewritten/bypassed. This initial deployment uses
|
|
# its reviewed non-TLS private endpoint; other topologies need a new plan.
|
|
if params.get('tls', params.get('ssl', ['false']))[0].lower() == 'true':
|
|
listener.close(); stop.set(); raise ValueError('TLS relay needs separately verified server-name handling')
|
|
try: yield local
|
|
finally:
|
|
stop.set(); listener.close(); thread.join(timeout=1)
|
|
for child in children:
|
|
if child.poll() is None: child.terminate()
|
|
for child in children:
|
|
try: child.wait(timeout=2)
|
|
except subprocess.TimeoutExpired: child.kill()
|
|
|
|
@contextlib.contextmanager
|
|
def target_forward(self, uri):
|
|
with socket.socket() as listener:
|
|
listener.bind(('127.0.0.1', 0)); port = listener.getsockname()[1]
|
|
process = subprocess.Popen(self.k + ['-n', self.b['shadowNamespace'], 'port-forward', '--address=127.0.0.1',
|
|
'service/query-mongo', str(port) + ':27129'],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
try:
|
|
for _ in range(40):
|
|
if process.poll() is not None: raise RuntimeError('query tunnel unavailable')
|
|
try:
|
|
with socket.create_connection(('127.0.0.1', port), timeout=.1): break
|
|
except OSError: time.sleep(.1)
|
|
else: raise RuntimeError('query tunnel deadline')
|
|
u = urllib.parse.urlsplit(uri)
|
|
credential = u.netloc.rsplit('@', 1)[0] + '@' if '@' in u.netloc else ''
|
|
yield urllib.parse.urlunsplit(('mongodb', credential + '127.0.0.1:' + str(port), u.path, u.query, ''))
|
|
finally:
|
|
process.terminate()
|
|
try: process.wait(timeout=2)
|
|
except subprocess.TimeoutExpired: process.kill()
|
|
|
|
def source_sample(self):
|
|
with self.source_relay(self.secret('query-shadow-source-connection', 'uri')) as uri:
|
|
env = self.target_environment(''); env.update(MONGO_URI=uri, MONGO_DB=self.b['sourceDatabase'])
|
|
return json.loads(self.run_binary(['--sample-source'], env))
|
|
|
|
def readiness(self):
|
|
try:
|
|
with urllib.request.urlopen(self.b['readyURL'], timeout=3) as response:
|
|
return response.status == 200 and json.load(response).get('ready') is True
|
|
except urllib.error.HTTPError:
|
|
return False
|
|
except (OSError, ValueError) as error:
|
|
raise ProbeUnavailable('readiness transport unavailable') from None
|
|
|
|
def host_filesystem(self):
|
|
return json.loads(self.call('get', '--raw', '/api/v1/nodes/10.189.16.4/proxy/stats/summary'))['node']['fs']
|
|
|
|
def verify_clocks(self, pods):
|
|
return check_clocks(self.call, self.b['sourceNamespace'], pods)
|
|
|
|
def observation(self, cpu, sampled_at, source=None):
|
|
self.verify()
|
|
ready = retry_probe(self.readiness, 'readiness')
|
|
ns = self.b['shadowNamespace']; pod = self.get(ns, 'pod', 'query-mongo-0')
|
|
if pod['metadata']['uid'] != self.b['mongoPodUID']:
|
|
raise ValueError('query Mongo rebuilt; recovery verification required')
|
|
fs = self.host_filesystem()
|
|
text = self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'df', '-P', '-B1', '/lab/data')
|
|
fields = text.splitlines()[-1].split(); used = int(fields[2]) / int(fields[1])
|
|
protected = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '--field-selector=spec.nodeName=10.189.16.4', '-o', 'json'))['items']
|
|
io = 0.
|
|
for p in protected:
|
|
if p['status'].get('phase') != 'Running': continue
|
|
uid = p['metadata']['uid'].replace('-', '_')
|
|
# Current production workloads are Burstable; refuse unknown cgroups.
|
|
path = '/hostcg/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod' + uid + '.slice/io.pressure'
|
|
io = max(io, full_io_pressure(self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'cat', path)))
|
|
# A trusted business-health feed is required; readiness/log absence is
|
|
# not evidence that saves/submits meet their latency/error baseline.
|
|
work_errors = business_health(self.args.work_health, binding_sha256=self.b.get('workHealthBindingSHA256'))
|
|
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
|
|
for p in pods:
|
|
if p['metadata'].get('deletionTimestamp'): continue
|
|
logs = self.call('-n', self.b['sourceNamespace'], 'logs', p['metadata']['name'], '--since=45s', '--tail=2000').lower()
|
|
work_errors |= api_business_log_failure(logs)
|
|
result = dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True,
|
|
workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes'])
|
|
if self.b.get('maintenanceInitialization'):
|
|
if not source or source.get('memorySupported') is not True or type(source.get('residentMiB')) not in (int,float) or not math.isfinite(source['residentMiB']) or source['residentMiB'] <= 0 or type(self.b.get('sourceMemoryLimitMiB')) not in (int,float) or not math.isfinite(self.b['sourceMemoryLimitMiB']) or self.b['sourceMemoryLimitMiB'] <= 0:
|
|
raise ProbeUnavailable('source memory evidence incomplete')
|
|
group='/hostcg/kubepods.slice/kubepods-pod'+pod['metadata']['uid'].replace('-','_')+'.slice/'
|
|
used_mem=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.current')
|
|
limit_mem=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.max')
|
|
events=self.call('-n',ns,'exec','query-mongo-0','-c','mongo','--','cat',group+'memory.events')
|
|
memory=parse_memory(used_mem,limit_mem,events)
|
|
node=json.loads(self.call('get','node','10.189.16.4','-o','json'))
|
|
pressure = [c['status'] for c in node['status']['conditions'] if c['type']=='MemoryPressure']
|
|
if len(pressure)!=1 or pressure[0] not in ('True','False'): raise ProbeUnavailable('node memory evidence incomplete')
|
|
node_pressure = pressure[0]=='True'
|
|
mongo_logs=self.call('-n',ns,'logs','query-mongo-0','-c','mongo','--since=45s','--tail=2000')
|
|
if len(mongo_logs.splitlines())>=2000: raise ProbeUnavailable('persistence log window truncated')
|
|
result.update(safetySignalsVersion=1,sourceAvailable=True,
|
|
memoryPressure=memory['pressure'] or node_pressure or source['residentMiB']>=.98*self.b['sourceMemoryLimitMiB'],
|
|
persistenceFailure=persistence_failed(mongo_logs))
|
|
return result
|
|
|
|
def publish(self, observation):
|
|
with self.target_forward(self.secret('query-shadow-query-connections', 'builder-uri')) as uri:
|
|
self.run_binary(['--record-observation', '-'], self.target_environment(uri), json.dumps(observation))
|
|
|
|
def unsafe(self, o):
|
|
self.high = self.high + 1 if not self.b.get('maintenanceInitialization') and o['cpu'] > self.b['baselineCPU'] + 3 else 0
|
|
self.high_io = self.high_io + 1 if o['protectedIOFull'] > 1 else 0
|
|
return (not o['ready'] or not o['writersCovered'] or o['workErrors'] or (not self.b.get('maintenanceInitialization') and (o['cpu'] >= 95 or self.high >= 2))
|
|
or (self.b.get('maintenanceInitialization') and (not self.b.get('maintenanceUntil') or dt.datetime.now(dt.timezone.utc)>=dt.datetime.fromisoformat(self.b['maintenanceUntil'].replace('Z','+00:00'))))
|
|
or o.get('memoryPressure',False) or o.get('persistenceFailure',False) or (self.b.get('maintenanceInitialization') and (o.get('safetySignalsVersion')!=1 or not o.get('sourceAvailable')))
|
|
or self.high_io >= 2 or o['protectedIOFull'] > 5 or o['targetUsedFraction'] >= .7 or o['hostFreeBytes'] < (100 << 30))
|
|
|
|
def stop_shadow(self):
|
|
for c in self.b['controllers']:
|
|
d = self.get(self.b['shadowNamespace'], 'deployment', c['name'])
|
|
if d['metadata']['uid'] != c['uid']: raise ValueError('stop controller replaced')
|
|
if d['spec']['replicas'] == 0: continue
|
|
patch = [{'op': 'test', 'path': '/metadata/uid', 'value': c['uid']},
|
|
{'op': 'test', 'path': '/metadata/resourceVersion', 'value': d['metadata']['resourceVersion']},
|
|
{'op': 'replace', 'path': '/spec/replicas', 'value': 0}]
|
|
self.call('-n', self.b['shadowNamespace'], 'patch', 'deployment', c['name'], '--type=json', '-p', json.dumps(patch))
|
|
|
|
def run(self):
|
|
try:
|
|
self.verify()
|
|
pods=json.loads(self.call('-n',self.b['sourceNamespace'],'get','pods','-l','app=droboticflow-api','-o','json'))['items']
|
|
self.verify_clocks(pods)
|
|
prior = retry_probe(self.source_sample,'initial-source')
|
|
except Exception:
|
|
if self.args.stop_on_failure: self.stop_shadow()
|
|
raise RuntimeError('initial source observation unavailable') from None
|
|
while True:
|
|
try:
|
|
for _ in range(3):
|
|
time.sleep(10)
|
|
if not retry_probe(self.readiness, 'readiness'):
|
|
raise ValueError('production readiness unavailable')
|
|
current = retry_probe(self.source_sample,'source-sample'); cpu = cpu_percent(prior, current); prior = current
|
|
sampled_at=current['sampledAt'];age=(dt.datetime.now(dt.timezone.utc)-dt.datetime.fromisoformat(sampled_at.replace('Z','+00:00'))).total_seconds()
|
|
if not 0<=age<=45: raise ValueError('source observation stale')
|
|
o = retry_probe(lambda: self.observation(cpu,sampled_at,current),'health-observation'); retry_probe(lambda: self.publish(o),'health-publish')
|
|
unsafe=self.unsafe(o)
|
|
print(json.dumps({'at':o['at'],'cpu':cpu,'ready':o['ready'],'unsafe':unsafe,'cpuObservedOnly':bool(self.b.get('maintenanceInitialization'))}),flush=True)
|
|
if unsafe:
|
|
if self.args.stop_on_failure: self.stop_shadow()
|
|
return
|
|
except Exception:
|
|
# No raw exception may print a URI, secret or customer log line.
|
|
with contextlib.suppress(Exception):
|
|
self.publish(dict(at=dt.datetime.now(dt.timezone.utc).isoformat(), cpu=0, ready=False, writersCovered=False,
|
|
workErrors=True, protectedIOFull=0, targetUsedFraction=0, hostFreeBytes=0))
|
|
if self.args.stop_on_failure: self.stop_shadow()
|
|
raise RuntimeError('observer paused; inspect private metadata and verify before resume') from None
|
|
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('--kubeconfig', required=True); p.add_argument('--bindings', required=True)
|
|
p.add_argument('--bindings-sha256', required=True); p.add_argument('--binary', required=True)
|
|
p.add_argument('--binary-sha256', required=True); p.add_argument('--work-health', required=True); p.add_argument('--stop-on-failure', action='store_true')
|
|
args = p.parse_args(); raw = Path(args.bindings).read_bytes()
|
|
if hashlib.sha256(raw).hexdigest() != args.bindings_sha256: raise ValueError('bindings changed')
|
|
b = json.loads(raw); check_bindings(b)
|
|
if not re.fullmatch(r'[a-f0-9]{64}', b.get('workHealthBindingSHA256', '')):
|
|
raise ValueError('reviewed business-health binding required')
|
|
Observer(args, b).run()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try: main()
|
|
except KeyboardInterrupt: raise SystemExit(0)
|
|
except Exception: raise SystemExit('observer stopped; private checkpoint retained; no automatic resume')
|