🛡️ Add cluster-resident query shadow health observer

This commit is contained in:
d-robotics 2026-11-14 20:59:54 -08:00
parent ad1d91a9fc
commit aaa788d77d
7 changed files with 441 additions and 23 deletions

View File

@ -0,0 +1,23 @@
FROM --platform=$BUILDPLATFORM golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS build
WORKDIR /src
ARG GOPROXY=https://proxy.golang.org,direct
COPY go.mod go.sum ./
RUN GOPROXY="${GOPROXY}" go mod download
COPY cmd/query-replica ./cmd/query-replica
COPY internal ./internal
ARG TARGETARCH
RUN CGO_ENABLED=0 GOOS=linux GOARCH="${TARGETARCH}" go build -trimpath -o /out/query-replica ./cmd/query-replica
ARG KUBECTL_SHA256
RUN test -n "${KUBECTL_SHA256}" && \
curl --fail --location --retry 2 "https://dl.k8s.io/release/v1.37.0/bin/linux/${TARGETARCH}/kubectl" -o /out/kubectl && \
echo "${KUBECTL_SHA256} /out/kubectl" | sha256sum --check --strict
FROM python@sha256:9534e5a8e315485d4061ed659af0fd78a284c015f9b73661b41d6bab25604534
COPY --from=build --chmod=0555 /out/kubectl /usr/local/bin/kubectl
COPY --from=build --chmod=0555 /out/query-replica /ops/query-replica
COPY --chmod=0444 scripts/local/query-shadow-monitor.py scripts/local/query-shadow-observer.py scripts/local/query-shadow-work-health.py scripts/local/query_shadow_health.py /ops/
ARG DROBOTICFLOW_BUILD_COMMIT=unknown
LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT}
ENV HOME=/tmp PYTHONDONTWRITEBYTECODE=1
USER 10001:10001
ENTRYPOINT ["python3", "-u", "/ops/query-shadow-monitor.py"]

View File

@ -0,0 +1,37 @@
# 集群内影子观察器
状态:本地实现与验证,新增组件尚未部署。生产入口保持关闭,不代表分流或原 Mongo 降载已完成。
## 原因与兼容范围
影子同步已复制200条记录、消耗325/1000预留,随后因健康证据过期持久暂停。本机 Linux VM 的实时时钟曾倒退十余秒,误触发“日志来自未来”;本机时间不能作为长期影子同步的基础设施。此前 NodeLocalDNS 经 Cilium 转到本节点 host:10053 的丢包已按精确授权修复。
新增观察器运行在集群,沿用原业务日志分类、阈值、45秒新鲜度、1秒时钟偏差、硬故障停止和多数持久化要求。维护初始化期间 CPU 仅观察,不恢复旧 CPU 启动门槛。既有本地观察器默认行为不变。观察器不领取、保存、切片、送检、审核或验收,不更改客户标注。重启不解除持久暂停,也不重置预算。
## 实现
`scripts/local/query-shadow-monitor.py` 直接使用已批准的源只读指标连接和查询库健康记录连接;凭证通过现有 Secret env 引用注入,子进程按角色移除另一端凭证。无需读取 Secret API,不再借生产 Pod 转发 Mongo TCP,也不依赖本机端口转发。
每轮读取全部已绑定 API Pod 的日志,最多4路并发、每个集群请求3秒/进程5秒。任一 Pod 缺失、重启、权限拒绝或日志截断都不能记为健康。仅保留分类计数,不存客户正文或原始日志。HTTP慢/错误尾部不等于全量p95,也不能证明异步作业完成。
时钟通过每个 API Pod 的 `HEAD /` 响应 Date 验证,不执行生产容器命令,不读取业务数据。两次测量间隔0.55秒,保留秒级Date的整个误差区间,要求交集完整落在±1秒内,RTT≤2秒;检测本地实时/单调时钟差异。瞬时探测最多3次重试,明确不健康不重试成成功。日志时钟失败记录独立分项。
共享盘剩余容量从查询 Mongo 现有 `/lab` 父挂载的 `stat -f` 获取;25Gi占用仍读取 `/lab/data`。正式部署前需只读证明父挂载与原 node summary 指向同一底层文件系统、容量结果一致。保留现有 IO、内存、源可用性和持久化检查。
停止仅允许对两条已绑定控制器的 `deployments/scale` 以UID/RV CAS置0;第一条停止失败仍尝试另一条。健康记录过期继续触发已有持久暂停。观察器不自动恢复同步、不增加预算、不延长截止时间。
## 待审批的完整资源范围
- 新增1个 `query-shadow-monitor` Deployment、ServiceAccount、非机密ConfigMap;复用专用拉取Secret,不新增账号/Secret/PVC。资源请求250m/256Mi、上限500m/512Mi;非root、只读根、去除全部capabilities,无hostPath、hostNetwork或入站Service。
- 影子配额请求CPU由6到6250m、内存12Gi到12544Mi,上限CPU8到8500m、内存16Gi到16896Mi;ConfigMap2到3。Pod上限6、Secret上限8及磁盘边界不变。部署前重新核对节点实际余量。
- 专用RBAC只读源命名空间5个写者Deployment、Pod列表、当前4个API Pod日志;只读两个命名空间身份与`.4`节点健康。影子侧只读两条Deployment、查询Mongo Pod及日志,允许对`query-mongo-0`读取诊断的exec和两条控制器scale CAS。不授予Secret、生产exec、节点proxy、客户数据写或通配管理权限。`pods/exec`可执行容器命令,须在审批中明确该权限边界,不能称为技术上只读。
- 新增观察器专用网络策略:本节点host UDP/TCP10053、API Pod TCP3001、源10.189.16.32 TCP27017、查询Mongo TCP27129、kube-apiserver TCP443/6443;查询Mongo仅增加同影子命名空间观察器入站。无world、remote-node、全局Cilium或生产策略修改。
- 投影器/构建器既有ad1d91a9镜像及参数不变;观察器需独立候选、主线同步、上传和最终部署授权。不能以原DNS授权部署新组件。
## 启动、验收与回退
1. 本地回归及镜像检查通过,候选合入同步dev;准备不可变镜像、精确资源清单、UID/RV前置快照和回退命令,最终计划单独确认。
2. 新观察器先独立观察,两个影子作业保持0;至少两个完整健康周期和目标写入确认通过,核对身份、日志覆盖、时钟和共享盘容量,不执行客户业务写测试。
3. 复用同run及当前恢复令牌,保留325已消耗预留和200条记录,仅恢复原1000小批余量675。禁止按空库初始化,禁止自动解暂停。核验后按原批准分档继续,不超1500万或原固定截止2026-11-15 14:38:42 UTC。
4. 追平、对账后才能启动构建器,在查询库发布完整结果到Valkey;缓存失联/查询库重启不能回源风暴。正确性、权限隔离、速度与原库降载验收尚待执行,生产查询入口另案批准。
5. 异常先停两影子作业,保留断点、已有事实和安全暂停;再停观察器。仅按所创建UID删除本次新增对象、解除其RoleBinding,恢复配额原值前核对实际使用量。既有DNS策略、专用拉取Secret、查询Mongo、Valkey和客户数据不删除。回退不重置run或预留量。

View File

@ -0,0 +1,220 @@
#!/usr/bin/env python3
"""Cluster-resident shadow observer; requires a separately approved deployment.
No production exec, Secret API access, node proxy, automatic resume or query cutover.
Database credentials come from reviewed Secret env references, never log arguments.
"""
import argparse
import contextlib
import datetime as dt
import email.utils
import hashlib
import importlib.util
import json
import os
from pathlib import Path
import subprocess
import time
import urllib.parse
import urllib.request
from query_shadow_health import ProbeUnavailable, emit, retry_probe
def load(name, filename):
spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(filename))
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
observer = load('cluster_observer_base', 'query-shadow-observer.py')
work = load('cluster_work_health', 'query-shadow-work-health.py')
def date_interval(header, start, end, elapsed):
"""HTTP Date has one-second resolution; retain its whole uncertainty interval."""
try:
remote = email.utils.parsedate_to_datetime(header)
if remote.tzinfo is None:
raise ValueError()
except (ValueError, TypeError, OverflowError):
raise ProbeUnavailable('clock header unavailable') from None
if not 0 <= elapsed <= 2 or abs((end - start).total_seconds() - elapsed) > .1:
raise ProbeUnavailable('clock transport or local clock changed')
return (remote - end).total_seconds(), (remote + dt.timedelta(seconds=1) - start).total_seconds()
def check_http_clocks(pods, port, opener=urllib.request.urlopen,
utcnow=lambda: dt.datetime.now(dt.timezone.utc), monotonic=time.monotonic,
sleep=time.sleep, diagnostic=emit):
for pod in pods:
if pod['metadata'].get('deletionTimestamp') or pod['status'].get('phase') != 'Running':
raise ValueError('clock pod unavailable')
ip = pod['status']['podIP']
def measure():
intervals = []
for index in range(2):
if index:
sleep(.55)
start = utcnow(); tick = monotonic()
request = urllib.request.Request('http://' + ip + ':' + str(port) + '/', method='HEAD')
try:
with opener(request, timeout=2) as response:
if response.status >= 500:
raise ValueError('API clock endpoint unhealthy')
header = response.headers.get('Date')
except (OSError, urllib.error.HTTPError) as error:
if isinstance(error, urllib.error.HTTPError) and error.code >= 500:
raise ValueError('API clock endpoint unhealthy') from None
raise ProbeUnavailable('clock transport unavailable') from None
end = utcnow(); elapsed = monotonic() - tick
intervals.append(date_interval(header, start, end, elapsed))
low = max(x[0] for x in intervals); high = min(x[1] for x in intervals)
passed = -1 <= low <= high <= 1
diagnostic(dict(event='clock_probe', podUID=pod['metadata']['uid'],
offsetLowerSeconds=low, offsetUpperSeconds=high, passed=passed))
if not passed:
raise ProbeUnavailable('clock interval outside reviewed bound')
return low, high
retry_probe(measure, 'api-http-clock', diagnostic=diagnostic, sleep=sleep)
class ClusterObserver(observer.Observer):
def __init__(self, args, binding, work_binding):
super().__init__(args, binding)
self.work_binding = work_binding
# Service-account tokenFile is read by kubectl on each request (rotation).
self.k += ['--request-timeout=3s']
def call(self, *args, data=None, timeout=5):
try:
return super().call(*args, data=data, timeout=min(timeout, 5))
except (ProbeUnavailable, subprocess.TimeoutExpired, OSError) as error:
# Never include stderr, URLs, argv, credentials or customer log lines.
phase = next((x for x in ('logs', 'exec', 'patch', 'get') if x in args), 'cluster')
emit(dict(event='probe_failure', phase=phase, failureType=type(error).__name__))
raise
def run_binary(self, args, env, data=None):
try:
return super().run_binary(args, env, data)
except Exception as error:
emit(dict(event='probe_failure', phase='source-metadata' if args == ['--sample-source'] else 'health-publish',
failureType=type(error).__name__))
raise
def source_sample(self):
uri = os.environ['SHADOW_MONITOR_SOURCE_URI']
parsed = urllib.parse.urlsplit(uri)
endpoint = (parsed.hostname or '') + ':' + str(parsed.port or 27017)
if (parsed.scheme != 'mongodb' or ',' in parsed.netloc or
hashlib.sha256(endpoint.encode()).hexdigest() != self.b['sourceEndpointSHA256']):
raise ValueError('source endpoint changed')
env = self.target_environment('')
env.update(MONGO_URI=uri, MONGO_DB=self.b['sourceDatabase'])
return json.loads(self.run_binary(['--sample-source'], env))
def target_environment(self, uri):
# Do not pass both authority and query credentials to either child.
env = {k: v for k, v in os.environ.items()
if not k.startswith(('MONGO_', 'QUERY_', 'SHADOW_MONITOR_'))}
env.update(QUERY_READ_MONGO_URI=uri, QUERY_READ_MONGO_DB=self.b['targetDatabase'],
QUERY_SERVING_NAMESPACE=self.b['servingNamespace'])
return env
def publish(self, observation):
self.run_binary(['--record-observation', '-'],
self.target_environment(os.environ['SHADOW_MONITOR_TARGET_URI']), json.dumps(observation))
def host_filesystem(self):
# /lab is the existing host mount's parent filesystem, /lab/data the 25Gi loop.
fields = self.call('-n', self.b['shadowNamespace'], 'exec', 'query-mongo-0',
'-c', 'mongo', '--', 'stat', '-f', '-c', '%a %S', '/lab').split()
if len(fields) != 2 or any(not x.isdigit() for x in fields) or int(fields[1]) <= 0:
raise ProbeUnavailable('host free-space evidence unavailable')
return dict(availableBytes=int(fields[0]) * int(fields[1]))
def verify_clocks(self, pods):
return check_http_clocks(pods, self.b['apiHTTPPort'])
def observation(self, cpu, sampled_at, source=None):
# Check every cycle, not just startup. No timestamp rewriting on retry.
pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods',
'-l', 'app=droboticflow-api', '-o', 'json'))['items']
self.verify_clocks(pods)
def collect():
try:
return work.collect(self.k, self.work_binding, log_workers=4,
caller=lambda _k, *values: self.call(*values))
except ValueError as error:
if str(error) in ('log clock unavailable', 'log clock in future'):
emit(dict(event='probe_failure', phase='business-log-clock', reason=str(error)))
raise ProbeUnavailable('business log clock unavailable') from None
raise
health = retry_probe(collect, 'business-health')
health['bindingSHA256'] = self.b['workHealthBindingSHA256']
work.atomic_write(self.args.work_health, health)
return super().observation(cpu, sampled_at, source)
def stop_shadow(self):
# Deployment body is read-only; only exact two /scale subresources writable.
failed = False
for c in self.b['controllers']:
try:
scale = self.get(self.b['shadowNamespace'], 'deployment/' + c['name'],
'--subresource=scale')
if scale['metadata']['uid'] != c['uid']:
raise ValueError('stop controller replaced')
if scale['spec'].get('replicas', 0) == 0:
continue
patch = [dict(op='test', path='/metadata/uid', value=c['uid']),
dict(op='test', path='/metadata/resourceVersion', value=scale['metadata']['resourceVersion']),
dict(op='replace', path='/spec/replicas', value=0)]
self.call('-n', self.b['shadowNamespace'], 'patch', 'deployment', c['name'],
'--subresource=scale', '--type=json', '-p', json.dumps(patch))
except Exception as error:
failed = True
emit(dict(event='shadow_stop_failed', controller=c['name'], failureType=type(error).__name__))
if failed:
raise ValueError('shadow stop incomplete; durable health expiry still applies')
def checked_json(path, digest):
raw = Path(path).read_bytes()
if hashlib.sha256(raw).hexdigest() != digest:
raise ValueError('reviewed binding changed')
return json.loads(raw)
def main():
parser = argparse.ArgumentParser(description=__doc__)
for key in ('kubeconfig', 'bindings', 'bindings-sha256', 'work-bindings', 'binary', 'binary-sha256'):
parser.add_argument('--' + key, required=True)
parser.add_argument('--work-health', default='/tmp/work-health.json')
args = parser.parse_args()
args.stop_on_failure = True
binding = checked_json(args.bindings, args.bindings_sha256)
observer.check_bindings(binding)
if type(binding.get('apiHTTPPort')) is not int or not 1 <= binding['apiHTTPPort'] <= 65535:
raise ValueError('reviewed API port required')
work_binding = checked_json(args.work_bindings, binding['workHealthBindingSHA256'])
if (work_binding['sourceNamespace'] != binding['sourceNamespace'] or
work_binding['sourceNamespaceUID'] != binding['sourceNamespaceUID']):
raise ValueError('business-health namespace mismatch')
ready = urllib.parse.urlsplit(binding['readyURL'])
if (ready.scheme != 'http' or ready.hostname != 'droboticflow-api.' + binding['sourceNamespace'] + '.svc'
or ready.port not in (None, 80) or ready.path != '/api/ready'
or ready.username or ready.password or ready.query or ready.fragment):
raise ValueError('reviewed internal readiness endpoint required')
ClusterObserver(args, binding, work_binding).run()
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
raise SystemExit(0)
except Exception as error:
emit(dict(event='monitor_stopped', failureType=type(error).__name__, automaticResume=False))
raise SystemExit(2) from None

View File

@ -214,13 +214,19 @@ class Observer:
except (OSError, ValueError) as error:
raise ProbeUnavailable('readiness transport unavailable') from None
def host_filesystem(self):
return json.loads(self.call('get', '--raw', '/api/v1/nodes/10.189.16.4/proxy/stats/summary'))['node']['fs']
def verify_clocks(self, pods):
return check_clocks(self.call, self.b['sourceNamespace'], pods)
def observation(self, cpu, sampled_at, source=None):
self.verify()
ready = retry_probe(self.readiness, 'readiness')
ns = self.b['shadowNamespace']; pod = self.get(ns, 'pod', 'query-mongo-0')
if pod['metadata']['uid'] != self.b['mongoPodUID']:
raise ValueError('query Mongo rebuilt; recovery verification required')
fs = json.loads(self.call('get', '--raw', '/api/v1/nodes/10.189.16.4/proxy/stats/summary'))['node']['fs']
fs = self.host_filesystem()
text = self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'df', '-P', '-B1', '/lab/data')
fields = text.splitlines()[-1].split(); used = int(fields[2]) / int(fields[1])
protected = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '--field-selector=spec.nodeName=10.189.16.4', '-o', 'json'))['items']
@ -286,7 +292,7 @@ class Observer:
try:
self.verify()
pods=json.loads(self.call('-n',self.b['sourceNamespace'],'get','pods','-l','app=droboticflow-api','-o','json'))['items']
check_clocks(self.call,self.b['sourceNamespace'],pods)
self.verify_clocks(pods)
prior = retry_probe(self.source_sample,'initial-source')
except Exception:
if self.args.stop_on_failure: self.stop_shadow()

View File

@ -14,6 +14,7 @@ from pathlib import Path
import re
import subprocess
import time
from concurrent.futures import ThreadPoolExecutor
OPS = ('save', 'slice', 'submit', 'review', 'acceptance')
TRACE = re.compile(r'\brequest_trace id=([a-f0-9]{32}) kind=(slow|failed) method=(PUT|POST) path=(/api/\S+) status=(\d{3}) elapsed_ms=(\d+) observations=')
@ -80,27 +81,31 @@ def call(k, *args):
if p.returncode: raise RuntimeError('bounded read-only metrics unavailable')
return p.stdout
def collect(k, binding, seconds=30, max_lines=4000):
def collect(k, binding, seconds=30, max_lines=4000, caller=None, log_workers=1):
caller = caller or call
if not 10 <= seconds <= 30: raise ValueError('invalid observation window')
if not 1 <= log_workers <= 4: raise ValueError('invalid log concurrency')
now = dt.datetime.now(dt.timezone.utc); start = now - dt.timedelta(seconds=seconds)
ns = binding['sourceNamespace']
namespace = json.loads(call(k, 'get', 'namespace', ns, '-o', 'json'))
namespace = json.loads(caller(k, 'get', 'namespace', ns, '-o', 'json'))
if namespace['metadata']['uid'] != binding['sourceNamespaceUID']: raise ValueError('namespace replaced')
d = json.loads(call(k, '-n', ns, 'get', 'deployment', 'droboticflow-api', '-o', 'json'))
d = json.loads(caller(k, '-n', ns, 'get', 'deployment', 'droboticflow-api', '-o', 'json'))
if d['metadata']['uid'] != binding['apiDeploymentUID']: raise ValueError('API replaced')
c = d['spec']['template']['spec']['containers'][0]
if c['image'] != binding['apiImage'] or d.get('status', {}).get('updatedReplicas') != binding['apiReplicas'] or d.get('status', {}).get('availableReplicas') != binding['apiReplicas']:
raise ValueError('API identity or coverage changed')
pods = json.loads(call(k, '-n', ns, 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
pods = json.loads(caller(k, '-n', ns, 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items']
active = [p for p in pods if p['status'].get('phase') not in ('Failed', 'Succeeded')]
if {p['metadata']['uid'] for p in active} != set(binding['apiPodUIDs']): raise ValueError('API pod set changed')
logs = []
for p in active:
statuses = p['status'].get('containerStatuses', [])
if (p['metadata'].get('deletionTimestamp') or p['status'].get('phase') != 'Running'
or not statuses or not all(s.get('ready') and s.get('restartCount', 0) == binding['podRestarts'][p['metadata']['uid']] for s in statuses)):
raise ValueError('API pod health changed')
logs.append(call(k, '-n', ns, 'logs', p['metadata']['name'], '-c', c['name'], '--timestamps=true', '--since-time=' + start.isoformat(), '--tail=' + str(max_lines)))
def read_log(p):
return caller(k, '-n', ns, 'logs', p['metadata']['name'], '-c', c['name'], '--timestamps=true', '--since-time=' + start.isoformat(), '--tail=' + str(max_lines))
with ThreadPoolExecutor(max_workers=log_workers) as pool:
logs = list(pool.map(read_log, active))
rows = summarize(logs, binding['tailLimitsMs'], start, now, max_lines)
# Mark the start of collection, not publication, so slow reads cannot refresh
# evidence which is already stale. Normal requests below1s remain uncounted.

View File

@ -0,0 +1,137 @@
import datetime as dt
import hashlib
import importlib.util
import io
import json
import os
from pathlib import Path
from types import SimpleNamespace
import unittest
from unittest.mock import patch
spec = importlib.util.spec_from_file_location('monitor', Path(__file__).with_name('query-shadow-monitor.py'))
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
UTC = dt.timezone.utc
class MonitorTests(unittest.TestCase):
def make(self):
b = dict(sourceNamespace='authority', shadowNamespace='shadow', sourceDatabase='source', targetDatabase='query',
servingNamespace='serving', controllers=[dict(name='query-replica', uid='p'), dict(name='query-read-builder', uid='b')],
sourceEndpointSHA256=hashlib.sha256(b'private-source:27017').hexdigest())
return m.ClusterObserver(SimpleNamespace(kubeconfig='unused'), b, {})
def test_clock_resolution_and_backwards_step(self):
start = dt.datetime(2026, 11, 15, 5, 0, 0, 400000, tzinfo=UTC)
low, high = m.date_interval('Sun, 15 Nov 2026 05:00:00 GMT', start, start + dt.timedelta(seconds=.1), .1)
self.assertAlmostEqual(low, -.5); self.assertAlmostEqual(high, .6)
for header, end, elapsed in [(None, start, .1), ('bad', start, .1),
('Sun, 15 Nov 2026 05:00:00 GMT', start - dt.timedelta(seconds=15), .1),
('Sun, 15 Nov 2026 05:00:00 GMT', start + dt.timedelta(seconds=3), 3)]:
with self.assertRaises(m.ProbeUnavailable): m.date_interval(header, start, end, elapsed)
def test_http_clock_all_pods_bounded_without_exec(self):
pods = [dict(metadata=dict(uid='pod' + str(i)), status=dict(phase='Running', podIP='10.0.0.' + str(i))) for i in range(1, 5)]
now = dt.datetime(2026, 11, 15, 5, 0, 0, 400000, tzinfo=UTC)
dates = [now + dt.timedelta(seconds=x) for x in (0, .05, .6, .65)] * 4
response = SimpleNamespace(status=200, headers={'Date': 'Sun, 15 Nov 2026 05:00:00 GMT'})
class Context:
def __enter__(self): return response
def __exit__(self, *args): pass
calls = []
def opener(request, timeout):
calls.append((request.get_method(), request.full_url, timeout)); return Context()
m.check_http_clocks(pods, 3001, opener=opener, utcnow=iter(dates).__next__,
monotonic=iter([0, .05, .6, .65] * 4).__next__, sleep=lambda _: None, diagnostic=lambda _: None)
self.assertEqual(len(calls), 8)
self.assertTrue(all(method == 'HEAD' and url.endswith(':3001/') and timeout == 2 for method, url, timeout in calls))
def test_clock_failure_is_diagnosed_and_retried_three_times(self):
pod = dict(metadata=dict(uid='pod'), status=dict(phase='Running', podIP='10.0.0.1'))
diagnostics = []
with self.assertRaises(m.ProbeUnavailable):
m.check_http_clocks([pod], 3001, opener=lambda *a, **k: (_ for _ in ()).throw(OSError('credential must not print')),
sleep=lambda _: None, diagnostic=diagnostics.append)
self.assertEqual(len([x for x in diagnostics if x.get('event') == 'probe_retry']), 3)
self.assertNotIn('credential', json.dumps(diagnostics))
def test_source_and_target_credentials_are_separated_no_relay(self):
o = self.make(); calls = []
o.run_binary = lambda args, env, data=None: calls.append((args, env, data)) or '{}'
o.secret = lambda *a: self.fail('Secret API must never be used')
with patch.dict(os.environ, dict(SHADOW_MONITOR_SOURCE_URI='mongodb://user:password@private-source:27017/source',
SHADOW_MONITOR_TARGET_URI='mongodb://target:secret@query:27129/query',
MONGO_URI='untrusted', QUERY_READ_MONGO_URI='untrusted')):
o.source_sample(); o.publish(dict(ready=True))
self.assertIn('private-source', calls[0][1]['MONGO_URI'])
self.assertEqual(calls[0][1]['QUERY_READ_MONGO_URI'], '')
self.assertNotIn('MONGO_URI', calls[1][1])
self.assertTrue(all(not any(k.startswith('SHADOW_MONITOR_') for k in c[1]) for c in calls))
with patch.dict(os.environ, dict(SHADOW_MONITOR_SOURCE_URI='mongodb://wrong:27017/source')):
with self.assertRaises(ValueError): o.source_sample()
def test_host_space_uses_parent_mount_not_node_proxy(self):
o = self.make(); calls = []
o.call = lambda *a, **k: calls.append(a) or '30000000 4096\n'
self.assertEqual(o.host_filesystem()['availableBytes'], 30000000 * 4096)
self.assertEqual(calls[0][-1], '/lab')
self.assertNotIn('proxy', str(calls))
o.call = lambda *a: 'unknown'
with self.assertRaises(m.ProbeUnavailable): o.host_filesystem()
def test_stop_only_scale_uid_rv_cas_and_restart_does_not_resume(self):
o = self.make(); calls = []
o.get = lambda ns, kind, name: dict(metadata=dict(uid='p' if kind.endswith('query-replica') else 'b', resourceVersion='7'), spec=dict(replicas=1))
o.call = lambda *a, **k: calls.append(a) or ''
o.stop_shadow()
self.assertEqual(len(calls), 2)
for call in calls:
self.assertIn('--subresource=scale', call)
payload = json.loads(call[-1])
self.assertEqual([(x['op'], x['path']) for x in payload], [('test', '/metadata/uid'), ('test', '/metadata/resourceVersion'), ('replace', '/spec/replicas')])
self.assertEqual(payload[-1]['value'], 0)
other = self.make()
self.assertEqual(other.high, 0) # Constructor performs no Kubernetes/database writes.
other.get = lambda *a: dict(metadata=dict(uid='replaced'), spec=dict(replicas=1))
with self.assertRaises(ValueError): other.stop_shadow()
def test_parallel_business_logs_require_complete_coverage(self):
b = dict(sourceNamespace='authority', sourceNamespaceUID='ns', apiDeploymentUID='api', apiImage='image', apiReplicas=4,
apiPodUIDs=['p'+str(i) for i in range(4)], podRestarts={'p'+str(i): 0 for i in range(4)},
tailLimitsMs=dict(save=5000, slice=10000, submit=15000, review=5000, acceptance=15000))
pods = [dict(metadata=dict(uid='p'+str(i), name='api-'+str(i)), status=dict(phase='Running', containerStatuses=[dict(ready=True, restartCount=0)])) for i in range(4)]
deployment = dict(metadata=dict(uid='api'), spec=dict(template=dict(spec=dict(containers=[dict(name='api', image='image')]))), status=dict(updatedReplicas=4, availableReplicas=4))
logs = []
def call(k, *a):
if 'namespace' in a: return json.dumps(dict(metadata=dict(uid='ns')))
if 'deployment' in a: return json.dumps(deployment)
if 'pods' in a: return json.dumps(dict(items=pods))
logs.append(a[a.index('logs')+1]); return ''
result = m.work.collect([], b, caller=call, log_workers=4)
self.assertEqual(set(logs), {'api-'+str(i) for i in range(4)})
self.assertEqual(result['monitoredPods'], 4)
def denied(k, *a):
if 'api-3' in a: raise m.ProbeUnavailable('denied')
return call(k, *a)
with self.assertRaises(m.ProbeUnavailable): m.work.collect([], b, caller=denied, log_workers=4)
def test_stop_failure_still_attempts_other_controller(self):
o = self.make(); calls = []
def get(ns, kind, name):
calls.append(kind)
if kind.endswith('query-replica'): raise OSError('private error')
return dict(metadata=dict(uid='b', resourceVersion='8'), spec=dict(replicas=0))
o.get = get
with patch.object(m, 'emit') as diagnostic:
with self.assertRaises(ValueError): o.stop_shadow()
self.assertEqual(len(calls), 2)
self.assertNotIn('private error', str(diagnostic.call_args))
def test_scale_omits_zero_replicas(self):
o = self.make()
o.get = lambda ns, kind, name: dict(metadata=dict(uid='p' if kind.endswith('query-replica') else 'b'), spec={})
o.call = lambda *a, **k: self.fail('already stopped Scale must not be patched')
o.stop_shadow()
if __name__ == '__main__': unittest.main()

View File

@ -1018,27 +1018,17 @@
{
"name": "shadow-work-health",
"paths": [
"^scripts/local/(query-shadow-(observer|work-health)|query_shadow_health|test_query_shadow_(observer|work_health|health))\\.py$"
"^scripts/local/(query-shadow-(observer|work-health|monitor)|query_shadow_health|test_query_shadow_(observer|work_health|health|monitor))\\.py$",
"^Dockerfile\\.query-shadow-monitor$",
"^docs/operations/query-shadow-monitor\\.md$"
],
"frontend": [],
"go": {
"./internal/api": [
"^TestDiagnosticTrace"
]
"./internal/api": ["^TestDiagnosticTrace"]
},
"task": [
{
"argv": [
"python3",
"-m",
"unittest",
"discover",
"-s",
"scripts/local",
"-p",
"test_query_shadow*.py",
"-v"
],
"argv": ["python3", "-m", "unittest", "discover", "-s", "scripts/local", "-p", "test_query_shadow*.py", "-v"],
"tests": true
}
],