diff --git a/Dockerfile.query-shadow-monitor b/Dockerfile.query-shadow-monitor new file mode 100644 index 00000000..85a2f7ad --- /dev/null +++ b/Dockerfile.query-shadow-monitor @@ -0,0 +1,23 @@ +FROM --platform=$BUILDPLATFORM golang:1.24-bookworm@sha256:1a6d4452c65dea36aac2e2d606b01b4a029ec90cc1ae53890540ce6173ea77ac AS build +WORKDIR /src +ARG GOPROXY=https://proxy.golang.org,direct +COPY go.mod go.sum ./ +RUN GOPROXY="${GOPROXY}" go mod download +COPY cmd/query-replica ./cmd/query-replica +COPY internal ./internal +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=linux GOARCH="${TARGETARCH}" go build -trimpath -o /out/query-replica ./cmd/query-replica +ARG KUBECTL_SHA256 +RUN test -n "${KUBECTL_SHA256}" && \ + curl --fail --location --retry 2 "https://dl.k8s.io/release/v1.37.0/bin/linux/${TARGETARCH}/kubectl" -o /out/kubectl && \ + echo "${KUBECTL_SHA256} /out/kubectl" | sha256sum --check --strict + +FROM python@sha256:9534e5a8e315485d4061ed659af0fd78a284c015f9b73661b41d6bab25604534 +COPY --from=build --chmod=0555 /out/kubectl /usr/local/bin/kubectl +COPY --from=build --chmod=0555 /out/query-replica /ops/query-replica +COPY --chmod=0444 scripts/local/query-shadow-monitor.py scripts/local/query-shadow-observer.py scripts/local/query-shadow-work-health.py scripts/local/query_shadow_health.py /ops/ +ARG DROBOTICFLOW_BUILD_COMMIT=unknown +LABEL org.opencontainers.image.revision=${DROBOTICFLOW_BUILD_COMMIT} +ENV HOME=/tmp PYTHONDONTWRITEBYTECODE=1 +USER 10001:10001 +ENTRYPOINT ["python3", "-u", "/ops/query-shadow-monitor.py"] diff --git a/docs/operations/query-shadow-monitor.md b/docs/operations/query-shadow-monitor.md new file mode 100644 index 00000000..718c2c6b --- /dev/null +++ b/docs/operations/query-shadow-monitor.md @@ -0,0 +1,37 @@ +# 集群内影子观察器 + +状态:本地实现与验证,新增组件尚未部署。生产入口保持关闭,不代表分流或原 Mongo 降载已完成。 + +## 原因与兼容范围 + +影子同步已复制200条记录、消耗325/1000预留,随后因健康证据过期持久暂停。本机 Linux VM 的实时时钟曾倒退十余秒,误触发“日志来自未来”;本机时间不能作为长期影子同步的基础设施。此前 NodeLocalDNS 经 Cilium 转到本节点 host:10053 的丢包已按精确授权修复。 + +新增观察器运行在集群,沿用原业务日志分类、阈值、45秒新鲜度、1秒时钟偏差、硬故障停止和多数持久化要求。维护初始化期间 CPU 仅观察,不恢复旧 CPU 启动门槛。既有本地观察器默认行为不变。观察器不领取、保存、切片、送检、审核或验收,不更改客户标注。重启不解除持久暂停,也不重置预算。 + +## 实现 + +`scripts/local/query-shadow-monitor.py` 直接使用已批准的源只读指标连接和查询库健康记录连接;凭证通过现有 Secret env 引用注入,子进程按角色移除另一端凭证。无需读取 Secret API,不再借生产 Pod 转发 Mongo TCP,也不依赖本机端口转发。 + +每轮读取全部已绑定 API Pod 的日志,最多4路并发、每个集群请求3秒/进程5秒。任一 Pod 缺失、重启、权限拒绝或日志截断都不能记为健康。仅保留分类计数,不存客户正文或原始日志。HTTP慢/错误尾部不等于全量p95,也不能证明异步作业完成。 + +时钟通过每个 API Pod 的 `HEAD /` 响应 Date 验证,不执行生产容器命令,不读取业务数据。两次测量间隔0.55秒,保留秒级Date的整个误差区间,要求交集完整落在±1秒内,RTT≤2秒;检测本地实时/单调时钟差异。瞬时探测最多3次重试,明确不健康不重试成成功。日志时钟失败记录独立分项。 + +共享盘剩余容量从查询 Mongo 现有 `/lab` 父挂载的 `stat -f` 获取;25Gi占用仍读取 `/lab/data`。正式部署前需只读证明父挂载与原 node summary 指向同一底层文件系统、容量结果一致。保留现有 IO、内存、源可用性和持久化检查。 + +停止仅允许对两条已绑定控制器的 `deployments/scale` 以UID/RV CAS置0;第一条停止失败仍尝试另一条。健康记录过期继续触发已有持久暂停。观察器不自动恢复同步、不增加预算、不延长截止时间。 + +## 待审批的完整资源范围 + +- 新增1个 `query-shadow-monitor` Deployment、ServiceAccount、非机密ConfigMap;复用专用拉取Secret,不新增账号/Secret/PVC。资源请求250m/256Mi、上限500m/512Mi;非root、只读根、去除全部capabilities,无hostPath、hostNetwork或入站Service。 +- 影子配额请求CPU由6到6250m、内存12Gi到12544Mi,上限CPU8到8500m、内存16Gi到16896Mi;ConfigMap2到3。Pod上限6、Secret上限8及磁盘边界不变。部署前重新核对节点实际余量。 +- 专用RBAC只读源命名空间5个写者Deployment、Pod列表、当前4个API Pod日志;只读两个命名空间身份与`.4`节点健康。影子侧只读两条Deployment、查询Mongo Pod及日志,允许对`query-mongo-0`读取诊断的exec和两条控制器scale CAS。不授予Secret、生产exec、节点proxy、客户数据写或通配管理权限。`pods/exec`可执行容器命令,须在审批中明确该权限边界,不能称为技术上只读。 +- 新增观察器专用网络策略:本节点host UDP/TCP10053、API Pod TCP3001、源10.189.16.32 TCP27017、查询Mongo TCP27129、kube-apiserver TCP443/6443;查询Mongo仅增加同影子命名空间观察器入站。无world、remote-node、全局Cilium或生产策略修改。 +- 投影器/构建器既有ad1d91a9镜像及参数不变;观察器需独立候选、主线同步、上传和最终部署授权。不能以原DNS授权部署新组件。 + +## 启动、验收与回退 + +1. 本地回归及镜像检查通过,候选合入同步dev;准备不可变镜像、精确资源清单、UID/RV前置快照和回退命令,最终计划单独确认。 +2. 新观察器先独立观察,两个影子作业保持0;至少两个完整健康周期和目标写入确认通过,核对身份、日志覆盖、时钟和共享盘容量,不执行客户业务写测试。 +3. 复用同run及当前恢复令牌,保留325已消耗预留和200条记录,仅恢复原1000小批余量675。禁止按空库初始化,禁止自动解暂停。核验后按原批准分档继续,不超1500万或原固定截止2026-11-15 14:38:42 UTC。 +4. 追平、对账后才能启动构建器,在查询库发布完整结果到Valkey;缓存失联/查询库重启不能回源风暴。正确性、权限隔离、速度与原库降载验收尚待执行,生产查询入口另案批准。 +5. 异常先停两影子作业,保留断点、已有事实和安全暂停;再停观察器。仅按所创建UID删除本次新增对象、解除其RoleBinding,恢复配额原值前核对实际使用量。既有DNS策略、专用拉取Secret、查询Mongo、Valkey和客户数据不删除。回退不重置run或预留量。 diff --git a/scripts/local/query-shadow-monitor.py b/scripts/local/query-shadow-monitor.py new file mode 100644 index 00000000..526621d1 --- /dev/null +++ b/scripts/local/query-shadow-monitor.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Cluster-resident shadow observer; requires a separately approved deployment. + +No production exec, Secret API access, node proxy, automatic resume or query cutover. +Database credentials come from reviewed Secret env references, never log arguments. +""" +import argparse +import contextlib +import datetime as dt +import email.utils +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import time +import urllib.parse +import urllib.request + +from query_shadow_health import ProbeUnavailable, emit, retry_probe + + +def load(name, filename): + spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(filename)) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +observer = load('cluster_observer_base', 'query-shadow-observer.py') +work = load('cluster_work_health', 'query-shadow-work-health.py') + + +def date_interval(header, start, end, elapsed): + """HTTP Date has one-second resolution; retain its whole uncertainty interval.""" + try: + remote = email.utils.parsedate_to_datetime(header) + if remote.tzinfo is None: + raise ValueError() + except (ValueError, TypeError, OverflowError): + raise ProbeUnavailable('clock header unavailable') from None + if not 0 <= elapsed <= 2 or abs((end - start).total_seconds() - elapsed) > .1: + raise ProbeUnavailable('clock transport or local clock changed') + return (remote - end).total_seconds(), (remote + dt.timedelta(seconds=1) - start).total_seconds() + + +def check_http_clocks(pods, port, opener=urllib.request.urlopen, + utcnow=lambda: dt.datetime.now(dt.timezone.utc), monotonic=time.monotonic, + sleep=time.sleep, diagnostic=emit): + for pod in pods: + if pod['metadata'].get('deletionTimestamp') or pod['status'].get('phase') != 'Running': + raise ValueError('clock pod unavailable') + ip = pod['status']['podIP'] + def measure(): + intervals = [] + for index in range(2): + if index: + sleep(.55) + start = utcnow(); tick = monotonic() + request = urllib.request.Request('http://' + ip + ':' + str(port) + '/', method='HEAD') + try: + with opener(request, timeout=2) as response: + if response.status >= 500: + raise ValueError('API clock endpoint unhealthy') + header = response.headers.get('Date') + except (OSError, urllib.error.HTTPError) as error: + if isinstance(error, urllib.error.HTTPError) and error.code >= 500: + raise ValueError('API clock endpoint unhealthy') from None + raise ProbeUnavailable('clock transport unavailable') from None + end = utcnow(); elapsed = monotonic() - tick + intervals.append(date_interval(header, start, end, elapsed)) + low = max(x[0] for x in intervals); high = min(x[1] for x in intervals) + passed = -1 <= low <= high <= 1 + diagnostic(dict(event='clock_probe', podUID=pod['metadata']['uid'], + offsetLowerSeconds=low, offsetUpperSeconds=high, passed=passed)) + if not passed: + raise ProbeUnavailable('clock interval outside reviewed bound') + return low, high + retry_probe(measure, 'api-http-clock', diagnostic=diagnostic, sleep=sleep) + + +class ClusterObserver(observer.Observer): + def __init__(self, args, binding, work_binding): + super().__init__(args, binding) + self.work_binding = work_binding + # Service-account tokenFile is read by kubectl on each request (rotation). + self.k += ['--request-timeout=3s'] + + def call(self, *args, data=None, timeout=5): + try: + return super().call(*args, data=data, timeout=min(timeout, 5)) + except (ProbeUnavailable, subprocess.TimeoutExpired, OSError) as error: + # Never include stderr, URLs, argv, credentials or customer log lines. + phase = next((x for x in ('logs', 'exec', 'patch', 'get') if x in args), 'cluster') + emit(dict(event='probe_failure', phase=phase, failureType=type(error).__name__)) + raise + + def run_binary(self, args, env, data=None): + try: + return super().run_binary(args, env, data) + except Exception as error: + emit(dict(event='probe_failure', phase='source-metadata' if args == ['--sample-source'] else 'health-publish', + failureType=type(error).__name__)) + raise + + def source_sample(self): + uri = os.environ['SHADOW_MONITOR_SOURCE_URI'] + parsed = urllib.parse.urlsplit(uri) + endpoint = (parsed.hostname or '') + ':' + str(parsed.port or 27017) + if (parsed.scheme != 'mongodb' or ',' in parsed.netloc or + hashlib.sha256(endpoint.encode()).hexdigest() != self.b['sourceEndpointSHA256']): + raise ValueError('source endpoint changed') + env = self.target_environment('') + env.update(MONGO_URI=uri, MONGO_DB=self.b['sourceDatabase']) + return json.loads(self.run_binary(['--sample-source'], env)) + + def target_environment(self, uri): + # Do not pass both authority and query credentials to either child. + env = {k: v for k, v in os.environ.items() + if not k.startswith(('MONGO_', 'QUERY_', 'SHADOW_MONITOR_'))} + env.update(QUERY_READ_MONGO_URI=uri, QUERY_READ_MONGO_DB=self.b['targetDatabase'], + QUERY_SERVING_NAMESPACE=self.b['servingNamespace']) + return env + + def publish(self, observation): + self.run_binary(['--record-observation', '-'], + self.target_environment(os.environ['SHADOW_MONITOR_TARGET_URI']), json.dumps(observation)) + + def host_filesystem(self): + # /lab is the existing host mount's parent filesystem, /lab/data the 25Gi loop. + fields = self.call('-n', self.b['shadowNamespace'], 'exec', 'query-mongo-0', + '-c', 'mongo', '--', 'stat', '-f', '-c', '%a %S', '/lab').split() + if len(fields) != 2 or any(not x.isdigit() for x in fields) or int(fields[1]) <= 0: + raise ProbeUnavailable('host free-space evidence unavailable') + return dict(availableBytes=int(fields[0]) * int(fields[1])) + + def verify_clocks(self, pods): + return check_http_clocks(pods, self.b['apiHTTPPort']) + + def observation(self, cpu, sampled_at, source=None): + # Check every cycle, not just startup. No timestamp rewriting on retry. + pods = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', + '-l', 'app=droboticflow-api', '-o', 'json'))['items'] + self.verify_clocks(pods) + def collect(): + try: + return work.collect(self.k, self.work_binding, log_workers=4, + caller=lambda _k, *values: self.call(*values)) + except ValueError as error: + if str(error) in ('log clock unavailable', 'log clock in future'): + emit(dict(event='probe_failure', phase='business-log-clock', reason=str(error))) + raise ProbeUnavailable('business log clock unavailable') from None + raise + health = retry_probe(collect, 'business-health') + health['bindingSHA256'] = self.b['workHealthBindingSHA256'] + work.atomic_write(self.args.work_health, health) + return super().observation(cpu, sampled_at, source) + + def stop_shadow(self): + # Deployment body is read-only; only exact two /scale subresources writable. + failed = False + for c in self.b['controllers']: + try: + scale = self.get(self.b['shadowNamespace'], 'deployment/' + c['name'], + '--subresource=scale') + if scale['metadata']['uid'] != c['uid']: + raise ValueError('stop controller replaced') + if scale['spec'].get('replicas', 0) == 0: + continue + patch = [dict(op='test', path='/metadata/uid', value=c['uid']), + dict(op='test', path='/metadata/resourceVersion', value=scale['metadata']['resourceVersion']), + dict(op='replace', path='/spec/replicas', value=0)] + self.call('-n', self.b['shadowNamespace'], 'patch', 'deployment', c['name'], + '--subresource=scale', '--type=json', '-p', json.dumps(patch)) + except Exception as error: + failed = True + emit(dict(event='shadow_stop_failed', controller=c['name'], failureType=type(error).__name__)) + if failed: + raise ValueError('shadow stop incomplete; durable health expiry still applies') + + +def checked_json(path, digest): + raw = Path(path).read_bytes() + if hashlib.sha256(raw).hexdigest() != digest: + raise ValueError('reviewed binding changed') + return json.loads(raw) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for key in ('kubeconfig', 'bindings', 'bindings-sha256', 'work-bindings', 'binary', 'binary-sha256'): + parser.add_argument('--' + key, required=True) + parser.add_argument('--work-health', default='/tmp/work-health.json') + args = parser.parse_args() + args.stop_on_failure = True + binding = checked_json(args.bindings, args.bindings_sha256) + observer.check_bindings(binding) + if type(binding.get('apiHTTPPort')) is not int or not 1 <= binding['apiHTTPPort'] <= 65535: + raise ValueError('reviewed API port required') + work_binding = checked_json(args.work_bindings, binding['workHealthBindingSHA256']) + if (work_binding['sourceNamespace'] != binding['sourceNamespace'] or + work_binding['sourceNamespaceUID'] != binding['sourceNamespaceUID']): + raise ValueError('business-health namespace mismatch') + ready = urllib.parse.urlsplit(binding['readyURL']) + if (ready.scheme != 'http' or ready.hostname != 'droboticflow-api.' + binding['sourceNamespace'] + '.svc' + or ready.port not in (None, 80) or ready.path != '/api/ready' + or ready.username or ready.password or ready.query or ready.fragment): + raise ValueError('reviewed internal readiness endpoint required') + ClusterObserver(args, binding, work_binding).run() + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + raise SystemExit(0) + except Exception as error: + emit(dict(event='monitor_stopped', failureType=type(error).__name__, automaticResume=False)) + raise SystemExit(2) from None diff --git a/scripts/local/query-shadow-observer.py b/scripts/local/query-shadow-observer.py index 22f9e86f..1466b31b 100644 --- a/scripts/local/query-shadow-observer.py +++ b/scripts/local/query-shadow-observer.py @@ -214,13 +214,19 @@ class Observer: except (OSError, ValueError) as error: raise ProbeUnavailable('readiness transport unavailable') from None + def host_filesystem(self): + return json.loads(self.call('get', '--raw', '/api/v1/nodes/10.189.16.4/proxy/stats/summary'))['node']['fs'] + + def verify_clocks(self, pods): + return check_clocks(self.call, self.b['sourceNamespace'], pods) + def observation(self, cpu, sampled_at, source=None): self.verify() ready = retry_probe(self.readiness, 'readiness') ns = self.b['shadowNamespace']; pod = self.get(ns, 'pod', 'query-mongo-0') if pod['metadata']['uid'] != self.b['mongoPodUID']: raise ValueError('query Mongo rebuilt; recovery verification required') - fs = json.loads(self.call('get', '--raw', '/api/v1/nodes/10.189.16.4/proxy/stats/summary'))['node']['fs'] + fs = self.host_filesystem() text = self.call('-n', ns, 'exec', 'query-mongo-0', '-c', 'mongo', '--', 'df', '-P', '-B1', '/lab/data') fields = text.splitlines()[-1].split(); used = int(fields[2]) / int(fields[1]) protected = json.loads(self.call('-n', self.b['sourceNamespace'], 'get', 'pods', '--field-selector=spec.nodeName=10.189.16.4', '-o', 'json'))['items'] @@ -286,7 +292,7 @@ class Observer: try: self.verify() pods=json.loads(self.call('-n',self.b['sourceNamespace'],'get','pods','-l','app=droboticflow-api','-o','json'))['items'] - check_clocks(self.call,self.b['sourceNamespace'],pods) + self.verify_clocks(pods) prior = retry_probe(self.source_sample,'initial-source') except Exception: if self.args.stop_on_failure: self.stop_shadow() diff --git a/scripts/local/query-shadow-work-health.py b/scripts/local/query-shadow-work-health.py index 221f6d16..33f65445 100644 --- a/scripts/local/query-shadow-work-health.py +++ b/scripts/local/query-shadow-work-health.py @@ -14,6 +14,7 @@ from pathlib import Path import re import subprocess import time +from concurrent.futures import ThreadPoolExecutor OPS = ('save', 'slice', 'submit', 'review', 'acceptance') TRACE = re.compile(r'\brequest_trace id=([a-f0-9]{32}) kind=(slow|failed) method=(PUT|POST) path=(/api/\S+) status=(\d{3}) elapsed_ms=(\d+) observations=') @@ -80,27 +81,31 @@ def call(k, *args): if p.returncode: raise RuntimeError('bounded read-only metrics unavailable') return p.stdout -def collect(k, binding, seconds=30, max_lines=4000): +def collect(k, binding, seconds=30, max_lines=4000, caller=None, log_workers=1): + caller = caller or call if not 10 <= seconds <= 30: raise ValueError('invalid observation window') + if not 1 <= log_workers <= 4: raise ValueError('invalid log concurrency') now = dt.datetime.now(dt.timezone.utc); start = now - dt.timedelta(seconds=seconds) ns = binding['sourceNamespace'] - namespace = json.loads(call(k, 'get', 'namespace', ns, '-o', 'json')) + namespace = json.loads(caller(k, 'get', 'namespace', ns, '-o', 'json')) if namespace['metadata']['uid'] != binding['sourceNamespaceUID']: raise ValueError('namespace replaced') - d = json.loads(call(k, '-n', ns, 'get', 'deployment', 'droboticflow-api', '-o', 'json')) + d = json.loads(caller(k, '-n', ns, 'get', 'deployment', 'droboticflow-api', '-o', 'json')) if d['metadata']['uid'] != binding['apiDeploymentUID']: raise ValueError('API replaced') c = d['spec']['template']['spec']['containers'][0] if c['image'] != binding['apiImage'] or d.get('status', {}).get('updatedReplicas') != binding['apiReplicas'] or d.get('status', {}).get('availableReplicas') != binding['apiReplicas']: raise ValueError('API identity or coverage changed') - pods = json.loads(call(k, '-n', ns, 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items'] + pods = json.loads(caller(k, '-n', ns, 'get', 'pods', '-l', 'app=droboticflow-api', '-o', 'json'))['items'] active = [p for p in pods if p['status'].get('phase') not in ('Failed', 'Succeeded')] if {p['metadata']['uid'] for p in active} != set(binding['apiPodUIDs']): raise ValueError('API pod set changed') - logs = [] for p in active: statuses = p['status'].get('containerStatuses', []) if (p['metadata'].get('deletionTimestamp') or p['status'].get('phase') != 'Running' or not statuses or not all(s.get('ready') and s.get('restartCount', 0) == binding['podRestarts'][p['metadata']['uid']] for s in statuses)): raise ValueError('API pod health changed') - logs.append(call(k, '-n', ns, 'logs', p['metadata']['name'], '-c', c['name'], '--timestamps=true', '--since-time=' + start.isoformat(), '--tail=' + str(max_lines))) + def read_log(p): + return caller(k, '-n', ns, 'logs', p['metadata']['name'], '-c', c['name'], '--timestamps=true', '--since-time=' + start.isoformat(), '--tail=' + str(max_lines)) + with ThreadPoolExecutor(max_workers=log_workers) as pool: + logs = list(pool.map(read_log, active)) rows = summarize(logs, binding['tailLimitsMs'], start, now, max_lines) # Mark the start of collection, not publication, so slow reads cannot refresh # evidence which is already stale. Normal requests below1s remain uncounted. diff --git a/scripts/local/test_query_shadow_monitor.py b/scripts/local/test_query_shadow_monitor.py new file mode 100644 index 00000000..7a1af3c0 --- /dev/null +++ b/scripts/local/test_query_shadow_monitor.py @@ -0,0 +1,137 @@ +import datetime as dt +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('monitor', Path(__file__).with_name('query-shadow-monitor.py')) +m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m) +UTC = dt.timezone.utc + + +class MonitorTests(unittest.TestCase): + def make(self): + b = dict(sourceNamespace='authority', shadowNamespace='shadow', sourceDatabase='source', targetDatabase='query', + servingNamespace='serving', controllers=[dict(name='query-replica', uid='p'), dict(name='query-read-builder', uid='b')], + sourceEndpointSHA256=hashlib.sha256(b'private-source:27017').hexdigest()) + return m.ClusterObserver(SimpleNamespace(kubeconfig='unused'), b, {}) + + def test_clock_resolution_and_backwards_step(self): + start = dt.datetime(2026, 11, 15, 5, 0, 0, 400000, tzinfo=UTC) + low, high = m.date_interval('Sun, 15 Nov 2026 05:00:00 GMT', start, start + dt.timedelta(seconds=.1), .1) + self.assertAlmostEqual(low, -.5); self.assertAlmostEqual(high, .6) + for header, end, elapsed in [(None, start, .1), ('bad', start, .1), + ('Sun, 15 Nov 2026 05:00:00 GMT', start - dt.timedelta(seconds=15), .1), + ('Sun, 15 Nov 2026 05:00:00 GMT', start + dt.timedelta(seconds=3), 3)]: + with self.assertRaises(m.ProbeUnavailable): m.date_interval(header, start, end, elapsed) + + def test_http_clock_all_pods_bounded_without_exec(self): + pods = [dict(metadata=dict(uid='pod' + str(i)), status=dict(phase='Running', podIP='10.0.0.' + str(i))) for i in range(1, 5)] + now = dt.datetime(2026, 11, 15, 5, 0, 0, 400000, tzinfo=UTC) + dates = [now + dt.timedelta(seconds=x) for x in (0, .05, .6, .65)] * 4 + response = SimpleNamespace(status=200, headers={'Date': 'Sun, 15 Nov 2026 05:00:00 GMT'}) + class Context: + def __enter__(self): return response + def __exit__(self, *args): pass + calls = [] + def opener(request, timeout): + calls.append((request.get_method(), request.full_url, timeout)); return Context() + m.check_http_clocks(pods, 3001, opener=opener, utcnow=iter(dates).__next__, + monotonic=iter([0, .05, .6, .65] * 4).__next__, sleep=lambda _: None, diagnostic=lambda _: None) + self.assertEqual(len(calls), 8) + self.assertTrue(all(method == 'HEAD' and url.endswith(':3001/') and timeout == 2 for method, url, timeout in calls)) + + def test_clock_failure_is_diagnosed_and_retried_three_times(self): + pod = dict(metadata=dict(uid='pod'), status=dict(phase='Running', podIP='10.0.0.1')) + diagnostics = [] + with self.assertRaises(m.ProbeUnavailable): + m.check_http_clocks([pod], 3001, opener=lambda *a, **k: (_ for _ in ()).throw(OSError('credential must not print')), + sleep=lambda _: None, diagnostic=diagnostics.append) + self.assertEqual(len([x for x in diagnostics if x.get('event') == 'probe_retry']), 3) + self.assertNotIn('credential', json.dumps(diagnostics)) + + def test_source_and_target_credentials_are_separated_no_relay(self): + o = self.make(); calls = [] + o.run_binary = lambda args, env, data=None: calls.append((args, env, data)) or '{}' + o.secret = lambda *a: self.fail('Secret API must never be used') + with patch.dict(os.environ, dict(SHADOW_MONITOR_SOURCE_URI='mongodb://user:password@private-source:27017/source', + SHADOW_MONITOR_TARGET_URI='mongodb://target:secret@query:27129/query', + MONGO_URI='untrusted', QUERY_READ_MONGO_URI='untrusted')): + o.source_sample(); o.publish(dict(ready=True)) + self.assertIn('private-source', calls[0][1]['MONGO_URI']) + self.assertEqual(calls[0][1]['QUERY_READ_MONGO_URI'], '') + self.assertNotIn('MONGO_URI', calls[1][1]) + self.assertTrue(all(not any(k.startswith('SHADOW_MONITOR_') for k in c[1]) for c in calls)) + with patch.dict(os.environ, dict(SHADOW_MONITOR_SOURCE_URI='mongodb://wrong:27017/source')): + with self.assertRaises(ValueError): o.source_sample() + + def test_host_space_uses_parent_mount_not_node_proxy(self): + o = self.make(); calls = [] + o.call = lambda *a, **k: calls.append(a) or '30000000 4096\n' + self.assertEqual(o.host_filesystem()['availableBytes'], 30000000 * 4096) + self.assertEqual(calls[0][-1], '/lab') + self.assertNotIn('proxy', str(calls)) + o.call = lambda *a: 'unknown' + with self.assertRaises(m.ProbeUnavailable): o.host_filesystem() + + def test_stop_only_scale_uid_rv_cas_and_restart_does_not_resume(self): + o = self.make(); calls = [] + o.get = lambda ns, kind, name: dict(metadata=dict(uid='p' if kind.endswith('query-replica') else 'b', resourceVersion='7'), spec=dict(replicas=1)) + o.call = lambda *a, **k: calls.append(a) or '' + o.stop_shadow() + self.assertEqual(len(calls), 2) + for call in calls: + self.assertIn('--subresource=scale', call) + payload = json.loads(call[-1]) + self.assertEqual([(x['op'], x['path']) for x in payload], [('test', '/metadata/uid'), ('test', '/metadata/resourceVersion'), ('replace', '/spec/replicas')]) + self.assertEqual(payload[-1]['value'], 0) + other = self.make() + self.assertEqual(other.high, 0) # Constructor performs no Kubernetes/database writes. + other.get = lambda *a: dict(metadata=dict(uid='replaced'), spec=dict(replicas=1)) + with self.assertRaises(ValueError): other.stop_shadow() + + def test_parallel_business_logs_require_complete_coverage(self): + b = dict(sourceNamespace='authority', sourceNamespaceUID='ns', apiDeploymentUID='api', apiImage='image', apiReplicas=4, + apiPodUIDs=['p'+str(i) for i in range(4)], podRestarts={'p'+str(i): 0 for i in range(4)}, + tailLimitsMs=dict(save=5000, slice=10000, submit=15000, review=5000, acceptance=15000)) + pods = [dict(metadata=dict(uid='p'+str(i), name='api-'+str(i)), status=dict(phase='Running', containerStatuses=[dict(ready=True, restartCount=0)])) for i in range(4)] + deployment = dict(metadata=dict(uid='api'), spec=dict(template=dict(spec=dict(containers=[dict(name='api', image='image')]))), status=dict(updatedReplicas=4, availableReplicas=4)) + logs = [] + def call(k, *a): + if 'namespace' in a: return json.dumps(dict(metadata=dict(uid='ns'))) + if 'deployment' in a: return json.dumps(deployment) + if 'pods' in a: return json.dumps(dict(items=pods)) + logs.append(a[a.index('logs')+1]); return '' + result = m.work.collect([], b, caller=call, log_workers=4) + self.assertEqual(set(logs), {'api-'+str(i) for i in range(4)}) + self.assertEqual(result['monitoredPods'], 4) + def denied(k, *a): + if 'api-3' in a: raise m.ProbeUnavailable('denied') + return call(k, *a) + with self.assertRaises(m.ProbeUnavailable): m.work.collect([], b, caller=denied, log_workers=4) + + def test_stop_failure_still_attempts_other_controller(self): + o = self.make(); calls = [] + def get(ns, kind, name): + calls.append(kind) + if kind.endswith('query-replica'): raise OSError('private error') + return dict(metadata=dict(uid='b', resourceVersion='8'), spec=dict(replicas=0)) + o.get = get + with patch.object(m, 'emit') as diagnostic: + with self.assertRaises(ValueError): o.stop_shadow() + self.assertEqual(len(calls), 2) + self.assertNotIn('private error', str(diagnostic.call_args)) + + def test_scale_omits_zero_replicas(self): + o = self.make() + o.get = lambda ns, kind, name: dict(metadata=dict(uid='p' if kind.endswith('query-replica') else 'b'), spec={}) + o.call = lambda *a, **k: self.fail('already stopped Scale must not be patched') + o.stop_shadow() + + +if __name__ == '__main__': unittest.main() diff --git a/scripts/verification-scopes.json b/scripts/verification-scopes.json index 6fb9081c..ffaf6b73 100644 --- a/scripts/verification-scopes.json +++ b/scripts/verification-scopes.json @@ -1018,27 +1018,17 @@ { "name": "shadow-work-health", "paths": [ - "^scripts/local/(query-shadow-(observer|work-health)|query_shadow_health|test_query_shadow_(observer|work_health|health))\\.py$" + "^scripts/local/(query-shadow-(observer|work-health|monitor)|query_shadow_health|test_query_shadow_(observer|work_health|health|monitor))\\.py$", + "^Dockerfile\\.query-shadow-monitor$", + "^docs/operations/query-shadow-monitor\\.md$" ], "frontend": [], "go": { - "./internal/api": [ - "^TestDiagnosticTrace" - ] + "./internal/api": ["^TestDiagnosticTrace"] }, "task": [ { - "argv": [ - "python3", - "-m", - "unittest", - "discover", - "-s", - "scripts/local", - "-p", - "test_query_shadow*.py", - "-v" - ], + "argv": ["python3", "-m", "unittest", "discover", "-s", "scripts/local", "-p", "test_query_shadow*.py", "-v"], "tests": true } ],