From 7a0201dbca888a8a6985f1e37d5e784daaeea98a Mon Sep 17 00:00:00 2001 From: d-robotics Date: Mon, 16 Nov 2026 17:48:17 -0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20Bind=20shadow=20health?= =?UTF-8?q?=20checks=20to=20each=20deployed=20writer=20revision?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/operations/query-shadow-monitor.md | 10 ++++++++ scripts/local/query-shadow-observer.py | 6 ++++- scripts/local/test_query_shadow_observer.py | 26 +++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/docs/operations/query-shadow-monitor.md b/docs/operations/query-shadow-monitor.md index b98ca531..33c66c33 100644 --- a/docs/operations/query-shadow-monitor.md +++ b/docs/operations/query-shadow-monitor.md @@ -1,5 +1,15 @@ # 集群内影子观察器 +## 2026-11-16 恢复准备:逐写者版本绑定 + +当前 API/切片处理器为 `88992d7e`,其他写者保留各自已发布版本;这不要求将五个组件一起部署。旧观察器用单个 `writerCommit` 校验所有组件,会拒绝这一合法组合。 + +新增可选的 `writers[].commit`,逐组件绑定已核实的提交号、不可变镜像、UID、副本数及同步标记;省略时沿用原 `writerCommit`,兼容已有配置。缺失/空提交号、任一组件版本漂移或同步标记丢失仍拒绝启动。不通过关闭观察器、伪造构建标记或更新其他生产组件解决。 + +恢复前须单独确认新观察器镜像及非机密绑定更新。当前已完成 22 集合历史断点不重置;沿用原 run、读取累计预算、majority+journal 和 25Gi 边界。维护期限过期不能自动恢复,必须明确新期限。原生产入口保持关闭,影子成品完整发布且真实结果/权限对照通过后,另列切换和回退方案。部署准备、启动、切读是三个独立授权范围。 + +本地回归覆盖混合版本、逐组件版本漂移、旧单版本配置和缺失绑定;线上新旧结果对照仍须在增量追平后执行,不能用本地通过替代。 + 状态:本地实现与验证,新增组件尚未部署。生产入口保持关闭,不代表分流或原 Mongo 降载已完成。 ## 原因与兼容范围 diff --git a/scripts/local/query-shadow-observer.py b/scripts/local/query-shadow-observer.py index 1466b31b..cfd0092e 100644 --- a/scripts/local/query-shadow-observer.py +++ b/scripts/local/query-shadow-observer.py @@ -87,7 +87,11 @@ class Observer: for w in self.b['writers']: d = self.get(self.b['sourceNamespace'], 'deployment', w['name']) env = {x['name']: x.get('value') for x in d['spec']['template']['spec']['containers'][0].get('env', [])} - if (d['metadata']['uid'] != w['uid'] or env.get('DROBOTICFLOW_BUILD_COMMIT') != self.b['writerCommit'] + # Independently released writers may legitimately have different + # commits. Bind each one explicitly; retain legacy uniform bindings. + commit = w.get('commit', self.b.get('writerCommit')) + if (not isinstance(commit, str) or not commit + or d['metadata']['uid'] != w['uid'] or env.get('DROBOTICFLOW_BUILD_COMMIT') != commit or env.get('QUERY_SOURCE_TRACKING') != '1' or d['spec']['template']['spec']['containers'][0]['image'] != w['image'] or d.get('status', {}).get('availableReplicas', 0) != w['replicas']): diff --git a/scripts/local/test_query_shadow_observer.py b/scripts/local/test_query_shadow_observer.py index 17fb5f4a..513d0786 100644 --- a/scripts/local/test_query_shadow_observer.py +++ b/scripts/local/test_query_shadow_observer.py @@ -10,6 +10,32 @@ spec = importlib.util.spec_from_file_location('shadow_observer', Path(__file__). m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m) class ObserverTests(unittest.TestCase): + def test_mixed_writer_versions_are_bound_individually(self): + b = self.bindings() + b.update(sourceNamespaceUID='authority-uid', shadowNamespaceUID='shadow-uid', writerCommit='old') + b['writers'] = [dict(name='writer-' + str(i), uid=str(i), image='image-' + str(i), replicas=1, + commit='new' if i < 2 else 'old') for i in range(5)] + deployments = {w['name']: dict(metadata=dict(uid=w['uid']), + spec=dict(template=dict(spec=dict(containers=[dict(image=w['image'], env=[ + dict(name='DROBOTICFLOW_BUILD_COMMIT', value=w['commit']), + dict(name='QUERY_SOURCE_TRACKING', value='1')])]))), + status=dict(availableReplicas=1)) for w in b['writers']} + o = m.Observer(SimpleNamespace(kubeconfig='unused'), b) + o.get = lambda ns, kind, name: dict(metadata=dict(uid=ns + '-uid')) if kind == 'namespace' else deployments[name] + o.verify() + for i, writer in enumerate(b['writers']): + with self.subTest(writer=i): + env = deployments[writer['name']]['spec']['template']['spec']['containers'][0]['env'] + env[0]['value'] = 'unexpected' + with self.assertRaises(ValueError): o.verify() + env[0]['value'] = writer['commit'] + for writer in b['writers']: + writer.pop('commit') + deployments[writer['name']]['spec']['template']['spec']['containers'][0]['env'][0]['value'] = 'old' + o.verify() # Existing single-version ConfigMaps remain compatible. + b['writerCommit'] = '' + with self.assertRaises(ValueError): o.verify() + def bindings(self): return dict(sourceNamespace='authority', shadowNamespace='shadow', sourceDatabase='source', targetDatabase='query', baselineCPU=85, writers=[{} for _ in range(5)], controllers=[dict(name='query-replica', uid='p'), dict(name='query-read-builder', uid='b')])