🛡️ Bind shadow health checks to each deployed writer revision

This commit is contained in:
d-robotics 2026-11-16 17:48:17 -08:00
parent 88992d7e61
commit 7a0201dbca
3 changed files with 41 additions and 1 deletions

View File

@ -1,5 +1,15 @@
# 集群内影子观察器
## 2026-11-16 恢复准备:逐写者版本绑定
当前 API/切片处理器为 `88992d7e`,其他写者保留各自已发布版本;这不要求将五个组件一起部署。旧观察器用单个 `writerCommit` 校验所有组件,会拒绝这一合法组合。
新增可选的 `writers[].commit`,逐组件绑定已核实的提交号、不可变镜像、UID、副本数及同步标记;省略时沿用原 `writerCommit`,兼容已有配置。缺失/空提交号、任一组件版本漂移或同步标记丢失仍拒绝启动。不通过关闭观察器、伪造构建标记或更新其他生产组件解决。
恢复前须单独确认新观察器镜像及非机密绑定更新。当前已完成 22 集合历史断点不重置;沿用原 run、读取累计预算、majority+journal 和 25Gi 边界。维护期限过期不能自动恢复,必须明确新期限。原生产入口保持关闭,影子成品完整发布且真实结果/权限对照通过后,另列切换和回退方案。部署准备、启动、切读是三个独立授权范围。
本地回归覆盖混合版本、逐组件版本漂移、旧单版本配置和缺失绑定;线上新旧结果对照仍须在增量追平后执行,不能用本地通过替代。
状态:本地实现与验证,新增组件尚未部署。生产入口保持关闭,不代表分流或原 Mongo 降载已完成。
## 原因与兼容范围

View File

@ -87,7 +87,11 @@ class Observer:
for w in self.b['writers']:
d = self.get(self.b['sourceNamespace'], 'deployment', w['name'])
env = {x['name']: x.get('value') for x in d['spec']['template']['spec']['containers'][0].get('env', [])}
if (d['metadata']['uid'] != w['uid'] or env.get('DROBOTICFLOW_BUILD_COMMIT') != self.b['writerCommit']
# Independently released writers may legitimately have different
# commits. Bind each one explicitly; retain legacy uniform bindings.
commit = w.get('commit', self.b.get('writerCommit'))
if (not isinstance(commit, str) or not commit
or d['metadata']['uid'] != w['uid'] or env.get('DROBOTICFLOW_BUILD_COMMIT') != commit
or env.get('QUERY_SOURCE_TRACKING') != '1'
or d['spec']['template']['spec']['containers'][0]['image'] != w['image']
or d.get('status', {}).get('availableReplicas', 0) != w['replicas']):

View File

@ -10,6 +10,32 @@ spec = importlib.util.spec_from_file_location('shadow_observer', Path(__file__).
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
class ObserverTests(unittest.TestCase):
def test_mixed_writer_versions_are_bound_individually(self):
b = self.bindings()
b.update(sourceNamespaceUID='authority-uid', shadowNamespaceUID='shadow-uid', writerCommit='old')
b['writers'] = [dict(name='writer-' + str(i), uid=str(i), image='image-' + str(i), replicas=1,
commit='new' if i < 2 else 'old') for i in range(5)]
deployments = {w['name']: dict(metadata=dict(uid=w['uid']),
spec=dict(template=dict(spec=dict(containers=[dict(image=w['image'], env=[
dict(name='DROBOTICFLOW_BUILD_COMMIT', value=w['commit']),
dict(name='QUERY_SOURCE_TRACKING', value='1')])]))),
status=dict(availableReplicas=1)) for w in b['writers']}
o = m.Observer(SimpleNamespace(kubeconfig='unused'), b)
o.get = lambda ns, kind, name: dict(metadata=dict(uid=ns + '-uid')) if kind == 'namespace' else deployments[name]
o.verify()
for i, writer in enumerate(b['writers']):
with self.subTest(writer=i):
env = deployments[writer['name']]['spec']['template']['spec']['containers'][0]['env']
env[0]['value'] = 'unexpected'
with self.assertRaises(ValueError): o.verify()
env[0]['value'] = writer['commit']
for writer in b['writers']:
writer.pop('commit')
deployments[writer['name']]['spec']['template']['spec']['containers'][0]['env'][0]['value'] = 'old'
o.verify() # Existing single-version ConfigMaps remain compatible.
b['writerCommit'] = ''
with self.assertRaises(ValueError): o.verify()
def bindings(self):
return dict(sourceNamespace='authority', shadowNamespace='shadow', sourceDatabase='source', targetDatabase='query',
baselineCPU=85, writers=[{} for _ in range(5)], controllers=[dict(name='query-replica', uid='p'), dict(name='query-read-builder', uid='b')])