forked from longtao.wu/droboticflow
🛡️ Bind shadow health checks to each deployed writer revision
This commit is contained in:
parent
88992d7e61
commit
7a0201dbca
@ -1,5 +1,15 @@
|
||||
# 集群内影子观察器
|
||||
|
||||
## 2026-11-16 恢复准备:逐写者版本绑定
|
||||
|
||||
当前 API/切片处理器为 `88992d7e`,其他写者保留各自已发布版本;这不要求将五个组件一起部署。旧观察器用单个 `writerCommit` 校验所有组件,会拒绝这一合法组合。
|
||||
|
||||
新增可选的 `writers[].commit`,逐组件绑定已核实的提交号、不可变镜像、UID、副本数及同步标记;省略时沿用原 `writerCommit`,兼容已有配置。缺失/空提交号、任一组件版本漂移或同步标记丢失仍拒绝启动。不通过关闭观察器、伪造构建标记或更新其他生产组件解决。
|
||||
|
||||
恢复前须单独确认新观察器镜像及非机密绑定更新。当前已完成 22 集合历史断点不重置;沿用原 run、读取累计预算、majority+journal 和 25Gi 边界。维护期限过期不能自动恢复,必须明确新期限。原生产入口保持关闭,影子成品完整发布且真实结果/权限对照通过后,另列切换和回退方案。部署准备、启动、切读是三个独立授权范围。
|
||||
|
||||
本地回归覆盖混合版本、逐组件版本漂移、旧单版本配置和缺失绑定;线上新旧结果对照仍须在增量追平后执行,不能用本地通过替代。
|
||||
|
||||
状态:本地实现与验证,新增组件尚未部署。生产入口保持关闭,不代表分流或原 Mongo 降载已完成。
|
||||
|
||||
## 原因与兼容范围
|
||||
|
||||
@ -87,7 +87,11 @@ class Observer:
|
||||
for w in self.b['writers']:
|
||||
d = self.get(self.b['sourceNamespace'], 'deployment', w['name'])
|
||||
env = {x['name']: x.get('value') for x in d['spec']['template']['spec']['containers'][0].get('env', [])}
|
||||
if (d['metadata']['uid'] != w['uid'] or env.get('DROBOTICFLOW_BUILD_COMMIT') != self.b['writerCommit']
|
||||
# Independently released writers may legitimately have different
|
||||
# commits. Bind each one explicitly; retain legacy uniform bindings.
|
||||
commit = w.get('commit', self.b.get('writerCommit'))
|
||||
if (not isinstance(commit, str) or not commit
|
||||
or d['metadata']['uid'] != w['uid'] or env.get('DROBOTICFLOW_BUILD_COMMIT') != commit
|
||||
or env.get('QUERY_SOURCE_TRACKING') != '1'
|
||||
or d['spec']['template']['spec']['containers'][0]['image'] != w['image']
|
||||
or d.get('status', {}).get('availableReplicas', 0) != w['replicas']):
|
||||
|
||||
@ -10,6 +10,32 @@ spec = importlib.util.spec_from_file_location('shadow_observer', Path(__file__).
|
||||
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
|
||||
|
||||
class ObserverTests(unittest.TestCase):
|
||||
def test_mixed_writer_versions_are_bound_individually(self):
|
||||
b = self.bindings()
|
||||
b.update(sourceNamespaceUID='authority-uid', shadowNamespaceUID='shadow-uid', writerCommit='old')
|
||||
b['writers'] = [dict(name='writer-' + str(i), uid=str(i), image='image-' + str(i), replicas=1,
|
||||
commit='new' if i < 2 else 'old') for i in range(5)]
|
||||
deployments = {w['name']: dict(metadata=dict(uid=w['uid']),
|
||||
spec=dict(template=dict(spec=dict(containers=[dict(image=w['image'], env=[
|
||||
dict(name='DROBOTICFLOW_BUILD_COMMIT', value=w['commit']),
|
||||
dict(name='QUERY_SOURCE_TRACKING', value='1')])]))),
|
||||
status=dict(availableReplicas=1)) for w in b['writers']}
|
||||
o = m.Observer(SimpleNamespace(kubeconfig='unused'), b)
|
||||
o.get = lambda ns, kind, name: dict(metadata=dict(uid=ns + '-uid')) if kind == 'namespace' else deployments[name]
|
||||
o.verify()
|
||||
for i, writer in enumerate(b['writers']):
|
||||
with self.subTest(writer=i):
|
||||
env = deployments[writer['name']]['spec']['template']['spec']['containers'][0]['env']
|
||||
env[0]['value'] = 'unexpected'
|
||||
with self.assertRaises(ValueError): o.verify()
|
||||
env[0]['value'] = writer['commit']
|
||||
for writer in b['writers']:
|
||||
writer.pop('commit')
|
||||
deployments[writer['name']]['spec']['template']['spec']['containers'][0]['env'][0]['value'] = 'old'
|
||||
o.verify() # Existing single-version ConfigMaps remain compatible.
|
||||
b['writerCommit'] = ''
|
||||
with self.assertRaises(ValueError): o.verify()
|
||||
|
||||
def bindings(self):
|
||||
return dict(sourceNamespace='authority', shadowNamespace='shadow', sourceDatabase='source', targetDatabase='query',
|
||||
baselineCPU=85, writers=[{} for _ in range(5)], controllers=[dict(name='query-replica', uid='p'), dict(name='query-read-builder', uid='b')])
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user