🛡️ Gate API release on processor rollout readiness
This commit is contained in:
parent
aa729a62c9
commit
d8a8530cda
@ -109,3 +109,15 @@ rollback must first disable retry entry and finish/reconcile outstanding jobs.
|
||||
Production index installation needs its separately approved plan. Source-media
|
||||
failures in TEST-data remain outside this repair and are not made recoverable by
|
||||
retrying an invalid source.
|
||||
|
||||
## Comments: release preparation
|
||||
|
||||
The user subsequently requested deployment and online testing. Release preparation
|
||||
found that the existing tool patched API before processor, conflicting with the
|
||||
retry payload compatibility requirement above. The release tool now patches and
|
||||
fully verifies the processor rollout before patching API. Timeout/non-convergence
|
||||
preserves the known partial outcome and leaves API untouched. Three fake-cluster
|
||||
regressions cover ordering, wait timeout and unsuccessful convergence; the full
|
||||
release-tool test suite passes. This changes release tooling only; production
|
||||
application behavior remains the aa729a62 repair. A concrete final plan still
|
||||
requires approval under docs/release/README.md before its first cluster write.
|
||||
|
||||
@ -106,6 +106,10 @@ python3 scripts/release.py deploy <release-id> \
|
||||
核对实际状态再制定恢复计划。JSON 条件更新防止覆盖同一资源的并发修改,但不能保证跨资源
|
||||
原子性;正式部署应使用团队约定的维护窗口,后续可增加服务端发布锁。
|
||||
|
||||
处理器先更新,完整 rollout 并核对候选镜像与构建标记后才更新 API,避免旧处理器接到
|
||||
无法理解的新持久任务载荷。等待失败保留已确认的处理器 patch,API 不更新;按部分发布
|
||||
处理,先只读 verify,不重复 deploy、不自动回退。计划展示此固定顺序。
|
||||
|
||||
完成条件:仅代表 patch 已确认,状态仍为 `deployed_pending_verification`。
|
||||
|
||||
### 5. Verify:分开基础设施与业务验收
|
||||
|
||||
@ -134,10 +134,10 @@ def fresh(r):
|
||||
require(digest(policy()) == r['policyHash'], 'Release policy changed; prepare a new release.')
|
||||
|
||||
|
||||
def kubectl(r, args, *, stdin=None):
|
||||
def kubectl(r, args, *, stdin=None, timeout=60):
|
||||
t = r['target']
|
||||
return run(['kubectl', '--kubeconfig', t['kubeconfig'], '--context', t['context'],
|
||||
'--request-timeout=20s', '-n', t['namespace']] + args, stdin=stdin, timeout=60)
|
||||
'--request-timeout=20s', '-n', t['namespace']] + args, stdin=stdin, timeout=timeout)
|
||||
|
||||
|
||||
def get(r, kind, name):
|
||||
@ -340,7 +340,8 @@ def make_plan(r):
|
||||
r['state'] = 'awaiting_approval'
|
||||
event(r, 'plan prepared; no authorization inferred')
|
||||
print(json.dumps({'planHash': r['planHash'], 'target': r['target'], 'changes': [
|
||||
{'deployment': n, 'from': d['image'], 'to': r['image'], 'buildCommit': r['commit']} for n, d in r['baseline']['deployments'].items()],
|
||||
{'deployment': n, 'from': r['baseline']['deployments'][n]['image'], 'to': r['image'], 'buildCommit': r['commit']} for n in sorted(r['baseline']['deployments'])],
|
||||
'sequence': 'Processor first; verify its complete rollout before patching API.',
|
||||
'bandwidthMbps': r['baseline']['publicService']['bandwidthMbps'],
|
||||
'warnings': ['Only image/build stamp changes; startup migrations require change-safety evidence.',
|
||||
'Image-to-source provenance requires independent evidence.',
|
||||
@ -363,7 +364,9 @@ def deploy(r, a):
|
||||
r['approval'] = {'reference': a.approval_ref, 'at': now(), 'planHash': r['planHash']}
|
||||
r['state'] = 'deploying'
|
||||
event(r, 'explicit approval reference recorded; deployment starting')
|
||||
for name in current['deployments']:
|
||||
# The processor must understand new durable payloads before API replicas can
|
||||
# enqueue them. The fixed Fanyu names sort processor before API.
|
||||
for name in sorted(current['deployments']):
|
||||
latest = snapshot(r)
|
||||
require(all(latest[k] == current[k] for k in ('configs', 'pvc', 'publicService')), 'Configuration changed during deployment; inspect partial outcome.')
|
||||
for n, live in latest['deployments'].items():
|
||||
@ -391,6 +394,17 @@ def deploy(r, a):
|
||||
raise
|
||||
op['result'] = 'acknowledged'
|
||||
save(r)
|
||||
if policy()['deployments'][name] == 'processor':
|
||||
event(r, 'processor patch acknowledged; waiting for rollout before API')
|
||||
try:
|
||||
kubectl(r, ['rollout', 'status', 'deployment/' + name, '--timeout=300s'], timeout=330)
|
||||
ready = snapshot(r)['deployments'][name]
|
||||
require(ready['ready'] and ready['image'] == r['image'] and ready['buildCommit'] == r['commit'],
|
||||
'Processor rollout has not converged to the candidate.')
|
||||
except Blocked:
|
||||
r['state'] = 'deployed_pending_verification'
|
||||
event(r, 'processor rollout not verified; API was not patched; inspect partial outcome')
|
||||
raise
|
||||
r['state'] = 'deployed_pending_verification'
|
||||
event(r, 'image patches acknowledged; rollout and acceptance not yet verified')
|
||||
|
||||
|
||||
@ -57,6 +57,8 @@ class ReleaseTests(unittest.TestCase):
|
||||
self.ds[n], config = fixture(n, c)
|
||||
self.configs = {'config': config}
|
||||
self.writes = []
|
||||
self.rollout_waits = []
|
||||
self.fail_rollout = False
|
||||
self.fail_patch = None
|
||||
self.before_snapshot = None
|
||||
self.start_patch('snapshot', self.snapshot)
|
||||
@ -81,7 +83,12 @@ class ReleaseTests(unittest.TestCase):
|
||||
'configs': {'config': {'uid': 'config-uid', 'hash': m.digest(self.configs)}},
|
||||
'pvc': {'uid': 'pvc-uid'}, 'publicService': {'bandwidthMbps': '5'}}
|
||||
|
||||
def kubectl(self, r, args, *, stdin=None):
|
||||
def kubectl(self, r, args, *, stdin=None, timeout=60):
|
||||
if args[:2] == ['rollout', 'status']:
|
||||
self.rollout_waits.append((args[2], list(self.writes)))
|
||||
if self.fail_rollout:
|
||||
raise m.Blocked('simulated rollout timeout')
|
||||
return 'successfully rolled out'
|
||||
self.assertEqual(args[:2], ['patch', 'deployment'])
|
||||
name = args[2]
|
||||
self.writes.append(name)
|
||||
@ -281,6 +288,38 @@ class ReleaseTests(unittest.TestCase):
|
||||
m.verify(self.r)
|
||||
self.assertEqual(self.r['state'], 'deployed_pending_acceptance')
|
||||
|
||||
def test_processor_rollout_precedes_api_patch(self):
|
||||
self.plan()
|
||||
m.deploy(self.r, self.approve())
|
||||
self.assertEqual(self.writes, ['droboticflow-annotation-processor', 'droboticflow-api'])
|
||||
self.assertEqual(self.rollout_waits, [
|
||||
('deployment/droboticflow-annotation-processor', ['droboticflow-annotation-processor'])])
|
||||
|
||||
def test_processor_rollout_timeout_leaves_api_untouched(self):
|
||||
self.plan()
|
||||
a = self.approve()
|
||||
self.fail_rollout = True
|
||||
with self.assertRaises(m.Blocked):
|
||||
m.deploy(self.r, a)
|
||||
self.assertEqual(self.writes, ['droboticflow-annotation-processor'])
|
||||
saved = self.load()
|
||||
self.assertEqual(saved['state'], 'deployed_pending_verification')
|
||||
self.assertEqual([o['result'] for o in saved['operations']], ['acknowledged'])
|
||||
with self.assertRaises(m.Blocked):
|
||||
m.deploy(saved, a)
|
||||
|
||||
def test_processor_not_converged_leaves_api_untouched(self):
|
||||
self.plan()
|
||||
a = self.approve()
|
||||
def not_ready():
|
||||
if self.writes:
|
||||
self.ds['droboticflow-annotation-processor']['status']['readyReplicas'] = 0
|
||||
self.before_snapshot = not_ready
|
||||
with self.assertRaises(m.Blocked):
|
||||
m.deploy(self.r, a)
|
||||
self.assertEqual(self.writes, ['droboticflow-annotation-processor'])
|
||||
self.assertEqual(self.load()['state'], 'deployed_pending_verification')
|
||||
|
||||
def test_no_predeployment_business_evidence(self):
|
||||
self.plan()
|
||||
with self.assertRaises(m.Blocked):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user