🛡️ Gate API release on processor rollout readiness

This commit is contained in:
d-robotics 2026-10-15 18:28:29 +08:00
parent aa729a62c9
commit d8a8530cda
4 changed files with 74 additions and 5 deletions

View File

@ -109,3 +109,15 @@ rollback must first disable retry entry and finish/reconcile outstanding jobs.
Production index installation needs its separately approved plan. Source-media
failures in TEST-data remain outside this repair and are not made recoverable by
retrying an invalid source.
## Comments: release preparation
The user subsequently requested deployment and online testing. Release preparation
found that the existing tool patched API before processor, conflicting with the
retry payload compatibility requirement above. The release tool now patches and
fully verifies the processor rollout before patching API. Timeout/non-convergence
preserves the known partial outcome and leaves API untouched. Three fake-cluster
regressions cover ordering, wait timeout and unsuccessful convergence; the full
release-tool test suite passes. This changes release tooling only; production
application behavior remains the aa729a62 repair. A concrete final plan still
requires approval under docs/release/README.md before its first cluster write.

View File

@ -106,6 +106,10 @@ python3 scripts/release.py deploy <release-id> \
核对实际状态再制定恢复计划。JSON 条件更新防止覆盖同一资源的并发修改,但不能保证跨资源
原子性;正式部署应使用团队约定的维护窗口,后续可增加服务端发布锁。
处理器先更新,完整 rollout 并核对候选镜像与构建标记后才更新 API,避免旧处理器接到
无法理解的新持久任务载荷。等待失败保留已确认的处理器 patch,API 不更新;按部分发布
处理,先只读 verify,不重复 deploy、不自动回退。计划展示此固定顺序。
完成条件:仅代表 patch 已确认,状态仍为 `deployed_pending_verification`。
### 5. Verify:分开基础设施与业务验收

View File

@ -134,10 +134,10 @@ def fresh(r):
require(digest(policy()) == r['policyHash'], 'Release policy changed; prepare a new release.')
def kubectl(r, args, *, stdin=None):
def kubectl(r, args, *, stdin=None, timeout=60):
t = r['target']
return run(['kubectl', '--kubeconfig', t['kubeconfig'], '--context', t['context'],
'--request-timeout=20s', '-n', t['namespace']] + args, stdin=stdin, timeout=60)
'--request-timeout=20s', '-n', t['namespace']] + args, stdin=stdin, timeout=timeout)
def get(r, kind, name):
@ -340,7 +340,8 @@ def make_plan(r):
r['state'] = 'awaiting_approval'
event(r, 'plan prepared; no authorization inferred')
print(json.dumps({'planHash': r['planHash'], 'target': r['target'], 'changes': [
{'deployment': n, 'from': d['image'], 'to': r['image'], 'buildCommit': r['commit']} for n, d in r['baseline']['deployments'].items()],
{'deployment': n, 'from': r['baseline']['deployments'][n]['image'], 'to': r['image'], 'buildCommit': r['commit']} for n in sorted(r['baseline']['deployments'])],
'sequence': 'Processor first; verify its complete rollout before patching API.',
'bandwidthMbps': r['baseline']['publicService']['bandwidthMbps'],
'warnings': ['Only image/build stamp changes; startup migrations require change-safety evidence.',
'Image-to-source provenance requires independent evidence.',
@ -363,7 +364,9 @@ def deploy(r, a):
r['approval'] = {'reference': a.approval_ref, 'at': now(), 'planHash': r['planHash']}
r['state'] = 'deploying'
event(r, 'explicit approval reference recorded; deployment starting')
for name in current['deployments']:
# The processor must understand new durable payloads before API replicas can
# enqueue them. The fixed Fanyu names sort processor before API.
for name in sorted(current['deployments']):
latest = snapshot(r)
require(all(latest[k] == current[k] for k in ('configs', 'pvc', 'publicService')), 'Configuration changed during deployment; inspect partial outcome.')
for n, live in latest['deployments'].items():
@ -391,6 +394,17 @@ def deploy(r, a):
raise
op['result'] = 'acknowledged'
save(r)
if policy()['deployments'][name] == 'processor':
event(r, 'processor patch acknowledged; waiting for rollout before API')
try:
kubectl(r, ['rollout', 'status', 'deployment/' + name, '--timeout=300s'], timeout=330)
ready = snapshot(r)['deployments'][name]
require(ready['ready'] and ready['image'] == r['image'] and ready['buildCommit'] == r['commit'],
'Processor rollout has not converged to the candidate.')
except Blocked:
r['state'] = 'deployed_pending_verification'
event(r, 'processor rollout not verified; API was not patched; inspect partial outcome')
raise
r['state'] = 'deployed_pending_verification'
event(r, 'image patches acknowledged; rollout and acceptance not yet verified')

View File

@ -57,6 +57,8 @@ class ReleaseTests(unittest.TestCase):
self.ds[n], config = fixture(n, c)
self.configs = {'config': config}
self.writes = []
self.rollout_waits = []
self.fail_rollout = False
self.fail_patch = None
self.before_snapshot = None
self.start_patch('snapshot', self.snapshot)
@ -81,7 +83,12 @@ class ReleaseTests(unittest.TestCase):
'configs': {'config': {'uid': 'config-uid', 'hash': m.digest(self.configs)}},
'pvc': {'uid': 'pvc-uid'}, 'publicService': {'bandwidthMbps': '5'}}
def kubectl(self, r, args, *, stdin=None):
def kubectl(self, r, args, *, stdin=None, timeout=60):
if args[:2] == ['rollout', 'status']:
self.rollout_waits.append((args[2], list(self.writes)))
if self.fail_rollout:
raise m.Blocked('simulated rollout timeout')
return 'successfully rolled out'
self.assertEqual(args[:2], ['patch', 'deployment'])
name = args[2]
self.writes.append(name)
@ -281,6 +288,38 @@ class ReleaseTests(unittest.TestCase):
m.verify(self.r)
self.assertEqual(self.r['state'], 'deployed_pending_acceptance')
def test_processor_rollout_precedes_api_patch(self):
self.plan()
m.deploy(self.r, self.approve())
self.assertEqual(self.writes, ['droboticflow-annotation-processor', 'droboticflow-api'])
self.assertEqual(self.rollout_waits, [
('deployment/droboticflow-annotation-processor', ['droboticflow-annotation-processor'])])
def test_processor_rollout_timeout_leaves_api_untouched(self):
self.plan()
a = self.approve()
self.fail_rollout = True
with self.assertRaises(m.Blocked):
m.deploy(self.r, a)
self.assertEqual(self.writes, ['droboticflow-annotation-processor'])
saved = self.load()
self.assertEqual(saved['state'], 'deployed_pending_verification')
self.assertEqual([o['result'] for o in saved['operations']], ['acknowledged'])
with self.assertRaises(m.Blocked):
m.deploy(saved, a)
def test_processor_not_converged_leaves_api_untouched(self):
self.plan()
a = self.approve()
def not_ready():
if self.writes:
self.ds['droboticflow-annotation-processor']['status']['readyReplicas'] = 0
self.before_snapshot = not_ready
with self.assertRaises(m.Blocked):
m.deploy(self.r, a)
self.assertEqual(self.writes, ['droboticflow-annotation-processor'])
self.assertEqual(self.load()['state'], 'deployed_pending_verification')
def test_no_predeployment_business_evidence(self):
self.plan()
with self.assertRaises(m.Blocked):