From d8a8530cda77b380698c3f5bbc7319b2fd436fb9 Mon Sep 17 00:00:00 2001 From: d-robotics Date: Thu, 15 Oct 2026 18:28:29 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20Gate=20API=20release=20?= =?UTF-8?q?on=20processor=20rollout=20readiness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../workspace-reliability-20261015/spec.md | 12 ++++++ docs/release/README.md | 4 ++ scripts/release.py | 22 ++++++++-- tests/release/test_release.py | 41 ++++++++++++++++++- 4 files changed, 74 insertions(+), 5 deletions(-) diff --git a/.scratch/workspace-reliability-20261015/spec.md b/.scratch/workspace-reliability-20261015/spec.md index 1623478e..54588f76 100644 --- a/.scratch/workspace-reliability-20261015/spec.md +++ b/.scratch/workspace-reliability-20261015/spec.md @@ -109,3 +109,15 @@ rollback must first disable retry entry and finish/reconcile outstanding jobs. Production index installation needs its separately approved plan. Source-media failures in TEST-data remain outside this repair and are not made recoverable by retrying an invalid source. + +## Comments: release preparation + +The user subsequently requested deployment and online testing. Release preparation +found that the existing tool patched API before processor, conflicting with the +retry payload compatibility requirement above. The release tool now patches and +fully verifies the processor rollout before patching API. Timeout/non-convergence +preserves the known partial outcome and leaves API untouched. Three fake-cluster +regressions cover ordering, wait timeout and unsuccessful convergence; the full +release-tool test suite passes. This changes release tooling only; production +application behavior remains the aa729a62 repair. A concrete final plan still +requires approval under docs/release/README.md before its first cluster write. diff --git a/docs/release/README.md b/docs/release/README.md index 720208fb..a8d064c3 100644 --- a/docs/release/README.md +++ b/docs/release/README.md @@ -106,6 +106,10 @@ python3 scripts/release.py deploy \ 核对实际状态再制定恢复计划。JSON 条件更新防止覆盖同一资源的并发修改,但不能保证跨资源 原子性;正式部署应使用团队约定的维护窗口,后续可增加服务端发布锁。 +处理器先更新,完整 rollout 并核对候选镜像与构建标记后才更新 API,避免旧处理器接到 +无法理解的新持久任务载荷。等待失败保留已确认的处理器 patch,API 不更新;按部分发布 +处理,先只读 verify,不重复 deploy、不自动回退。计划展示此固定顺序。 + 完成条件:仅代表 patch 已确认,状态仍为 `deployed_pending_verification`。 ### 5. Verify:分开基础设施与业务验收 diff --git a/scripts/release.py b/scripts/release.py index f92aa7b6..214eba77 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -134,10 +134,10 @@ def fresh(r): require(digest(policy()) == r['policyHash'], 'Release policy changed; prepare a new release.') -def kubectl(r, args, *, stdin=None): +def kubectl(r, args, *, stdin=None, timeout=60): t = r['target'] return run(['kubectl', '--kubeconfig', t['kubeconfig'], '--context', t['context'], - '--request-timeout=20s', '-n', t['namespace']] + args, stdin=stdin, timeout=60) + '--request-timeout=20s', '-n', t['namespace']] + args, stdin=stdin, timeout=timeout) def get(r, kind, name): @@ -340,7 +340,8 @@ def make_plan(r): r['state'] = 'awaiting_approval' event(r, 'plan prepared; no authorization inferred') print(json.dumps({'planHash': r['planHash'], 'target': r['target'], 'changes': [ - {'deployment': n, 'from': d['image'], 'to': r['image'], 'buildCommit': r['commit']} for n, d in r['baseline']['deployments'].items()], + {'deployment': n, 'from': r['baseline']['deployments'][n]['image'], 'to': r['image'], 'buildCommit': r['commit']} for n in sorted(r['baseline']['deployments'])], + 'sequence': 'Processor first; verify its complete rollout before patching API.', 'bandwidthMbps': r['baseline']['publicService']['bandwidthMbps'], 'warnings': ['Only image/build stamp changes; startup migrations require change-safety evidence.', 'Image-to-source provenance requires independent evidence.', @@ -363,7 +364,9 @@ def deploy(r, a): r['approval'] = {'reference': a.approval_ref, 'at': now(), 'planHash': r['planHash']} r['state'] = 'deploying' event(r, 'explicit approval reference recorded; deployment starting') - for name in current['deployments']: + # The processor must understand new durable payloads before API replicas can + # enqueue them. The fixed Fanyu names sort processor before API. + for name in sorted(current['deployments']): latest = snapshot(r) require(all(latest[k] == current[k] for k in ('configs', 'pvc', 'publicService')), 'Configuration changed during deployment; inspect partial outcome.') for n, live in latest['deployments'].items(): @@ -391,6 +394,17 @@ def deploy(r, a): raise op['result'] = 'acknowledged' save(r) + if policy()['deployments'][name] == 'processor': + event(r, 'processor patch acknowledged; waiting for rollout before API') + try: + kubectl(r, ['rollout', 'status', 'deployment/' + name, '--timeout=300s'], timeout=330) + ready = snapshot(r)['deployments'][name] + require(ready['ready'] and ready['image'] == r['image'] and ready['buildCommit'] == r['commit'], + 'Processor rollout has not converged to the candidate.') + except Blocked: + r['state'] = 'deployed_pending_verification' + event(r, 'processor rollout not verified; API was not patched; inspect partial outcome') + raise r['state'] = 'deployed_pending_verification' event(r, 'image patches acknowledged; rollout and acceptance not yet verified') diff --git a/tests/release/test_release.py b/tests/release/test_release.py index 057cf628..15ae401b 100644 --- a/tests/release/test_release.py +++ b/tests/release/test_release.py @@ -57,6 +57,8 @@ class ReleaseTests(unittest.TestCase): self.ds[n], config = fixture(n, c) self.configs = {'config': config} self.writes = [] + self.rollout_waits = [] + self.fail_rollout = False self.fail_patch = None self.before_snapshot = None self.start_patch('snapshot', self.snapshot) @@ -81,7 +83,12 @@ class ReleaseTests(unittest.TestCase): 'configs': {'config': {'uid': 'config-uid', 'hash': m.digest(self.configs)}}, 'pvc': {'uid': 'pvc-uid'}, 'publicService': {'bandwidthMbps': '5'}} - def kubectl(self, r, args, *, stdin=None): + def kubectl(self, r, args, *, stdin=None, timeout=60): + if args[:2] == ['rollout', 'status']: + self.rollout_waits.append((args[2], list(self.writes))) + if self.fail_rollout: + raise m.Blocked('simulated rollout timeout') + return 'successfully rolled out' self.assertEqual(args[:2], ['patch', 'deployment']) name = args[2] self.writes.append(name) @@ -281,6 +288,38 @@ class ReleaseTests(unittest.TestCase): m.verify(self.r) self.assertEqual(self.r['state'], 'deployed_pending_acceptance') + def test_processor_rollout_precedes_api_patch(self): + self.plan() + m.deploy(self.r, self.approve()) + self.assertEqual(self.writes, ['droboticflow-annotation-processor', 'droboticflow-api']) + self.assertEqual(self.rollout_waits, [ + ('deployment/droboticflow-annotation-processor', ['droboticflow-annotation-processor'])]) + + def test_processor_rollout_timeout_leaves_api_untouched(self): + self.plan() + a = self.approve() + self.fail_rollout = True + with self.assertRaises(m.Blocked): + m.deploy(self.r, a) + self.assertEqual(self.writes, ['droboticflow-annotation-processor']) + saved = self.load() + self.assertEqual(saved['state'], 'deployed_pending_verification') + self.assertEqual([o['result'] for o in saved['operations']], ['acknowledged']) + with self.assertRaises(m.Blocked): + m.deploy(saved, a) + + def test_processor_not_converged_leaves_api_untouched(self): + self.plan() + a = self.approve() + def not_ready(): + if self.writes: + self.ds['droboticflow-annotation-processor']['status']['readyReplicas'] = 0 + self.before_snapshot = not_ready + with self.assertRaises(m.Blocked): + m.deploy(self.r, a) + self.assertEqual(self.writes, ['droboticflow-annotation-processor']) + self.assertEqual(self.load()['state'], 'deployed_pending_verification') + def test_no_predeployment_business_evidence(self): self.plan() with self.assertRaises(m.Blocked):