diff --git a/docs/operations/query-shadow-monitor.md b/docs/operations/query-shadow-monitor.md index 33c66c33..f641515a 100644 --- a/docs/operations/query-shadow-monitor.md +++ b/docs/operations/query-shadow-monitor.md @@ -1,5 +1,13 @@ # 集群内影子观察器 +## 2026-11-16 运行恢复:可选追溯读取超时分类 + +恢复时已修正生产 Role 中仍绑定旧 API Pod 名的两条只读规则,四个当前 Pod 日志授权通过。今后 API 滚动更新后的观察器预检须同时核对 Role 的 `resourceNames`、实际授权与 ConfigMap 的 Pod UID,不能只核对 RoleBinding。 + +在原200档运行时,观察器被 `[review-trace] list summary unavailable` 的 `i/o timeout` 暂停。该固定日志来自可选的 GET 审查标签补充;其失败不使已构成的任务列表失败。观察器将任意 API 日志中的超时都计为作业错误,导致误停。 + +仅对这条独立、精确匹配的可选读取日志排除写作业故障判定。结构化保存、切片、送检、审核、验收异常仍停止;未知 I/O 超时、panic、fatal、写冲突耗尽及明确保存/送检失败仍停止。不扩大时限、不减少持久性检查、不自动清除已有暂停。本地回放通过后仍须新候选发布授权,不能直接恢复旧观察器。 + ## 2026-11-16 恢复准备:逐写者版本绑定 当前 API/切片处理器为 `88992d7e`,其他写者保留各自已发布版本;这不要求将五个组件一起部署。旧观察器用单个 `writerCommit` 校验所有组件,会拒绝这一合法组合。 diff --git a/scripts/local/query-shadow-observer.py b/scripts/local/query-shadow-observer.py index cfd0092e..9741aaaa 100644 --- a/scripts/local/query-shadow-observer.py +++ b/scripts/local/query-shadow-observer.py @@ -51,6 +51,24 @@ def full_io_pressure(text): raise ValueError('protected IO evidence missing') +def api_business_log_failure(text): + """Keep hard failures; optional GET audit-label enrichment is not a write. + + Match only the API's exact, standalone log template. Unknown transport + failures and the structured save/slice/submit/review feed remain blocking. + """ + optional_trace = re.compile(r'^(?:\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)? )?' + r'\[review-trace\] list summary unavailable: [^\r\n]*$') + for line in text.splitlines(): + lower = line.lower() + if any(token in lower for token in ('panic:', 'fatal error:', 'write conflict retry exhausted', + 'submit failed', 'save failed')): + return True + if 'i/o timeout' in lower and not optional_trace.fullmatch(line): + return True + return False + + def business_health(path, now=None, binding_sha256=None): feed = json.loads(Path(path).read_text()) if binding_sha256 is not None and (feed.get('bindingSHA256') != binding_sha256 or feed.get('metricKind') != 'http-slow-error-tail'): @@ -248,7 +266,7 @@ class Observer: for p in pods: if p['metadata'].get('deletionTimestamp'): continue logs = self.call('-n', self.b['sourceNamespace'], 'logs', p['metadata']['name'], '--since=45s', '--tail=2000').lower() - work_errors |= any(s in logs for s in ('panic:', 'i/o timeout', 'write conflict retry exhausted', 'submit failed', 'save failed')) + work_errors |= api_business_log_failure(logs) result = dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True, workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes']) if self.b.get('maintenanceInitialization'): diff --git a/scripts/local/test_query_shadow_observer.py b/scripts/local/test_query_shadow_observer.py index 513d0786..e957502e 100644 --- a/scripts/local/test_query_shadow_observer.py +++ b/scripts/local/test_query_shadow_observer.py @@ -10,6 +10,44 @@ spec = importlib.util.spec_from_file_location('shadow_observer', Path(__file__). m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m) class ObserverTests(unittest.TestCase): + def test_optional_trace_timeout_does_not_pause_successful_work(self): + # Replay the production warning through the actual observation call, + # alongside the authoritative five-operation health feed. + now = dt.datetime.now(dt.timezone.utc) + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / 'health.json' + path.write_text(json.dumps(dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance', + unexpectedErrors=False, latencyRegressed=False, bindingSHA256='a' * 64, metricKind='http-slow-error-tail'))) + b = self.bindings() + b.update(mongoPodUID='query', workHealthBindingSHA256='a' * 64) + o = m.Observer(SimpleNamespace(kubeconfig='unused', work_health=str(path)), b) + o.verify = lambda: None + o.readiness = lambda: True + o.get = lambda *a: dict(metadata=dict(uid='query')) + o.host_filesystem = lambda: dict(availableBytes=200 << 30) + warning = '2026/11/17 10:40:18 [review-trace] list summary unavailable: connection read: i/o timeout' + for log, expected in ((warning, False), ('unknown backend failure: i/o timeout', True), + (warning + '\nsave failed: i/o timeout', True), + (warning + '\npanic: unexpected fault', True), + (warning + '\nsubmit failed', True), + (warning + '\nwrite conflict retry exhausted', True), + ('prefix ' + warning, True)): + with self.subTest(log=log): + def call(*a, **kw): + if 'df' in a: return 'Filesystem 1B Used Available Capacity Mounted\nloop 100 10 90 10% /lab/data' + if 'pods' in a: + return json.dumps(dict(items=[] if '--field-selector=spec.nodeName=10.189.16.4' in a + else [dict(metadata=dict(name='api'))])) + if 'logs' in a: return log + self.fail('unexpected probe') + o.call = call + self.assertEqual(o.observation(50, now.isoformat())['workErrors'], expected) + feed = json.loads(path.read_text()) + feed['unexpectedErrors'] = True + path.write_text(json.dumps(feed)) + log = warning + self.assertTrue(o.observation(50, now.isoformat())['workErrors']) + def test_mixed_writer_versions_are_bound_individually(self): b = self.bindings() b.update(sourceNamespaceUID='authority-uid', shadowNamespaceUID='shadow-uid', writerCommit='old')