🛠️ Avoid pausing shadow sync on optional review-trace read timeouts
This commit is contained in:
parent
7a0201dbca
commit
5699e7cf58
@ -1,5 +1,13 @@
|
||||
# 集群内影子观察器
|
||||
|
||||
## 2026-11-16 运行恢复:可选追溯读取超时分类
|
||||
|
||||
恢复时已修正生产 Role 中仍绑定旧 API Pod 名的两条只读规则,四个当前 Pod 日志授权通过。今后 API 滚动更新后的观察器预检须同时核对 Role 的 `resourceNames`、实际授权与 ConfigMap 的 Pod UID,不能只核对 RoleBinding。
|
||||
|
||||
在原200档运行时,观察器被 `[review-trace] list summary unavailable` 的 `i/o timeout` 暂停。该固定日志来自可选的 GET 审查标签补充;其失败不使已构成的任务列表失败。观察器将任意 API 日志中的超时都计为作业错误,导致误停。
|
||||
|
||||
仅对这条独立、精确匹配的可选读取日志排除写作业故障判定。结构化保存、切片、送检、审核、验收异常仍停止;未知 I/O 超时、panic、fatal、写冲突耗尽及明确保存/送检失败仍停止。不扩大时限、不减少持久性检查、不自动清除已有暂停。本地回放通过后仍须新候选发布授权,不能直接恢复旧观察器。
|
||||
|
||||
## 2026-11-16 恢复准备:逐写者版本绑定
|
||||
|
||||
当前 API/切片处理器为 `88992d7e`,其他写者保留各自已发布版本;这不要求将五个组件一起部署。旧观察器用单个 `writerCommit` 校验所有组件,会拒绝这一合法组合。
|
||||
|
||||
@ -51,6 +51,24 @@ def full_io_pressure(text):
|
||||
raise ValueError('protected IO evidence missing')
|
||||
|
||||
|
||||
def api_business_log_failure(text):
|
||||
"""Keep hard failures; optional GET audit-label enrichment is not a write.
|
||||
|
||||
Match only the API's exact, standalone log template. Unknown transport
|
||||
failures and the structured save/slice/submit/review feed remain blocking.
|
||||
"""
|
||||
optional_trace = re.compile(r'^(?:\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)? )?'
|
||||
r'\[review-trace\] list summary unavailable: [^\r\n]*$')
|
||||
for line in text.splitlines():
|
||||
lower = line.lower()
|
||||
if any(token in lower for token in ('panic:', 'fatal error:', 'write conflict retry exhausted',
|
||||
'submit failed', 'save failed')):
|
||||
return True
|
||||
if 'i/o timeout' in lower and not optional_trace.fullmatch(line):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def business_health(path, now=None, binding_sha256=None):
|
||||
feed = json.loads(Path(path).read_text())
|
||||
if binding_sha256 is not None and (feed.get('bindingSHA256') != binding_sha256 or feed.get('metricKind') != 'http-slow-error-tail'):
|
||||
@ -248,7 +266,7 @@ class Observer:
|
||||
for p in pods:
|
||||
if p['metadata'].get('deletionTimestamp'): continue
|
||||
logs = self.call('-n', self.b['sourceNamespace'], 'logs', p['metadata']['name'], '--since=45s', '--tail=2000').lower()
|
||||
work_errors |= any(s in logs for s in ('panic:', 'i/o timeout', 'write conflict retry exhausted', 'submit failed', 'save failed'))
|
||||
work_errors |= api_business_log_failure(logs)
|
||||
result = dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True,
|
||||
workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes'])
|
||||
if self.b.get('maintenanceInitialization'):
|
||||
|
||||
@ -10,6 +10,44 @@ spec = importlib.util.spec_from_file_location('shadow_observer', Path(__file__).
|
||||
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
|
||||
|
||||
class ObserverTests(unittest.TestCase):
|
||||
def test_optional_trace_timeout_does_not_pause_successful_work(self):
|
||||
# Replay the production warning through the actual observation call,
|
||||
# alongside the authoritative five-operation health feed.
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / 'health.json'
|
||||
path.write_text(json.dumps(dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance',
|
||||
unexpectedErrors=False, latencyRegressed=False, bindingSHA256='a' * 64, metricKind='http-slow-error-tail')))
|
||||
b = self.bindings()
|
||||
b.update(mongoPodUID='query', workHealthBindingSHA256='a' * 64)
|
||||
o = m.Observer(SimpleNamespace(kubeconfig='unused', work_health=str(path)), b)
|
||||
o.verify = lambda: None
|
||||
o.readiness = lambda: True
|
||||
o.get = lambda *a: dict(metadata=dict(uid='query'))
|
||||
o.host_filesystem = lambda: dict(availableBytes=200 << 30)
|
||||
warning = '2026/11/17 10:40:18 [review-trace] list summary unavailable: connection read: i/o timeout'
|
||||
for log, expected in ((warning, False), ('unknown backend failure: i/o timeout', True),
|
||||
(warning + '\nsave failed: i/o timeout', True),
|
||||
(warning + '\npanic: unexpected fault', True),
|
||||
(warning + '\nsubmit failed', True),
|
||||
(warning + '\nwrite conflict retry exhausted', True),
|
||||
('prefix ' + warning, True)):
|
||||
with self.subTest(log=log):
|
||||
def call(*a, **kw):
|
||||
if 'df' in a: return 'Filesystem 1B Used Available Capacity Mounted\nloop 100 10 90 10% /lab/data'
|
||||
if 'pods' in a:
|
||||
return json.dumps(dict(items=[] if '--field-selector=spec.nodeName=10.189.16.4' in a
|
||||
else [dict(metadata=dict(name='api'))]))
|
||||
if 'logs' in a: return log
|
||||
self.fail('unexpected probe')
|
||||
o.call = call
|
||||
self.assertEqual(o.observation(50, now.isoformat())['workErrors'], expected)
|
||||
feed = json.loads(path.read_text())
|
||||
feed['unexpectedErrors'] = True
|
||||
path.write_text(json.dumps(feed))
|
||||
log = warning
|
||||
self.assertTrue(o.observation(50, now.isoformat())['workErrors'])
|
||||
|
||||
def test_mixed_writer_versions_are_bound_individually(self):
|
||||
b = self.bindings()
|
||||
b.update(sourceNamespaceUID='authority-uid', shadowNamespaceUID='shadow-uid', writerCommit='old')
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user