🛠️ Avoid pausing shadow sync on optional review-trace read timeouts

This commit is contained in:
d-robotics 2026-11-16 18:51:54 -08:00
parent 7a0201dbca
commit 5699e7cf58
3 changed files with 65 additions and 1 deletions

View File

@ -1,5 +1,13 @@
# 集群内影子观察器
## 2026-11-16 运行恢复:可选追溯读取超时分类
恢复时已修正生产 Role 中仍绑定旧 API Pod 名的两条只读规则,四个当前 Pod 日志授权通过。今后 API 滚动更新后的观察器预检须同时核对 Role 的 `resourceNames`、实际授权与 ConfigMap 的 Pod UID,不能只核对 RoleBinding。
在原200档运行时,观察器被 `[review-trace] list summary unavailable` 的 `i/o timeout` 暂停。该固定日志来自可选的 GET 审查标签补充;其失败不使已构成的任务列表失败。观察器将任意 API 日志中的超时都计为作业错误,导致误停。
仅对这条独立、精确匹配的可选读取日志排除写作业故障判定。结构化保存、切片、送检、审核、验收异常仍停止;未知 I/O 超时、panic、fatal、写冲突耗尽及明确保存/送检失败仍停止。不扩大时限、不减少持久性检查、不自动清除已有暂停。本地回放通过后仍须新候选发布授权,不能直接恢复旧观察器。
## 2026-11-16 恢复准备:逐写者版本绑定
当前 API/切片处理器为 `88992d7e`,其他写者保留各自已发布版本;这不要求将五个组件一起部署。旧观察器用单个 `writerCommit` 校验所有组件,会拒绝这一合法组合。

View File

@ -51,6 +51,24 @@ def full_io_pressure(text):
raise ValueError('protected IO evidence missing')
def api_business_log_failure(text):
"""Keep hard failures; optional GET audit-label enrichment is not a write.
Match only the API's exact, standalone log template. Unknown transport
failures and the structured save/slice/submit/review feed remain blocking.
"""
optional_trace = re.compile(r'^(?:\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}(?:\.\d+)? )?'
r'\[review-trace\] list summary unavailable: [^\r\n]*$')
for line in text.splitlines():
lower = line.lower()
if any(token in lower for token in ('panic:', 'fatal error:', 'write conflict retry exhausted',
'submit failed', 'save failed')):
return True
if 'i/o timeout' in lower and not optional_trace.fullmatch(line):
return True
return False
def business_health(path, now=None, binding_sha256=None):
feed = json.loads(Path(path).read_text())
if binding_sha256 is not None and (feed.get('bindingSHA256') != binding_sha256 or feed.get('metricKind') != 'http-slow-error-tail'):
@ -248,7 +266,7 @@ class Observer:
for p in pods:
if p['metadata'].get('deletionTimestamp'): continue
logs = self.call('-n', self.b['sourceNamespace'], 'logs', p['metadata']['name'], '--since=45s', '--tail=2000').lower()
work_errors |= any(s in logs for s in ('panic:', 'i/o timeout', 'write conflict retry exhausted', 'submit failed', 'save failed'))
work_errors |= api_business_log_failure(logs)
result = dict(at=sampled_at, cpu=cpu, ready=ready, writersCovered=True,
workErrors=work_errors, protectedIOFull=io, targetUsedFraction=used, hostFreeBytes=fs['availableBytes'])
if self.b.get('maintenanceInitialization'):

View File

@ -10,6 +10,44 @@ spec = importlib.util.spec_from_file_location('shadow_observer', Path(__file__).
m = importlib.util.module_from_spec(spec); spec.loader.exec_module(m)
class ObserverTests(unittest.TestCase):
def test_optional_trace_timeout_does_not_pause_successful_work(self):
# Replay the production warning through the actual observation call,
# alongside the authoritative five-operation health feed.
now = dt.datetime.now(dt.timezone.utc)
with tempfile.TemporaryDirectory() as folder:
path = Path(folder) / 'health.json'
path.write_text(json.dumps(dict(at=now.isoformat(), coverage='save-slice-submit-review-acceptance',
unexpectedErrors=False, latencyRegressed=False, bindingSHA256='a' * 64, metricKind='http-slow-error-tail')))
b = self.bindings()
b.update(mongoPodUID='query', workHealthBindingSHA256='a' * 64)
o = m.Observer(SimpleNamespace(kubeconfig='unused', work_health=str(path)), b)
o.verify = lambda: None
o.readiness = lambda: True
o.get = lambda *a: dict(metadata=dict(uid='query'))
o.host_filesystem = lambda: dict(availableBytes=200 << 30)
warning = '2026/11/17 10:40:18 [review-trace] list summary unavailable: connection read: i/o timeout'
for log, expected in ((warning, False), ('unknown backend failure: i/o timeout', True),
(warning + '\nsave failed: i/o timeout', True),
(warning + '\npanic: unexpected fault', True),
(warning + '\nsubmit failed', True),
(warning + '\nwrite conflict retry exhausted', True),
('prefix ' + warning, True)):
with self.subTest(log=log):
def call(*a, **kw):
if 'df' in a: return 'Filesystem 1B Used Available Capacity Mounted\nloop 100 10 90 10% /lab/data'
if 'pods' in a:
return json.dumps(dict(items=[] if '--field-selector=spec.nodeName=10.189.16.4' in a
else [dict(metadata=dict(name='api'))]))
if 'logs' in a: return log
self.fail('unexpected probe')
o.call = call
self.assertEqual(o.observation(50, now.isoformat())['workErrors'], expected)
feed = json.loads(path.read_text())
feed['unexpectedErrors'] = True
path.write_text(json.dumps(feed))
log = warning
self.assertTrue(o.observation(50, now.isoformat())['workErrors'])
def test_mixed_writer_versions_are_bound_individually(self):
b = self.bindings()
b.update(sourceNamespaceUID='authority-uid', shadowNamespaceUID='shadow-uid', writerCommit='old')