网络小故障不再拖垮整条轨迹:原地重发、没发出去的暂停不扣续跑次数、结果不明的请求可以续跑

GLM 请求失败率约 0.3%,但一次失败就让整条轨迹暂停;3 次续跑次数被网络故障用完后,
轨迹只能挂到 24 小时上限(10-14 有 4 条);回复中途断开的请求过了等待期会被自动放弃,
同时整条执行被取消(10-13/14 共 16 条)。

A. 连接失败、域名解析失败(RETRY_UNSENT,确定什么都没发出去)在 broker 里等一会儿原地重发。
   每次重试把等待时间加上一次连接时限计入排队预算 wait_spent,保持
   客户端超时 = 排队预算 + 次数×单次时限 的上界;预算用完才照旧暂停。
B. 续跑次数只在最近一次请求可能已经送达时才扣(看账本最近一次 attempt 是否 sent)。
   连接/解析失败、因为别的请求结果不明而在准入时被挡下的暂停,续跑不扣次数。
   执行记录和账本两处计数同步,调度程序不再按原始次数预先过滤。
C. reconcile 自动放弃时(截止时间再过一个 token 窗口)不再取消执行,而是把这一轮
   标成 REQUEST_OUTCOME_UNKNOWN、可以续跑:重发一轮只花一次请求的 token,并计入续跑次数。
   人工放弃(operator)仍然永久隔离。续跑资格检查不再被已放弃的 attempt 挡住。
都在调度程序进程里运行,下次重启生效。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bench 2026-10-14 15:41:49 +08:00
parent f74fe00aaf
commit f83fa3948c
7 changed files with 110 additions and 23 deletions

View File

@ -558,7 +558,7 @@ class Execution:
checkpoint=checkpoint, paused_at=time.time())
return self._save(state, 'EXECUTION_PAUSED', reason=reason)
def resume(self, *, automatic=True):
def resume(self, *, automatic=True, counted=True):
with self.locked():
state = self.snapshot()
if self.epoch is not None and self.epoch != state['epoch']:
@ -582,12 +582,12 @@ class Execution:
if members != [] or pending:
raise ExecutionStopped('SIDE_EFFECT_UNKNOWN' if members is None else 'WORK_STILL_ACTIVE')
count = state.get('auto_resume_count', 0)
if automatic and count >= state['config'].get('max_auto_resumes', 3):
if automatic and counted and count >= state['config'].get('max_auto_resumes', 3):
raise ExecutionStopped('AUTO_RESUME_BUDGET_EXHAUSTED')
state['epoch'] += 1
self.epoch = state['epoch']
state.update(owner=process_identity(os.getpid()), execution_state='ACTIVE', stop_reason=None,
auto_resume_count=count + int(automatic), resumed_at=time.time(), activity='WAITING_QUOTA')
auto_resume_count=count + int(automatic and counted), resumed_at=time.time(), activity='WAITING_QUOTA')
return self._save(state, 'EXECUTION_RESUMED', automatic=automatic)
def cleanup(self):

View File

@ -560,7 +560,7 @@ def _resume_snapshot(run_id, job_ids=None, modes=None):
cancel=bool(state.get('cancel_requested')), timeout_s=0)
continue
if (state['execution_state'] != 'PAUSED_INFRA' or state['cleanup_state'] == 'QUARANTINED'
or state.get('cancel_requested') or state.get('auto_resume_count', 0) >= state['config'].get('max_auto_resumes', 3)):
or state.get('cancel_requested')):
continue
try:
resume_policy(state)

View File

@ -29,6 +29,10 @@ from urllib.parse import urlsplit, quote
import uuid
# Failures proved to have sent nothing: wait and resend inside the broker instead of pausing the rollout.
RETRY_UNSENT = ('CONNECT_FAILED', 'DNS_FAILED', 'DNS_TIMEOUT')
class QuotaError(Exception):
def __init__(self, reason: str, status: int = 503):
self.reason, self.status = reason, status
@ -584,7 +588,7 @@ class Broker:
self._event(con, group, 'UNKNOWN_SETTLED_PROBE_ALLOWED')
for attempt_id in stale: # outside the group flock: abandon_unknown takes it itself
try:
self.abandon_unknown(attempt_id, operator_reason='AUTO: local deadline + token window elapsed')
self.abandon_unknown(attempt_id, operator_reason='AUTO: local deadline + token window elapsed', resumable=True)
except QuotaError:
pass # another broker settled it first
@ -600,11 +604,14 @@ class Broker:
con.execute("UPDATE quota_groups SET state='COOLING',epoch=epoch+1,target=1,reason=NULL,cooldown_until=MAX(cooldown_until,?),updated_at=? WHERE group_id=?", (now, now, group))
return True
def abandon_unknown(self, attempt_id: str, *, operator_reason: str):
def abandon_unknown(self, attempt_id: str, *, operator_reason: str, resumable: bool = False):
"""Explicit operator release of local capacity, never provider-end proof.
The old generation is fenced forever. Original usage and ended_at stay
untouched. reconcile() also calls this once a token window has passed.
The attempt stays unknown for audit (usage and ended_at untouched). An operator
release fences the generation forever. reconcile() calls this with resumable=True
once a token window has passed: the provider can no longer charge that window, so
the turn may be replayed (one request's tokens, counted against the resume budget)
instead of losing the rollout (10-13/14: 16 rollouts lost this way).
"""
if not isinstance(operator_reason, str) or not operator_reason.strip():
raise ValueError('explicit operator reason required')
@ -625,15 +632,19 @@ class Broker:
execution_id = con.execute('SELECT execution_id FROM generations WHERE generation_id=?',
(generation,)).fetchone()[0]
con.execute("UPDATE attempts SET state='ABANDONED_UNKNOWN' WHERE attempt_id=?", (attempt_id,))
con.execute("UPDATE generations SET state='ABANDONED_UNKNOWN',reason='OPERATOR_ABANDONED_UNKNOWN',updated_at=? WHERE generation_id=?", (now, generation))
con.execute('UPDATE executions SET cancelled=1 WHERE execution_id=?', (execution_id,))
if resumable:
con.execute("UPDATE generations SET state='PAUSED',reason='REQUEST_OUTCOME_UNKNOWN',updated_at=? WHERE generation_id=?", (now, generation))
else:
con.execute("UPDATE generations SET state='ABANDONED_UNKNOWN',reason='OPERATOR_ABANDONED_UNKNOWN',updated_at=? WHERE generation_id=?", (now, generation))
con.execute('UPDATE executions SET cancelled=1 WHERE execution_id=?', (execution_id,))
probe_allowed = self._probe_if_settled(con, group)
self._event(con, group, 'OPERATOR_ABANDONED_UNKNOWN', generation,
attempt_id=attempt_id, operator_reason=operator_reason,
released_at=now, provider_end_confirmed=False,
original_reserved_tokens=row['reserved_tokens'],
original_charged_tokens=row['charged_tokens'], probe_allowed=bool(probe_allowed))
self._cancel_requested.setdefault(execution_id, threading.Event()).set()
original_charged_tokens=row['charged_tokens'], probe_allowed=bool(probe_allowed), resumable=resumable)
if not resumable:
self._cancel_requested.setdefault(execution_id, threading.Event()).set()
return True
def confirm_end(self, attempt_id: str, receipt_id: str):
@ -668,15 +679,19 @@ class Broker:
if not gen:
return result | {'reason':'NO_PAUSED_GENERATION'}
result['generation_id']=gen['generation_id']
# Only a resume after a possibly delivered request spends the budget; connect/DNS failures and
# waiting behind another request's unknown end sent nothing (10-14: 4 rollouts stuck at 3/3 on those).
last=con.execute('SELECT sent FROM attempts WHERE generation_id=? ORDER BY started_at DESC LIMIT 1',(gen['generation_id'],)).fetchone()
result['counted']=last is not None and last['sent'] != 0
policy=self.groups[ex['group_id']]
if (ex['cancelled'] or gen['state']!='PAUSED'
# CONNECT_FAILED / REQUEST_DEADLINE_BEFORE_SEND / DNS_*: nothing was sent, replay is safe.
# INCOMPLETE_GENERATION: a cut-off reply whose end the ledger recorded (the open-attempt check below still applies).
or gen['reason'] not in ('QUOTA_WAIT_BUDGET_EXHAUSTED','REQUEST_OUTCOME_UNKNOWN','CONNECT_FAILED','REQUEST_DEADLINE_BEFORE_SEND','INCOMPLETE_GENERATION','DNS_FAILED','DNS_TIMEOUT')
# A generation whose own request end is unknown never replays.
or con.execute('SELECT 1 FROM attempts WHERE generation_id=? AND ended_at IS NULL',(gen['generation_id'],)).fetchone()):
or con.execute("SELECT 1 FROM attempts WHERE generation_id=? AND ended_at IS NULL AND state <> 'ABANDONED_UNKNOWN'",(gen['generation_id'],)).fetchone()):
return result | {'reason':'RESUME_NOT_ELIGIBLE'}
if ex['resumes']>=policy.max_auto_resumes or gen['attempts']>=policy.max_attempts:
if (result['counted'] and ex['resumes']>=policy.max_auto_resumes) or gen['attempts']>=policy.max_attempts:
return result | {'reason':'RESUME_BUDGET_EXHAUSTED'}
if gen['reserved_tokens']<=0:
return result | {'reason':'RESERVATION_METADATA_MISSING'}
@ -722,7 +737,7 @@ class Broker:
readiness=self._resume_readiness(con,execution_id,self._now())
if not readiness['eligible'] or readiness['generation_id']!=generation_id:
raise QuotaError(readiness['reason'] if not readiness['eligible'] else 'GENERATION_MISMATCH',409)
con.execute('UPDATE executions SET resumes=resumes+1 WHERE execution_id=?',(execution_id,))
con.execute('UPDATE executions SET resumes=resumes+? WHERE execution_id=?',(int(readiness['counted']),execution_id))
con.execute("UPDATE generations SET state='RESUMABLE',wait_since=? WHERE generation_id=?",(self._now(),generation_id))
def _begin(self, token, body, requested_id):
@ -1044,12 +1059,37 @@ class Broker:
watcher = threading.Thread(target=watch_client, daemon=True)
watcher.start()
try:
return self._generate(scope, upstream, policy, generation, data, body, reserve, client_connected)
while True:
try:
return self._generate(scope, upstream, policy, generation, data, body, reserve, client_connected)
except QuotaError as exc:
if exc.reason not in RETRY_UNSENT or not self._wait_unsent(upstream.quota_group, generation, policy):
raise
finally:
stopped.set()
if watcher is not None:
watcher.join()
def _wait_unsent(self, group, generation, policy):
"""Nothing reached the provider: wait here and resend rather than pause the whole rollout.
Each retry is charged to the execution's queue budget (the wait plus one
connect/DNS deadline), so the client timeout bound holds; once the budget
is spent the generation stays paused as before.
"""
if self._closing.wait(policy.cooldown_initial_s):
return False
charge = policy.cooldown_initial_s + policy.connect_timeout_s
with self._group_lock(group), self._db() as con:
con.execute('BEGIN IMMEDIATE')
row = con.execute('SELECT g.state,g.execution_id,e.wait_spent,e.cancelled FROM generations g JOIN executions e ON e.execution_id=g.execution_id WHERE g.generation_id=?', (generation,)).fetchone()
if row['state'] != 'PAUSED' or row['cancelled'] or row['wait_spent'] + charge > policy.wait_budget_s:
return False
con.execute('UPDATE executions SET wait_spent=wait_spent+? WHERE execution_id=?', (charge, row['execution_id']))
con.execute("UPDATE generations SET state='QUEUED',updated_at=? WHERE generation_id=?", (self._now(), generation))
self._event(con, group, 'UNSENT_RETRY', generation)
return True
def _generate(self, scope, upstream, policy, generation, data, body, reserve, client_connected):
url=urlsplit(upstream.base_url)
started=time.monotonic()

View File

@ -425,7 +425,7 @@ def resume_rollout(run_id: str, job_id: str, kk: int) -> dict:
readiness = runtime.broker().resume_readiness(state['execution_id'])
if not readiness['eligible']:
raise ValueError('RESUME_NOT_READY: ' + str(readiness.get('reason')))
execution.resume()
execution.resume(counted=readiness.get('counted', True))
try:
runtime.broker().resume(state['execution_id'], checkpoint['generation_id'],
checkpoint_complete=checkpoint.get('resumable', False), side_effects_known=True)

View File

@ -359,6 +359,9 @@ class ExecutionTests(unittest.TestCase):
self.ex.pause('quota', checkpoint)
with self.assertRaisesRegex(execution.ExecutionStopped, 'AUTO_RESUME_BUDGET_EXHAUSTED'):
execution.Execution(self.rd).resume()
free = execution.Execution(self.rd)
free.resume(counted=False) # a pause that sent nothing (connect/DNS failure) is free
self.assertEqual(free.snapshot()['auto_resume_count'], 3)
@unittest.skipUnless(Path('/proc').is_dir(), 'Actual worker requires Linux owner identity')
def test_worker_executes_real_protected_stage_and_reuses_receipt(self):

View File

@ -579,6 +579,44 @@ class Quota(unittest.TestCase):
self.assertEqual(self.request(scope,'g')[0],200)
self.assertEqual(len(p.requests),1)
def test_connect_blip_is_resent_in_place_and_charged_to_the_wait_budget(self):
# 10-14: one refused connect paused a 10-hour rollout; four of them stranded it at 3/3 resumes.
p=self.provider();b=self.broker(p,self.policy())
scope=b.issue_scope('blip','teacher','glm-test')
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
refusals=[2]
def flaky(connection):
if connection.port==p.server.server_port and refusals[0]:
refusals[0]-=1
raise ConnectionRefusedError('injected Wi-Fi blip')
return original(connection)
with patch('harness.quota.http.client.HTTPConnection.connect',new=flaky):
self.assertEqual(self.request(scope)[0],200)
self.assertEqual(len(p.requests),1)
self.assertGreaterEqual(b.execution_status('blip')['wait_spent'],2*(.06+.2))
self.assertEqual(len([e for e in b.snapshot()['quota_events'] if e['event']=='UNSENT_RETRY']),2)
def test_pause_that_sent_nothing_does_not_spend_the_resume_budget(self):
p=self.provider();b=self.broker(p,self.policy())
scope=b.issue_scope('free','teacher','glm-test')
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
def refused(connection):
if connection.port==p.server.server_port:
raise ConnectionRefusedError('injected outage')
return original(connection)
with patch('harness.quota.http.client.HTTPConnection.connect',new=refused):
self.assertEqual(self.request(scope,'g')[0],503) # in-place retries spend the wait budget, then pause
cap=b.groups['shared'].max_auto_resumes
with b._db() as con:
con.execute('UPDATE executions SET resumes=? WHERE execution_id=?',(cap,'free'))
cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0]
with patch.object(b,'_now',return_value=max(b._now(),cooldown)+1):
ready=b.resume_readiness('free')
self.assertTrue(ready['eligible']);self.assertFalse(ready['counted'])
b.resume('free','g',checkpoint_complete=True,side_effects_known=True)
self.assertEqual(b.execution_status('free')['resumes'],cap)
self.assertEqual(p.requests,[])
def test_confirmed_connect_failure_does_not_spend_http_budget(self):
p=self.provider();b=self.broker(p,self.policy(rpm=1,window_s=1))
scope=b.issue_scope('connect-failure','teacher','glm-test')

View File

@ -1,5 +1,6 @@
"""Unknown request end self-heals one token window after its deadline;
innocent generations paused by it resume, its own generation never replays."""
"""Unknown request end self-heals one token window after its deadline: innocent
generations paused by it resume for free, its own generation replays once at the
cost of one resume instead of losing the rollout (10-14)."""
import json
import unittest
@ -16,7 +17,7 @@ class AutoAbandon(unittest.TestCase):
unknown = Abandon.unknown
def test_unknown_waits_one_window_then_group_probes_and_victim_resumes(self):
_, broker, _, attempt = self.unknown()
_, broker, old_scope, attempt = self.unknown()
# Paused at admission by the group, never sent anything itself.
self.assertEqual(self.request(broker.issue_scope('victim', 'teacher', 'glm-test'))[0], 503)
self.assertEqual(broker.resume_readiness('victim')['reason'], 'REQUEST_OUTCOME_UNKNOWN')
@ -35,8 +36,13 @@ class AutoAbandon(unittest.TestCase):
self.assertEqual(snap['groups'][0]['state'], 'COOLING')
events = [e for e in snap['quota_events'] if e['event'] == 'OPERATOR_ABANDONED_UNKNOWN']
self.assertTrue(json.loads(events[0]['detail'])['operator_reason'].startswith('AUTO'))
self.assertTrue(broker.resume_readiness('victim')['eligible'])
self.assertFalse(broker.resume_readiness('old')['eligible'])
victim, old = broker.resume_readiness('victim'), broker.resume_readiness('old')
self.assertTrue(victim['eligible']);self.assertFalse(victim['counted'])
self.assertTrue(old['eligible']);self.assertTrue(old['counted'])
self.assertEqual(broker.execution_status('old')['cancelled'], 0)
broker.resume('old', 'old-g', checkpoint_complete=True, side_effects_known=True)
self.assertEqual(broker.execution_status('old')['resumes'], 1)
self.assertEqual(self.request(old_scope, 'old-g')[0], 200) # the replayed turn is delivered
broker.reconcile() # idempotent
self.assertEqual(len([e for e in broker.snapshot()['quota_events']
@ -60,7 +66,7 @@ class AutoAbandon(unittest.TestCase):
broker.reconcile()
self.assertEqual(state(), 'COOLING')
self.assertTrue(broker.resume_readiness('victim')['eligible'])
self.assertFalse(broker.resume_readiness('old')['eligible'])
self.assertTrue(broker.resume_readiness('old')['eligible'])
if __name__ == '__main__':