网络小故障不再拖垮整条轨迹:原地重发、没发出去的暂停不扣续跑次数、结果不明的请求可以续跑
GLM 请求失败率约 0.3%,但一次失败就让整条轨迹暂停;3 次续跑次数被网络故障用完后, 轨迹只能挂到 24 小时上限(10-14 有 4 条);回复中途断开的请求过了等待期会被自动放弃, 同时整条执行被取消(10-13/14 共 16 条)。 A. 连接失败、域名解析失败(RETRY_UNSENT,确定什么都没发出去)在 broker 里等一会儿原地重发。 每次重试把等待时间加上一次连接时限计入排队预算 wait_spent,保持 客户端超时 = 排队预算 + 次数×单次时限 的上界;预算用完才照旧暂停。 B. 续跑次数只在最近一次请求可能已经送达时才扣(看账本最近一次 attempt 是否 sent)。 连接/解析失败、因为别的请求结果不明而在准入时被挡下的暂停,续跑不扣次数。 执行记录和账本两处计数同步,调度程序不再按原始次数预先过滤。 C. reconcile 自动放弃时(截止时间再过一个 token 窗口)不再取消执行,而是把这一轮 标成 REQUEST_OUTCOME_UNKNOWN、可以续跑:重发一轮只花一次请求的 token,并计入续跑次数。 人工放弃(operator)仍然永久隔离。续跑资格检查不再被已放弃的 attempt 挡住。 都在调度程序进程里运行,下次重启生效。 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f74fe00aaf
commit
f83fa3948c
@ -558,7 +558,7 @@ class Execution:
|
||||
checkpoint=checkpoint, paused_at=time.time())
|
||||
return self._save(state, 'EXECUTION_PAUSED', reason=reason)
|
||||
|
||||
def resume(self, *, automatic=True):
|
||||
def resume(self, *, automatic=True, counted=True):
|
||||
with self.locked():
|
||||
state = self.snapshot()
|
||||
if self.epoch is not None and self.epoch != state['epoch']:
|
||||
@ -582,12 +582,12 @@ class Execution:
|
||||
if members != [] or pending:
|
||||
raise ExecutionStopped('SIDE_EFFECT_UNKNOWN' if members is None else 'WORK_STILL_ACTIVE')
|
||||
count = state.get('auto_resume_count', 0)
|
||||
if automatic and count >= state['config'].get('max_auto_resumes', 3):
|
||||
if automatic and counted and count >= state['config'].get('max_auto_resumes', 3):
|
||||
raise ExecutionStopped('AUTO_RESUME_BUDGET_EXHAUSTED')
|
||||
state['epoch'] += 1
|
||||
self.epoch = state['epoch']
|
||||
state.update(owner=process_identity(os.getpid()), execution_state='ACTIVE', stop_reason=None,
|
||||
auto_resume_count=count + int(automatic), resumed_at=time.time(), activity='WAITING_QUOTA')
|
||||
auto_resume_count=count + int(automatic and counted), resumed_at=time.time(), activity='WAITING_QUOTA')
|
||||
return self._save(state, 'EXECUTION_RESUMED', automatic=automatic)
|
||||
|
||||
def cleanup(self):
|
||||
|
||||
@ -560,7 +560,7 @@ def _resume_snapshot(run_id, job_ids=None, modes=None):
|
||||
cancel=bool(state.get('cancel_requested')), timeout_s=0)
|
||||
continue
|
||||
if (state['execution_state'] != 'PAUSED_INFRA' or state['cleanup_state'] == 'QUARANTINED'
|
||||
or state.get('cancel_requested') or state.get('auto_resume_count', 0) >= state['config'].get('max_auto_resumes', 3)):
|
||||
or state.get('cancel_requested')):
|
||||
continue
|
||||
try:
|
||||
resume_policy(state)
|
||||
|
||||
@ -29,6 +29,10 @@ from urllib.parse import urlsplit, quote
|
||||
import uuid
|
||||
|
||||
|
||||
# Failures proved to have sent nothing: wait and resend inside the broker instead of pausing the rollout.
|
||||
RETRY_UNSENT = ('CONNECT_FAILED', 'DNS_FAILED', 'DNS_TIMEOUT')
|
||||
|
||||
|
||||
class QuotaError(Exception):
|
||||
def __init__(self, reason: str, status: int = 503):
|
||||
self.reason, self.status = reason, status
|
||||
@ -584,7 +588,7 @@ class Broker:
|
||||
self._event(con, group, 'UNKNOWN_SETTLED_PROBE_ALLOWED')
|
||||
for attempt_id in stale: # outside the group flock: abandon_unknown takes it itself
|
||||
try:
|
||||
self.abandon_unknown(attempt_id, operator_reason='AUTO: local deadline + token window elapsed')
|
||||
self.abandon_unknown(attempt_id, operator_reason='AUTO: local deadline + token window elapsed', resumable=True)
|
||||
except QuotaError:
|
||||
pass # another broker settled it first
|
||||
|
||||
@ -600,11 +604,14 @@ class Broker:
|
||||
con.execute("UPDATE quota_groups SET state='COOLING',epoch=epoch+1,target=1,reason=NULL,cooldown_until=MAX(cooldown_until,?),updated_at=? WHERE group_id=?", (now, now, group))
|
||||
return True
|
||||
|
||||
def abandon_unknown(self, attempt_id: str, *, operator_reason: str):
|
||||
def abandon_unknown(self, attempt_id: str, *, operator_reason: str, resumable: bool = False):
|
||||
"""Explicit operator release of local capacity, never provider-end proof.
|
||||
|
||||
The old generation is fenced forever. Original usage and ended_at stay
|
||||
untouched. reconcile() also calls this once a token window has passed.
|
||||
The attempt stays unknown for audit (usage and ended_at untouched). An operator
|
||||
release fences the generation forever. reconcile() calls this with resumable=True
|
||||
once a token window has passed: the provider can no longer charge that window, so
|
||||
the turn may be replayed (one request's tokens, counted against the resume budget)
|
||||
instead of losing the rollout (10-13/14: 16 rollouts lost this way).
|
||||
"""
|
||||
if not isinstance(operator_reason, str) or not operator_reason.strip():
|
||||
raise ValueError('explicit operator reason required')
|
||||
@ -625,15 +632,19 @@ class Broker:
|
||||
execution_id = con.execute('SELECT execution_id FROM generations WHERE generation_id=?',
|
||||
(generation,)).fetchone()[0]
|
||||
con.execute("UPDATE attempts SET state='ABANDONED_UNKNOWN' WHERE attempt_id=?", (attempt_id,))
|
||||
con.execute("UPDATE generations SET state='ABANDONED_UNKNOWN',reason='OPERATOR_ABANDONED_UNKNOWN',updated_at=? WHERE generation_id=?", (now, generation))
|
||||
con.execute('UPDATE executions SET cancelled=1 WHERE execution_id=?', (execution_id,))
|
||||
if resumable:
|
||||
con.execute("UPDATE generations SET state='PAUSED',reason='REQUEST_OUTCOME_UNKNOWN',updated_at=? WHERE generation_id=?", (now, generation))
|
||||
else:
|
||||
con.execute("UPDATE generations SET state='ABANDONED_UNKNOWN',reason='OPERATOR_ABANDONED_UNKNOWN',updated_at=? WHERE generation_id=?", (now, generation))
|
||||
con.execute('UPDATE executions SET cancelled=1 WHERE execution_id=?', (execution_id,))
|
||||
probe_allowed = self._probe_if_settled(con, group)
|
||||
self._event(con, group, 'OPERATOR_ABANDONED_UNKNOWN', generation,
|
||||
attempt_id=attempt_id, operator_reason=operator_reason,
|
||||
released_at=now, provider_end_confirmed=False,
|
||||
original_reserved_tokens=row['reserved_tokens'],
|
||||
original_charged_tokens=row['charged_tokens'], probe_allowed=bool(probe_allowed))
|
||||
self._cancel_requested.setdefault(execution_id, threading.Event()).set()
|
||||
original_charged_tokens=row['charged_tokens'], probe_allowed=bool(probe_allowed), resumable=resumable)
|
||||
if not resumable:
|
||||
self._cancel_requested.setdefault(execution_id, threading.Event()).set()
|
||||
return True
|
||||
|
||||
def confirm_end(self, attempt_id: str, receipt_id: str):
|
||||
@ -668,15 +679,19 @@ class Broker:
|
||||
if not gen:
|
||||
return result | {'reason':'NO_PAUSED_GENERATION'}
|
||||
result['generation_id']=gen['generation_id']
|
||||
# Only a resume after a possibly delivered request spends the budget; connect/DNS failures and
|
||||
# waiting behind another request's unknown end sent nothing (10-14: 4 rollouts stuck at 3/3 on those).
|
||||
last=con.execute('SELECT sent FROM attempts WHERE generation_id=? ORDER BY started_at DESC LIMIT 1',(gen['generation_id'],)).fetchone()
|
||||
result['counted']=last is not None and last['sent'] != 0
|
||||
policy=self.groups[ex['group_id']]
|
||||
if (ex['cancelled'] or gen['state']!='PAUSED'
|
||||
# CONNECT_FAILED / REQUEST_DEADLINE_BEFORE_SEND / DNS_*: nothing was sent, replay is safe.
|
||||
# INCOMPLETE_GENERATION: a cut-off reply whose end the ledger recorded (the open-attempt check below still applies).
|
||||
or gen['reason'] not in ('QUOTA_WAIT_BUDGET_EXHAUSTED','REQUEST_OUTCOME_UNKNOWN','CONNECT_FAILED','REQUEST_DEADLINE_BEFORE_SEND','INCOMPLETE_GENERATION','DNS_FAILED','DNS_TIMEOUT')
|
||||
# A generation whose own request end is unknown never replays.
|
||||
or con.execute('SELECT 1 FROM attempts WHERE generation_id=? AND ended_at IS NULL',(gen['generation_id'],)).fetchone()):
|
||||
or con.execute("SELECT 1 FROM attempts WHERE generation_id=? AND ended_at IS NULL AND state <> 'ABANDONED_UNKNOWN'",(gen['generation_id'],)).fetchone()):
|
||||
return result | {'reason':'RESUME_NOT_ELIGIBLE'}
|
||||
if ex['resumes']>=policy.max_auto_resumes or gen['attempts']>=policy.max_attempts:
|
||||
if (result['counted'] and ex['resumes']>=policy.max_auto_resumes) or gen['attempts']>=policy.max_attempts:
|
||||
return result | {'reason':'RESUME_BUDGET_EXHAUSTED'}
|
||||
if gen['reserved_tokens']<=0:
|
||||
return result | {'reason':'RESERVATION_METADATA_MISSING'}
|
||||
@ -722,7 +737,7 @@ class Broker:
|
||||
readiness=self._resume_readiness(con,execution_id,self._now())
|
||||
if not readiness['eligible'] or readiness['generation_id']!=generation_id:
|
||||
raise QuotaError(readiness['reason'] if not readiness['eligible'] else 'GENERATION_MISMATCH',409)
|
||||
con.execute('UPDATE executions SET resumes=resumes+1 WHERE execution_id=?',(execution_id,))
|
||||
con.execute('UPDATE executions SET resumes=resumes+? WHERE execution_id=?',(int(readiness['counted']),execution_id))
|
||||
con.execute("UPDATE generations SET state='RESUMABLE',wait_since=? WHERE generation_id=?",(self._now(),generation_id))
|
||||
|
||||
def _begin(self, token, body, requested_id):
|
||||
@ -1044,12 +1059,37 @@ class Broker:
|
||||
watcher = threading.Thread(target=watch_client, daemon=True)
|
||||
watcher.start()
|
||||
try:
|
||||
return self._generate(scope, upstream, policy, generation, data, body, reserve, client_connected)
|
||||
while True:
|
||||
try:
|
||||
return self._generate(scope, upstream, policy, generation, data, body, reserve, client_connected)
|
||||
except QuotaError as exc:
|
||||
if exc.reason not in RETRY_UNSENT or not self._wait_unsent(upstream.quota_group, generation, policy):
|
||||
raise
|
||||
finally:
|
||||
stopped.set()
|
||||
if watcher is not None:
|
||||
watcher.join()
|
||||
|
||||
def _wait_unsent(self, group, generation, policy):
|
||||
"""Nothing reached the provider: wait here and resend rather than pause the whole rollout.
|
||||
|
||||
Each retry is charged to the execution's queue budget (the wait plus one
|
||||
connect/DNS deadline), so the client timeout bound holds; once the budget
|
||||
is spent the generation stays paused as before.
|
||||
"""
|
||||
if self._closing.wait(policy.cooldown_initial_s):
|
||||
return False
|
||||
charge = policy.cooldown_initial_s + policy.connect_timeout_s
|
||||
with self._group_lock(group), self._db() as con:
|
||||
con.execute('BEGIN IMMEDIATE')
|
||||
row = con.execute('SELECT g.state,g.execution_id,e.wait_spent,e.cancelled FROM generations g JOIN executions e ON e.execution_id=g.execution_id WHERE g.generation_id=?', (generation,)).fetchone()
|
||||
if row['state'] != 'PAUSED' or row['cancelled'] or row['wait_spent'] + charge > policy.wait_budget_s:
|
||||
return False
|
||||
con.execute('UPDATE executions SET wait_spent=wait_spent+? WHERE execution_id=?', (charge, row['execution_id']))
|
||||
con.execute("UPDATE generations SET state='QUEUED',updated_at=? WHERE generation_id=?", (self._now(), generation))
|
||||
self._event(con, group, 'UNSENT_RETRY', generation)
|
||||
return True
|
||||
|
||||
def _generate(self, scope, upstream, policy, generation, data, body, reserve, client_connected):
|
||||
url=urlsplit(upstream.base_url)
|
||||
started=time.monotonic()
|
||||
|
||||
@ -425,7 +425,7 @@ def resume_rollout(run_id: str, job_id: str, kk: int) -> dict:
|
||||
readiness = runtime.broker().resume_readiness(state['execution_id'])
|
||||
if not readiness['eligible']:
|
||||
raise ValueError('RESUME_NOT_READY: ' + str(readiness.get('reason')))
|
||||
execution.resume()
|
||||
execution.resume(counted=readiness.get('counted', True))
|
||||
try:
|
||||
runtime.broker().resume(state['execution_id'], checkpoint['generation_id'],
|
||||
checkpoint_complete=checkpoint.get('resumable', False), side_effects_known=True)
|
||||
|
||||
@ -359,6 +359,9 @@ class ExecutionTests(unittest.TestCase):
|
||||
self.ex.pause('quota', checkpoint)
|
||||
with self.assertRaisesRegex(execution.ExecutionStopped, 'AUTO_RESUME_BUDGET_EXHAUSTED'):
|
||||
execution.Execution(self.rd).resume()
|
||||
free = execution.Execution(self.rd)
|
||||
free.resume(counted=False) # a pause that sent nothing (connect/DNS failure) is free
|
||||
self.assertEqual(free.snapshot()['auto_resume_count'], 3)
|
||||
|
||||
@unittest.skipUnless(Path('/proc').is_dir(), 'Actual worker requires Linux owner identity')
|
||||
def test_worker_executes_real_protected_stage_and_reuses_receipt(self):
|
||||
|
||||
@ -579,6 +579,44 @@ class Quota(unittest.TestCase):
|
||||
self.assertEqual(self.request(scope,'g')[0],200)
|
||||
self.assertEqual(len(p.requests),1)
|
||||
|
||||
def test_connect_blip_is_resent_in_place_and_charged_to_the_wait_budget(self):
|
||||
# 10-14: one refused connect paused a 10-hour rollout; four of them stranded it at 3/3 resumes.
|
||||
p=self.provider();b=self.broker(p,self.policy())
|
||||
scope=b.issue_scope('blip','teacher','glm-test')
|
||||
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
|
||||
refusals=[2]
|
||||
def flaky(connection):
|
||||
if connection.port==p.server.server_port and refusals[0]:
|
||||
refusals[0]-=1
|
||||
raise ConnectionRefusedError('injected Wi-Fi blip')
|
||||
return original(connection)
|
||||
with patch('harness.quota.http.client.HTTPConnection.connect',new=flaky):
|
||||
self.assertEqual(self.request(scope)[0],200)
|
||||
self.assertEqual(len(p.requests),1)
|
||||
self.assertGreaterEqual(b.execution_status('blip')['wait_spent'],2*(.06+.2))
|
||||
self.assertEqual(len([e for e in b.snapshot()['quota_events'] if e['event']=='UNSENT_RETRY']),2)
|
||||
|
||||
def test_pause_that_sent_nothing_does_not_spend_the_resume_budget(self):
|
||||
p=self.provider();b=self.broker(p,self.policy())
|
||||
scope=b.issue_scope('free','teacher','glm-test')
|
||||
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
|
||||
def refused(connection):
|
||||
if connection.port==p.server.server_port:
|
||||
raise ConnectionRefusedError('injected outage')
|
||||
return original(connection)
|
||||
with patch('harness.quota.http.client.HTTPConnection.connect',new=refused):
|
||||
self.assertEqual(self.request(scope,'g')[0],503) # in-place retries spend the wait budget, then pause
|
||||
cap=b.groups['shared'].max_auto_resumes
|
||||
with b._db() as con:
|
||||
con.execute('UPDATE executions SET resumes=? WHERE execution_id=?',(cap,'free'))
|
||||
cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0]
|
||||
with patch.object(b,'_now',return_value=max(b._now(),cooldown)+1):
|
||||
ready=b.resume_readiness('free')
|
||||
self.assertTrue(ready['eligible']);self.assertFalse(ready['counted'])
|
||||
b.resume('free','g',checkpoint_complete=True,side_effects_known=True)
|
||||
self.assertEqual(b.execution_status('free')['resumes'],cap)
|
||||
self.assertEqual(p.requests,[])
|
||||
|
||||
def test_confirmed_connect_failure_does_not_spend_http_budget(self):
|
||||
p=self.provider();b=self.broker(p,self.policy(rpm=1,window_s=1))
|
||||
scope=b.issue_scope('connect-failure','teacher','glm-test')
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
"""Unknown request end self-heals one token window after its deadline;
|
||||
innocent generations paused by it resume, its own generation never replays."""
|
||||
"""Unknown request end self-heals one token window after its deadline: innocent
|
||||
generations paused by it resume for free, its own generation replays once at the
|
||||
cost of one resume instead of losing the rollout (10-14)."""
|
||||
import json
|
||||
import unittest
|
||||
|
||||
@ -16,7 +17,7 @@ class AutoAbandon(unittest.TestCase):
|
||||
unknown = Abandon.unknown
|
||||
|
||||
def test_unknown_waits_one_window_then_group_probes_and_victim_resumes(self):
|
||||
_, broker, _, attempt = self.unknown()
|
||||
_, broker, old_scope, attempt = self.unknown()
|
||||
# Paused at admission by the group, never sent anything itself.
|
||||
self.assertEqual(self.request(broker.issue_scope('victim', 'teacher', 'glm-test'))[0], 503)
|
||||
self.assertEqual(broker.resume_readiness('victim')['reason'], 'REQUEST_OUTCOME_UNKNOWN')
|
||||
@ -35,8 +36,13 @@ class AutoAbandon(unittest.TestCase):
|
||||
self.assertEqual(snap['groups'][0]['state'], 'COOLING')
|
||||
events = [e for e in snap['quota_events'] if e['event'] == 'OPERATOR_ABANDONED_UNKNOWN']
|
||||
self.assertTrue(json.loads(events[0]['detail'])['operator_reason'].startswith('AUTO'))
|
||||
self.assertTrue(broker.resume_readiness('victim')['eligible'])
|
||||
self.assertFalse(broker.resume_readiness('old')['eligible'])
|
||||
victim, old = broker.resume_readiness('victim'), broker.resume_readiness('old')
|
||||
self.assertTrue(victim['eligible']);self.assertFalse(victim['counted'])
|
||||
self.assertTrue(old['eligible']);self.assertTrue(old['counted'])
|
||||
self.assertEqual(broker.execution_status('old')['cancelled'], 0)
|
||||
broker.resume('old', 'old-g', checkpoint_complete=True, side_effects_known=True)
|
||||
self.assertEqual(broker.execution_status('old')['resumes'], 1)
|
||||
self.assertEqual(self.request(old_scope, 'old-g')[0], 200) # the replayed turn is delivered
|
||||
|
||||
broker.reconcile() # idempotent
|
||||
self.assertEqual(len([e for e in broker.snapshot()['quota_events']
|
||||
@ -60,7 +66,7 @@ class AutoAbandon(unittest.TestCase):
|
||||
broker.reconcile()
|
||||
self.assertEqual(state(), 'COOLING')
|
||||
self.assertTrue(broker.resume_readiness('victim')['eligible'])
|
||||
self.assertFalse(broker.resume_readiness('old')['eligible'])
|
||||
self.assertTrue(broker.resume_readiness('old')['eligible'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user