From f83fa3948c33bbf4449cf84d434b97c19d8a3fa8 Mon Sep 17 00:00:00 2001 From: bench Date: Wed, 14 Oct 2026 15:41:49 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BD=91=E7=BB=9C=E5=B0=8F=E6=95=85=E9=9A=9C?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E6=8B=96=E5=9E=AE=E6=95=B4=E6=9D=A1=E8=BD=A8?= =?UTF-8?q?=E8=BF=B9=EF=BC=9A=E5=8E=9F=E5=9C=B0=E9=87=8D=E5=8F=91=E3=80=81?= =?UTF-8?q?=E6=B2=A1=E5=8F=91=E5=87=BA=E5=8E=BB=E7=9A=84=E6=9A=82=E5=81=9C?= =?UTF-8?q?=E4=B8=8D=E6=89=A3=E7=BB=AD=E8=B7=91=E6=AC=A1=E6=95=B0=E3=80=81?= =?UTF-8?q?=E7=BB=93=E6=9E=9C=E4=B8=8D=E6=98=8E=E7=9A=84=E8=AF=B7=E6=B1=82?= =?UTF-8?q?=E5=8F=AF=E4=BB=A5=E7=BB=AD=E8=B7=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GLM 请求失败率约 0.3%,但一次失败就让整条轨迹暂停;3 次续跑次数被网络故障用完后, 轨迹只能挂到 24 小时上限(10-14 有 4 条);回复中途断开的请求过了等待期会被自动放弃, 同时整条执行被取消(10-13/14 共 16 条)。 A. 连接失败、域名解析失败(RETRY_UNSENT,确定什么都没发出去)在 broker 里等一会儿原地重发。 每次重试把等待时间加上一次连接时限计入排队预算 wait_spent,保持 客户端超时 = 排队预算 + 次数×单次时限 的上界;预算用完才照旧暂停。 B. 续跑次数只在最近一次请求可能已经送达时才扣(看账本最近一次 attempt 是否 sent)。 连接/解析失败、因为别的请求结果不明而在准入时被挡下的暂停,续跑不扣次数。 执行记录和账本两处计数同步,调度程序不再按原始次数预先过滤。 C. reconcile 自动放弃时(截止时间再过一个 token 窗口)不再取消执行,而是把这一轮 标成 REQUEST_OUTCOME_UNKNOWN、可以续跑:重发一轮只花一次请求的 token,并计入续跑次数。 人工放弃(operator)仍然永久隔离。续跑资格检查不再被已放弃的 attempt 挡住。 都在调度程序进程里运行,下次重启生效。 Co-Authored-By: Claude Opus 5.5 --- harness/execution.py | 6 +-- harness/pipeline.py | 2 +- harness/quota.py | 64 ++++++++++++++++++++++++------ harness/rollout.py | 2 +- scripts/test_execution.py | 3 ++ scripts/test_quota.py | 38 ++++++++++++++++++ scripts/test_quota_auto_abandon.py | 18 ++++++--- 7 files changed, 110 insertions(+), 23 deletions(-) diff --git a/harness/execution.py b/harness/execution.py index 3ada9c1..950f974 100644 --- a/harness/execution.py +++ b/harness/execution.py @@ -558,7 +558,7 @@ class Execution: checkpoint=checkpoint, paused_at=time.time()) return self._save(state, 'EXECUTION_PAUSED', reason=reason) - def resume(self, *, automatic=True): + def resume(self, *, automatic=True, counted=True): with self.locked(): state = self.snapshot() if self.epoch is not None and self.epoch != state['epoch']: @@ -582,12 +582,12 @@ class Execution: if members != [] or pending: raise ExecutionStopped('SIDE_EFFECT_UNKNOWN' if members is None else 'WORK_STILL_ACTIVE') count = state.get('auto_resume_count', 0) - if automatic and count >= state['config'].get('max_auto_resumes', 3): + if automatic and counted and count >= state['config'].get('max_auto_resumes', 3): raise ExecutionStopped('AUTO_RESUME_BUDGET_EXHAUSTED') state['epoch'] += 1 self.epoch = state['epoch'] state.update(owner=process_identity(os.getpid()), execution_state='ACTIVE', stop_reason=None, - auto_resume_count=count + int(automatic), resumed_at=time.time(), activity='WAITING_QUOTA') + auto_resume_count=count + int(automatic and counted), resumed_at=time.time(), activity='WAITING_QUOTA') return self._save(state, 'EXECUTION_RESUMED', automatic=automatic) def cleanup(self): diff --git a/harness/pipeline.py b/harness/pipeline.py index 96bb407..d8db137 100644 --- a/harness/pipeline.py +++ b/harness/pipeline.py @@ -560,7 +560,7 @@ def _resume_snapshot(run_id, job_ids=None, modes=None): cancel=bool(state.get('cancel_requested')), timeout_s=0) continue if (state['execution_state'] != 'PAUSED_INFRA' or state['cleanup_state'] == 'QUARANTINED' - or state.get('cancel_requested') or state.get('auto_resume_count', 0) >= state['config'].get('max_auto_resumes', 3)): + or state.get('cancel_requested')): continue try: resume_policy(state) diff --git a/harness/quota.py b/harness/quota.py index 55ecd88..90be265 100644 --- a/harness/quota.py +++ b/harness/quota.py @@ -29,6 +29,10 @@ from urllib.parse import urlsplit, quote import uuid +# Failures proved to have sent nothing: wait and resend inside the broker instead of pausing the rollout. +RETRY_UNSENT = ('CONNECT_FAILED', 'DNS_FAILED', 'DNS_TIMEOUT') + + class QuotaError(Exception): def __init__(self, reason: str, status: int = 503): self.reason, self.status = reason, status @@ -584,7 +588,7 @@ class Broker: self._event(con, group, 'UNKNOWN_SETTLED_PROBE_ALLOWED') for attempt_id in stale: # outside the group flock: abandon_unknown takes it itself try: - self.abandon_unknown(attempt_id, operator_reason='AUTO: local deadline + token window elapsed') + self.abandon_unknown(attempt_id, operator_reason='AUTO: local deadline + token window elapsed', resumable=True) except QuotaError: pass # another broker settled it first @@ -600,11 +604,14 @@ class Broker: con.execute("UPDATE quota_groups SET state='COOLING',epoch=epoch+1,target=1,reason=NULL,cooldown_until=MAX(cooldown_until,?),updated_at=? WHERE group_id=?", (now, now, group)) return True - def abandon_unknown(self, attempt_id: str, *, operator_reason: str): + def abandon_unknown(self, attempt_id: str, *, operator_reason: str, resumable: bool = False): """Explicit operator release of local capacity, never provider-end proof. - The old generation is fenced forever. Original usage and ended_at stay - untouched. reconcile() also calls this once a token window has passed. + The attempt stays unknown for audit (usage and ended_at untouched). An operator + release fences the generation forever. reconcile() calls this with resumable=True + once a token window has passed: the provider can no longer charge that window, so + the turn may be replayed (one request's tokens, counted against the resume budget) + instead of losing the rollout (10-13/14: 16 rollouts lost this way). """ if not isinstance(operator_reason, str) or not operator_reason.strip(): raise ValueError('explicit operator reason required') @@ -625,15 +632,19 @@ class Broker: execution_id = con.execute('SELECT execution_id FROM generations WHERE generation_id=?', (generation,)).fetchone()[0] con.execute("UPDATE attempts SET state='ABANDONED_UNKNOWN' WHERE attempt_id=?", (attempt_id,)) - con.execute("UPDATE generations SET state='ABANDONED_UNKNOWN',reason='OPERATOR_ABANDONED_UNKNOWN',updated_at=? WHERE generation_id=?", (now, generation)) - con.execute('UPDATE executions SET cancelled=1 WHERE execution_id=?', (execution_id,)) + if resumable: + con.execute("UPDATE generations SET state='PAUSED',reason='REQUEST_OUTCOME_UNKNOWN',updated_at=? WHERE generation_id=?", (now, generation)) + else: + con.execute("UPDATE generations SET state='ABANDONED_UNKNOWN',reason='OPERATOR_ABANDONED_UNKNOWN',updated_at=? WHERE generation_id=?", (now, generation)) + con.execute('UPDATE executions SET cancelled=1 WHERE execution_id=?', (execution_id,)) probe_allowed = self._probe_if_settled(con, group) self._event(con, group, 'OPERATOR_ABANDONED_UNKNOWN', generation, attempt_id=attempt_id, operator_reason=operator_reason, released_at=now, provider_end_confirmed=False, original_reserved_tokens=row['reserved_tokens'], - original_charged_tokens=row['charged_tokens'], probe_allowed=bool(probe_allowed)) - self._cancel_requested.setdefault(execution_id, threading.Event()).set() + original_charged_tokens=row['charged_tokens'], probe_allowed=bool(probe_allowed), resumable=resumable) + if not resumable: + self._cancel_requested.setdefault(execution_id, threading.Event()).set() return True def confirm_end(self, attempt_id: str, receipt_id: str): @@ -668,15 +679,19 @@ class Broker: if not gen: return result | {'reason':'NO_PAUSED_GENERATION'} result['generation_id']=gen['generation_id'] + # Only a resume after a possibly delivered request spends the budget; connect/DNS failures and + # waiting behind another request's unknown end sent nothing (10-14: 4 rollouts stuck at 3/3 on those). + last=con.execute('SELECT sent FROM attempts WHERE generation_id=? ORDER BY started_at DESC LIMIT 1',(gen['generation_id'],)).fetchone() + result['counted']=last is not None and last['sent'] != 0 policy=self.groups[ex['group_id']] if (ex['cancelled'] or gen['state']!='PAUSED' # CONNECT_FAILED / REQUEST_DEADLINE_BEFORE_SEND / DNS_*: nothing was sent, replay is safe. # INCOMPLETE_GENERATION: a cut-off reply whose end the ledger recorded (the open-attempt check below still applies). or gen['reason'] not in ('QUOTA_WAIT_BUDGET_EXHAUSTED','REQUEST_OUTCOME_UNKNOWN','CONNECT_FAILED','REQUEST_DEADLINE_BEFORE_SEND','INCOMPLETE_GENERATION','DNS_FAILED','DNS_TIMEOUT') # A generation whose own request end is unknown never replays. - or con.execute('SELECT 1 FROM attempts WHERE generation_id=? AND ended_at IS NULL',(gen['generation_id'],)).fetchone()): + or con.execute("SELECT 1 FROM attempts WHERE generation_id=? AND ended_at IS NULL AND state <> 'ABANDONED_UNKNOWN'",(gen['generation_id'],)).fetchone()): return result | {'reason':'RESUME_NOT_ELIGIBLE'} - if ex['resumes']>=policy.max_auto_resumes or gen['attempts']>=policy.max_attempts: + if (result['counted'] and ex['resumes']>=policy.max_auto_resumes) or gen['attempts']>=policy.max_attempts: return result | {'reason':'RESUME_BUDGET_EXHAUSTED'} if gen['reserved_tokens']<=0: return result | {'reason':'RESERVATION_METADATA_MISSING'} @@ -722,7 +737,7 @@ class Broker: readiness=self._resume_readiness(con,execution_id,self._now()) if not readiness['eligible'] or readiness['generation_id']!=generation_id: raise QuotaError(readiness['reason'] if not readiness['eligible'] else 'GENERATION_MISMATCH',409) - con.execute('UPDATE executions SET resumes=resumes+1 WHERE execution_id=?',(execution_id,)) + con.execute('UPDATE executions SET resumes=resumes+? WHERE execution_id=?',(int(readiness['counted']),execution_id)) con.execute("UPDATE generations SET state='RESUMABLE',wait_since=? WHERE generation_id=?",(self._now(),generation_id)) def _begin(self, token, body, requested_id): @@ -1044,12 +1059,37 @@ class Broker: watcher = threading.Thread(target=watch_client, daemon=True) watcher.start() try: - return self._generate(scope, upstream, policy, generation, data, body, reserve, client_connected) + while True: + try: + return self._generate(scope, upstream, policy, generation, data, body, reserve, client_connected) + except QuotaError as exc: + if exc.reason not in RETRY_UNSENT or not self._wait_unsent(upstream.quota_group, generation, policy): + raise finally: stopped.set() if watcher is not None: watcher.join() + def _wait_unsent(self, group, generation, policy): + """Nothing reached the provider: wait here and resend rather than pause the whole rollout. + + Each retry is charged to the execution's queue budget (the wait plus one + connect/DNS deadline), so the client timeout bound holds; once the budget + is spent the generation stays paused as before. + """ + if self._closing.wait(policy.cooldown_initial_s): + return False + charge = policy.cooldown_initial_s + policy.connect_timeout_s + with self._group_lock(group), self._db() as con: + con.execute('BEGIN IMMEDIATE') + row = con.execute('SELECT g.state,g.execution_id,e.wait_spent,e.cancelled FROM generations g JOIN executions e ON e.execution_id=g.execution_id WHERE g.generation_id=?', (generation,)).fetchone() + if row['state'] != 'PAUSED' or row['cancelled'] or row['wait_spent'] + charge > policy.wait_budget_s: + return False + con.execute('UPDATE executions SET wait_spent=wait_spent+? WHERE execution_id=?', (charge, row['execution_id'])) + con.execute("UPDATE generations SET state='QUEUED',updated_at=? WHERE generation_id=?", (self._now(), generation)) + self._event(con, group, 'UNSENT_RETRY', generation) + return True + def _generate(self, scope, upstream, policy, generation, data, body, reserve, client_connected): url=urlsplit(upstream.base_url) started=time.monotonic() diff --git a/harness/rollout.py b/harness/rollout.py index 8f3f4e2..5ba8800 100644 --- a/harness/rollout.py +++ b/harness/rollout.py @@ -425,7 +425,7 @@ def resume_rollout(run_id: str, job_id: str, kk: int) -> dict: readiness = runtime.broker().resume_readiness(state['execution_id']) if not readiness['eligible']: raise ValueError('RESUME_NOT_READY: ' + str(readiness.get('reason'))) - execution.resume() + execution.resume(counted=readiness.get('counted', True)) try: runtime.broker().resume(state['execution_id'], checkpoint['generation_id'], checkpoint_complete=checkpoint.get('resumable', False), side_effects_known=True) diff --git a/scripts/test_execution.py b/scripts/test_execution.py index 6ddef97..1bb2181 100644 --- a/scripts/test_execution.py +++ b/scripts/test_execution.py @@ -359,6 +359,9 @@ class ExecutionTests(unittest.TestCase): self.ex.pause('quota', checkpoint) with self.assertRaisesRegex(execution.ExecutionStopped, 'AUTO_RESUME_BUDGET_EXHAUSTED'): execution.Execution(self.rd).resume() + free = execution.Execution(self.rd) + free.resume(counted=False) # a pause that sent nothing (connect/DNS failure) is free + self.assertEqual(free.snapshot()['auto_resume_count'], 3) @unittest.skipUnless(Path('/proc').is_dir(), 'Actual worker requires Linux owner identity') def test_worker_executes_real_protected_stage_and_reuses_receipt(self): diff --git a/scripts/test_quota.py b/scripts/test_quota.py index 3a85e4b..0ca8f16 100644 --- a/scripts/test_quota.py +++ b/scripts/test_quota.py @@ -579,6 +579,44 @@ class Quota(unittest.TestCase): self.assertEqual(self.request(scope,'g')[0],200) self.assertEqual(len(p.requests),1) + def test_connect_blip_is_resent_in_place_and_charged_to_the_wait_budget(self): + # 10-14: one refused connect paused a 10-hour rollout; four of them stranded it at 3/3 resumes. + p=self.provider();b=self.broker(p,self.policy()) + scope=b.issue_scope('blip','teacher','glm-test') + original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect + refusals=[2] + def flaky(connection): + if connection.port==p.server.server_port and refusals[0]: + refusals[0]-=1 + raise ConnectionRefusedError('injected Wi-Fi blip') + return original(connection) + with patch('harness.quota.http.client.HTTPConnection.connect',new=flaky): + self.assertEqual(self.request(scope)[0],200) + self.assertEqual(len(p.requests),1) + self.assertGreaterEqual(b.execution_status('blip')['wait_spent'],2*(.06+.2)) + self.assertEqual(len([e for e in b.snapshot()['quota_events'] if e['event']=='UNSENT_RETRY']),2) + + def test_pause_that_sent_nothing_does_not_spend_the_resume_budget(self): + p=self.provider();b=self.broker(p,self.policy()) + scope=b.issue_scope('free','teacher','glm-test') + original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect + def refused(connection): + if connection.port==p.server.server_port: + raise ConnectionRefusedError('injected outage') + return original(connection) + with patch('harness.quota.http.client.HTTPConnection.connect',new=refused): + self.assertEqual(self.request(scope,'g')[0],503) # in-place retries spend the wait budget, then pause + cap=b.groups['shared'].max_auto_resumes + with b._db() as con: + con.execute('UPDATE executions SET resumes=? WHERE execution_id=?',(cap,'free')) + cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0] + with patch.object(b,'_now',return_value=max(b._now(),cooldown)+1): + ready=b.resume_readiness('free') + self.assertTrue(ready['eligible']);self.assertFalse(ready['counted']) + b.resume('free','g',checkpoint_complete=True,side_effects_known=True) + self.assertEqual(b.execution_status('free')['resumes'],cap) + self.assertEqual(p.requests,[]) + def test_confirmed_connect_failure_does_not_spend_http_budget(self): p=self.provider();b=self.broker(p,self.policy(rpm=1,window_s=1)) scope=b.issue_scope('connect-failure','teacher','glm-test') diff --git a/scripts/test_quota_auto_abandon.py b/scripts/test_quota_auto_abandon.py index fbef106..5041c16 100644 --- a/scripts/test_quota_auto_abandon.py +++ b/scripts/test_quota_auto_abandon.py @@ -1,5 +1,6 @@ -"""Unknown request end self-heals one token window after its deadline; -innocent generations paused by it resume, its own generation never replays.""" +"""Unknown request end self-heals one token window after its deadline: innocent +generations paused by it resume for free, its own generation replays once at the +cost of one resume instead of losing the rollout (10-14).""" import json import unittest @@ -16,7 +17,7 @@ class AutoAbandon(unittest.TestCase): unknown = Abandon.unknown def test_unknown_waits_one_window_then_group_probes_and_victim_resumes(self): - _, broker, _, attempt = self.unknown() + _, broker, old_scope, attempt = self.unknown() # Paused at admission by the group, never sent anything itself. self.assertEqual(self.request(broker.issue_scope('victim', 'teacher', 'glm-test'))[0], 503) self.assertEqual(broker.resume_readiness('victim')['reason'], 'REQUEST_OUTCOME_UNKNOWN') @@ -35,8 +36,13 @@ class AutoAbandon(unittest.TestCase): self.assertEqual(snap['groups'][0]['state'], 'COOLING') events = [e for e in snap['quota_events'] if e['event'] == 'OPERATOR_ABANDONED_UNKNOWN'] self.assertTrue(json.loads(events[0]['detail'])['operator_reason'].startswith('AUTO')) - self.assertTrue(broker.resume_readiness('victim')['eligible']) - self.assertFalse(broker.resume_readiness('old')['eligible']) + victim, old = broker.resume_readiness('victim'), broker.resume_readiness('old') + self.assertTrue(victim['eligible']);self.assertFalse(victim['counted']) + self.assertTrue(old['eligible']);self.assertTrue(old['counted']) + self.assertEqual(broker.execution_status('old')['cancelled'], 0) + broker.resume('old', 'old-g', checkpoint_complete=True, side_effects_known=True) + self.assertEqual(broker.execution_status('old')['resumes'], 1) + self.assertEqual(self.request(old_scope, 'old-g')[0], 200) # the replayed turn is delivered broker.reconcile() # idempotent self.assertEqual(len([e for e in broker.snapshot()['quota_events'] @@ -60,7 +66,7 @@ class AutoAbandon(unittest.TestCase): broker.reconcile() self.assertEqual(state(), 'COOLING') self.assertTrue(broker.resume_readiness('victim')['eligible']) - self.assertFalse(broker.resume_readiness('old')['eligible']) + self.assertTrue(broker.resume_readiness('old')['eligible']) if __name__ == '__main__':