data-gen-bpu/scripts/test_quota.py
bench f83fa3948c 网络小故障不再拖垮整条轨迹:原地重发、没发出去的暂停不扣续跑次数、结果不明的请求可以续跑
GLM 请求失败率约 0.3%,但一次失败就让整条轨迹暂停;3 次续跑次数被网络故障用完后,
轨迹只能挂到 24 小时上限(10-14 有 4 条);回复中途断开的请求过了等待期会被自动放弃,
同时整条执行被取消(10-13/14 共 16 条)。

A. 连接失败、域名解析失败(RETRY_UNSENT,确定什么都没发出去)在 broker 里等一会儿原地重发。
   每次重试把等待时间加上一次连接时限计入排队预算 wait_spent,保持
   客户端超时 = 排队预算 + 次数×单次时限 的上界;预算用完才照旧暂停。
B. 续跑次数只在最近一次请求可能已经送达时才扣(看账本最近一次 attempt 是否 sent)。
   连接/解析失败、因为别的请求结果不明而在准入时被挡下的暂停,续跑不扣次数。
   执行记录和账本两处计数同步,调度程序不再按原始次数预先过滤。
C. reconcile 自动放弃时(截止时间再过一个 token 窗口)不再取消执行,而是把这一轮
   标成 REQUEST_OUTCOME_UNKNOWN、可以续跑:重发一轮只花一次请求的 token,并计入续跑次数。
   人工放弃(operator)仍然永久隔离。续跑资格检查不再被已放弃的 attempt 挡住。
都在调度程序进程里运行,下次重启生效。

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 15:41:49 +08:00

677 lines
40 KiB
Python

"""Quota acceptance uses real local HTTP, persistent ledgers and no provider account."""
from concurrent.futures import ThreadPoolExecutor
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import multiprocessing
import socket
import hashlib
from pathlib import Path
import tempfile
import threading
import time
import unittest
from unittest.mock import patch
import urllib.error
import urllib.request
from harness.quota import Broker, QuotaPolicy, Upstream, QuotaError, quota_snapshot, validate_completion
class Provider:
def __init__(self, responder):
self.responder, self.requests = responder, []
outer = self
class Handler(BaseHTTPRequestHandler):
def log_message(self, *_): pass
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
outer.requests.append({'at': time.time(), 'body': body,
'authorization': self.headers.get('Authorization')})
code, headers, payload = outer.responder(len(outer.requests), body)
self.send_response(code)
for key, value in headers.items(): self.send_header(key, str(value))
self.end_headers()
try:
if callable(payload): payload(self.wfile)
else: self.wfile.write(payload)
except (BrokenPipeError, ConnectionResetError): pass
self.server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
self.thread.start()
self.url = 'http://127.0.0.1:%s/v1' % self.server.server_port
def close(self):
self.server.shutdown(); self.server.server_close(); self.thread.join()
def ok(_n=0, body=None):
if body and body.get('stream'):
return 200, {'Content-Type': 'text/event-stream'}, (b'data: {"choices":[{"delta":{"content":"ok"},"finish_reason":null}]}\n\n'
b'data: {"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}\n\n'
b'data: [DONE]\n\n')
return 200, {'Content-Type': 'application/json'}, b'{"choices":[{"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}'
def _process_broker(ledger, url, policy, execution, ready, start, result):
# Spawned interpreters do not inherit the parent's locks or server objects.
with Broker(Path(ledger), {'shared':policy}, {'teacher':Upstream('shared',url,'upstream-secret',('glm-test',))}) as broker:
scope=broker.issue_scope(execution,'teacher','glm-test')
ready.put(True)
if not start.wait(5):
result.put('START_TIMEOUT');return
request=urllib.request.Request(scope.base_url+'/chat/completions',
json.dumps({'model':'glm-test','messages':[],'max_tokens':5}).encode(),
{'Authorization':'Bearer '+scope.api_key,'Content-Type':'application/json'})
try:
with urllib.request.urlopen(request,timeout=5) as response:
response.read();result.put(response.status)
except urllib.error.HTTPError as error:
error.read();result.put(error.code)
class Quota(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
self.ledger = Path(self.tmp.name) / 'shared.sqlite'
def provider(self, responder=ok):
item = Provider(responder); self.addCleanup(item.close); return item
def policy(self, **kw):
return QuotaPolicy(**dict({'max_concurrency':2, 'rpm':1000, 'tpm':1000000,
'window_s':.2, 'cooldown_initial_s':.06, 'cooldown_local_max_s':.1,
'jitter_fraction':0, 'recovery_window_s':.12, 'recovery_min_successes':2,
'wait_budget_s':2, 'connect_timeout_s':.2, 'first_byte_timeout_s':.3,
'stream_idle_timeout_s':.15, 'request_total_s':.6, 'poll_s':.01}, **kw))
def broker(self, provider, policy=None, aliases=None):
b = Broker(self.ledger, {'shared':policy or self.policy()}, aliases or {
'teacher':Upstream('shared', provider.url, 'upstream-secret', ('glm-test',))})
b.start(); self.addCleanup(b.close); return b
def request(self, scope, generation=None, body=None):
headers = {'Authorization':'Bearer '+scope.api_key, 'Content-Type':'application/json'}
if generation: headers['X-Bench-Generation-Id'] = generation
request = urllib.request.Request(scope.base_url+'/chat/completions',
json.dumps(body or {'model':'glm-test', 'messages':[{'role':'user','content':'hi'}], 'max_tokens':5}).encode(), headers)
try:
with urllib.request.urlopen(request, timeout=5) as response:
return response.status, response.read()
except urllib.error.HTTPError as error:
return error.code, error.read()
def test_success_scope_and_stream_usage(self):
p=self.provider(); b=self.broker(p); scope=b.issue_scope('execution', 'teacher', 'glm-test')
code, data=self.request(scope, 'generation', {'model':'glm-test','messages':[],'max_tokens':4,'stream':True})
self.assertEqual(code, 200); self.assertIn(b'[DONE]', data)
self.assertEqual(p.requests[0]['authorization'], 'Bearer upstream-secret')
self.assertNotIn('upstream-secret', scope.api_key)
self.assertEqual(b.snapshot()['attempts'][0]['charged_tokens'], 3)
self.assertEqual(len(p.requests), 1)
self.assertEqual(self.request(scope, 'generation')[0], 409)
def _raw_request(self, scope):
from urllib.parse import urlsplit
url = urlsplit(scope.base_url)
sock = socket.create_connection((url.hostname, url.port), timeout=2)
body = json.dumps({'model':'glm-test', 'messages':[], 'max_tokens':5}).encode()
sock.sendall((f'POST /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\n'
f'Authorization: Bearer {scope.api_key}\r\nContent-Length: {len(body)}\r\n'
'Content-Type: application/json\r\n\r\n').encode() + body)
return sock
def test_disconnected_queued_client_never_reaches_upstream(self):
p = self.provider(); b = self.broker(p, self.policy(rpm=1, window_s=.4))
self.assertEqual(self.request(b.issue_scope('first', 'teacher', 'glm-test'))[0], 200)
scope = b.issue_scope('disconnected', 'teacher', 'glm-test')
sock = self._raw_request(scope)
deadline = time.monotonic() + 1
while not b.execution_status('disconnected')['generations'] and time.monotonic() < deadline:
time.sleep(.005)
self.assertTrue(b.execution_status('disconnected')['generations'])
sock.close()
deadline = time.monotonic() + 1
while not b.execution_status('disconnected')['cancelled'] and time.monotonic() < deadline:
time.sleep(.005)
self.assertTrue(b.execution_status('disconnected')['cancelled'])
time.sleep(.45) # RPM capacity becomes free after the client has gone.
self.assertEqual(len(p.requests), 1)
gen = b.execution_status('disconnected')['generations'][0]
self.assertEqual((gen['state'], gen['attempts']), ('CANCELLED', 0))
self.assertEqual(b.snapshot()['groups'][0]['state'], 'OPEN')
def test_disconnected_sent_client_keeps_unknown_reservation(self):
release = threading.Event()
self.addCleanup(release.set)
p = self.provider(lambda *_: (release.wait(2) or True) and ok())
b = self.broker(p)
sock = self._raw_request(b.issue_scope('disconnected', 'teacher', 'glm-test'))
deadline = time.monotonic() + 1
while not p.requests and time.monotonic() < deadline:
time.sleep(.005)
self.assertEqual(len(p.requests), 1)
sock.close()
deadline = time.monotonic() + 1
while b.snapshot()['attempts'][0]['state'] != 'UNKNOWN' and time.monotonic() < deadline:
time.sleep(.005)
attempt = b.snapshot()['attempts'][0]
self.assertEqual(attempt['state'], 'UNKNOWN')
self.assertIsNone(attempt['ended_at'])
self.assertGreater(attempt['charged_tokens'], 0)
self.assertTrue(b.execution_status('disconnected')['cancelled'])
self.assertEqual(len(p.requests), 1)
release.set()
def test_valid_sse_done_does_not_wait_for_http_close(self):
for crlf in (False, True):
with self.subTest(crlf=crlf), tempfile.TemporaryDirectory() as temp:
payload = ok(body={'stream':True})[2]
if crlf:
payload = payload.replace(b'\n', b'\r\n')
release = threading.Event()
def send(writer):
# Split inside the terminator to exercise real read boundaries.
cut = payload.index(b'[DONE]') + 3
writer.write(payload[:cut]); writer.flush()
time.sleep(.02)
writer.write(payload[cut:]); writer.flush()
release.wait(2)
provider = self.provider(lambda *_: (200, {'Content-Type':'text/event-stream'}, send))
with Broker(Path(temp)/'quota.sqlite', {'shared':self.policy()}, {
'teacher':Upstream('shared',provider.url,'fake-only',('glm-test',))}) as broker:
try:
scope = broker.issue_scope('execution','teacher','glm-test')
code, data = self.request(scope,'generation',{'model':'glm-test','messages':[], 'max_tokens':4,'stream':True})
self.assertEqual((code,data),(200,payload))
attempt = broker.snapshot()['attempts'][0]
self.assertEqual((attempt['state'],attempt['reason'],attempt['charged_tokens']),('FINISHED','COMPLETE',3))
self.assertEqual(broker.snapshot()['groups'][0]['state'],'OPEN')
self.assertEqual(len(provider.requests),1)
finally:
release.set()
def test_sse_terminator_cannot_complete_invalid_or_unframed_response(self):
complete = ok(body={'stream':True})[2]
variants = {
'missing_blank_line': complete[:-2],
'content_literal_only': b'data: {"choices":[{"delta":{"content":"[DONE]"},"finish_reason":"stop"}]}\n\n',
'missing_finish': complete.split(b'data: {"choices":[{"delta":{}')[0] + b'data: [DONE]\n\n',
'partial_tool': b'data: {"choices":[{"delta":{"tool_calls":[{"id":"x","type":"function","function":{"name":"write","arguments":"{"}}]},"finish_reason":"tool_calls"}]}\n\ndata: [DONE]\n\n',
'extra_data_after_done': complete + b'data: {"choices":[]}\n\n',
}
for name,payload in variants.items():
with self.subTest(name=name), tempfile.TemporaryDirectory() as temp:
release = threading.Event()
def send(writer):
writer.write(payload); writer.flush(); release.wait(2)
provider = self.provider(lambda *_: (200, {'Content-Type':'text/event-stream'},send))
with Broker(Path(temp)/'quota.sqlite',{'shared':self.policy()}, {
'teacher':Upstream('shared',provider.url,'fake-only',('glm-test',))}) as broker:
try:
scope = broker.issue_scope('execution','teacher','glm-test')
code,_ = self.request(scope,'generation',{'model':'glm-test','messages':[],'max_tokens':4,'stream':True})
self.assertNotEqual(code,200)
attempt = broker.snapshot()['attempts'][0]
self.assertEqual((attempt['state'],attempt['transport_reason']),('UNKNOWN','STREAM_IDLE_TIMEOUT'))
self.assertEqual(len(provider.requests),1)
finally:
release.set()
def test_JOINT001_persistent_429_budget_and_single_probe(self):
p=self.provider(lambda n, body:(429, {'Retry-After':'.06'}, b'{"error":{"code":"rate_limit"}}'))
b=self.broker(p, self.policy(max_attempts=4))
scopes=[b.issue_scope('e'+str(i),'teacher','glm-test') for i in range(4)]
with ThreadPoolExecutor(max_workers=4) as pool:
results=list(pool.map(lambda s:self.request(s),scopes))
snapshot=b.snapshot()
self.assertTrue(all(code==503 for code,_ in results))
self.assertTrue(all(g['attempts']<=4 for g in snapshot['generations']))
self.assertTrue(all(t['state_at_send'] in ('OPEN','HALF_OPEN','RECOVERING') for t in snapshot['attempts']))
self.assertEqual(len(p.requests), len(snapshot['attempts']))
probes=[a for a in snapshot['attempts'] if a['state_at_send']=='HALF_OPEN']
for before,after in zip(probes,probes[1:]):
self.assertGreaterEqual(after['started_at'],before['ended_at'])
self.assertEqual(len({a['attempt_id'] for a in snapshot['attempts']}),len(snapshot['attempts']))
def test_QUOTA001_shared_alias_group_independent_group(self):
p=self.provider(lambda n, body:(429, {'Retry-After':'1'}, b'{}') if body['model']=='limited' else ok(n,body))
b=Broker(self.ledger, {'shared':self.policy(wait_budget_s=.12), 'independent':self.policy()}, {
'a':Upstream('shared',p.url,'a',('limited',)), 'b':Upstream('shared',p.url,'b',('limited',)),
'other':Upstream('independent',p.url,'c',('free',))})
b.start(); self.addCleanup(b.close)
self.request(b.issue_scope('a','a','limited'),body={'model':'limited','messages':[],'max_tokens':1})
self.request(b.issue_scope('b','b','limited'),body={'model':'limited','messages':[],'max_tokens':1})
self.assertEqual(len(p.requests),1)
self.assertEqual(self.request(b.issue_scope('c','other','free'),body={'model':'free','messages':[],'max_tokens':1})[0],200)
def test_JOINT003_restart_keeps_budget_and_missing_usage(self):
p=self.provider(lambda n,body:(200,{'Content-Type':'application/json'},b'{"choices":[{"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}'))
b=self.broker(p,self.policy(rpm=1,window_s=1,wait_budget_s=.06))
self.assertEqual(self.request(b.issue_scope('e1','teacher','glm-test'))[0],200)
first=b.snapshot()['attempts'][0]
self.assertEqual(first['charged_tokens'],first['reserved_tokens'])
b.close()
b2=self.broker(p,self.policy(rpm=1,window_s=1,wait_budget_s=.06))
self.assertEqual(self.request(b2.issue_scope('e2','teacher','glm-test'))[0],503)
self.assertEqual(len(p.requests),1)
def test_cancel_blocks_transport_without_hitting_upstream(self):
p=self.provider(); b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
b.cancel('e'); self.assertEqual(self.request(s)[0],409); self.assertEqual(p.requests,[])
def test_QUOTA002_retry_after_and_auth_pause(self):
p=self.provider(lambda n,body:(429,{'Retry-After':'4'},b'{}'))
b=self.broker(p,self.policy(wait_budget_s=.1)); before=time.time()
self.request(b.issue_scope('e','teacher','glm-test'))
self.assertGreaterEqual(b.snapshot()['groups'][0]['cooldown_until'],before+4)
self.assertEqual(len(p.requests),1)
def test_JOINT004_nonempty_stream_interruption_does_not_retry(self):
p=self.provider(lambda n,body:(200,{'Content-Type':'text/event-stream'}, b'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'))
b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
self.assertEqual(self.request(s,body={'model':'glm-test','messages':[],'max_tokens':5,'stream':True})[0],503)
self.assertEqual(len(p.requests),1)
self.assertEqual(b.snapshot()['generations'][0]['reason'],'INCOMPLETE_GENERATION')
self.assertEqual(self.request(s,'new-id')[0],409)
def test_JOINT002_recovery_adds_at_most_one_per_window(self):
p=self.provider(lambda n,body:(429,{},b'{}') if n==1 else ok(n,body))
policy=self.policy(max_concurrency=4,recovery_window_s=.12)
b=self.broker(p,policy)
self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'))[0],200)
self.assertEqual(b.snapshot()['groups'][0]['state'],'RECOVERING')
self.assertEqual(b.snapshot()['groups'][0]['target'],1)
clock=[b.snapshot()['groups'][0]['recovery_at']]
# Only quota's durable clock is controlled. HTTP itself remains real.
with patch.object(b,'_now',side_effect=lambda:clock[0]):
for i in range(3):
self.assertEqual(self.request(b.issue_scope('e'+str(i),'teacher','glm-test'))[0],200)
self.assertEqual(b.snapshot()['groups'][0]['target'],1)
clock[0]+=.13
self.request(b.issue_scope('next','teacher','glm-test'))
self.assertEqual(b.snapshot()['groups'][0]['target'],1)
self.request(b.issue_scope('next2','teacher','glm-test'))
self.assertEqual(b.snapshot()['groups'][0]['target'],2)
self.request(b.issue_scope('after','teacher','glm-test'))
self.assertEqual(b.snapshot()['groups'][0]['target'],2)
def test_shared_brokers_enforce_actual_RPM(self):
p=self.provider(); policy=self.policy(rpm=1,window_s=.7,wait_budget_s=.12)
a=self.broker(p,policy); b=self.broker(p,policy)
scopes=[a.issue_scope('a','teacher','glm-test'),b.issue_scope('b','teacher','glm-test')]
with ThreadPoolExecutor(max_workers=2) as pool:
results=list(pool.map(self.request,scopes))
self.assertEqual(sorted(code for code,_ in results),[200,503])
self.assertEqual(len(p.requests),1)
self.assertEqual(len(a.snapshot()['attempts']),1)
def test_missing_usage_reservation_blocks_TPM_until_end_window(self):
p=self.provider(lambda n,body:(200,{},b'{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}]}'))
b=self.broker(p,self.policy(tpm=300,window_s=.7,wait_budget_s=.08))
self.assertEqual(self.request(b.issue_scope('a','teacher','glm-test'))[0],200)
self.assertEqual(self.request(b.issue_scope('b','teacher','glm-test'))[0],503)
self.assertEqual(len(p.requests),1)
def test_QUOTA003_wait_and_attempts_never_reset_on_resume(self):
p=self.provider(lambda n,body:(429,{'Retry-After':'3'},b'{}'))
b=self.broker(p,self.policy(wait_budget_s=.07))
scope=b.issue_scope('e','teacher','glm-test')
self.assertEqual(self.request(scope,'stable')[0],503)
spent=b.execution_status('e')['wait_spent']
self.assertFalse(b.resume_readiness('e')['eligible'])
with self.assertRaises(QuotaError):
b.resume('e','stable',checkpoint_complete=True,side_effects_known=True)
self.assertEqual(b.execution_status('e')['resumes'],0)
clock=[b._now()]
with patch.object(b,'_now',side_effect=lambda:clock[0]):
for i in range(3):
clock[0]+=4
self.assertTrue(b.resume_readiness('e')['eligible'])
b.resume('e','stable',checkpoint_complete=True,side_effects_known=True)
self.assertEqual(self.request(scope,'stable')[0],503)
self.assertGreaterEqual(b.execution_status('e')['wait_spent'],spent)
self.assertEqual(len(p.requests),4)
self.assertEqual(b.execution_status('e')['resumes'],3)
with self.assertRaises(QuotaError):
b.resume('e','stable',checkpoint_complete=True,side_effects_known=True)
self.assertEqual(self.request(scope,'replacement')[0],409)
def test_first_byte_timeout_quarantines_unknown_request(self):
def delay(n,body):
time.sleep(.2);return ok(n,body)
p=self.provider(delay); b=self.broker(p,self.policy(first_byte_timeout_s=.04))
s=b.issue_scope('e','teacher','glm-test')
self.assertEqual(self.request(s)[0],503)
snap=b.snapshot(); self.assertEqual(len(p.requests),1)
self.assertEqual(snap['attempts'][0]['transport_reason'],'FIRST_BYTE_TIMEOUT')
self.assertIsNone(snap['attempts'][0]['ended_at'])
self.assertEqual(snap['groups'][0]['state'],'RECONCILING')
self.assertEqual(self.request(b.issue_scope('e2','teacher','glm-test'))[0],503)
self.assertEqual(len(p.requests),1)
def test_stream_idle_and_busy_total_timeout_are_bounded(self):
for mode in ('idle','busy'):
with self.subTest(mode=mode):
# Separate ledger avoids deliberately quarantined prior group.
self.ledger=Path(self.tmp.name)/(mode+'.sqlite')
def payload(out):
for i in range(30):
out.write(b'data: {"choices":[{"delta":{"content":"x"}}]}\n\n');out.flush()
time.sleep(.2 if mode=='idle' else .01)
p=self.provider(lambda n,body:(200,{'Content-Type':'text/event-stream'},payload))
b=self.broker(p,self.policy(stream_idle_timeout_s=.05,request_total_s=.12))
started=time.monotonic()
self.assertEqual(self.request(b.issue_scope(mode,'teacher','glm-test'),body={'model':'glm-test','messages':[],'stream':True,'max_tokens':4})[0],503)
self.assertLess(time.monotonic()-started,.3)
attempt=b.snapshot()['attempts'][0]
self.assertEqual(attempt['reason'],'INCOMPLETE_GENERATION')
self.assertEqual(attempt['transport_reason'],'STREAM_IDLE_TIMEOUT' if mode=='idle' else 'REQUEST_TOTAL_TIMEOUT')
self.assertEqual(len(p.requests),1)
def test_cancel_queued_execution_fences_later_send(self):
p=self.provider(lambda n,body:(429,{'Retry-After':'1'},b'{}'))
b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
with ThreadPoolExecutor(max_workers=1) as pool:
pending=pool.submit(self.request,s)
deadline=time.monotonic()+1
while not p.requests and time.monotonic()<deadline: time.sleep(.005)
b.cancel('e')
self.assertEqual(pending.result()[0],409)
self.assertEqual(len(p.requests),1)
def test_5xx_retries_and_auth_pauses_without_provider_body_leak(self):
p=self.provider(lambda n,body:(503,{},b'secret provider failure') if n==1 else ok(n,body))
b=self.broker(p); self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'))[0],200)
self.assertEqual(len(p.requests),2)
self.assertNotIn('upstream-secret',json.dumps(b.snapshot()))
self.assertNotIn('secret provider failure',json.dumps(b.snapshot()))
def test_concurrency_is_kept_until_response_end(self):
active=peak=0
lock=threading.Lock()
def responder(n,body):
nonlocal active,peak
with lock:
active+=1;peak=max(peak,active)
time.sleep(.05)
with lock: active-=1
return ok(n,body)
p=self.provider(responder); b=self.broker(p,self.policy(max_concurrency=1))
scopes=[b.issue_scope(str(i),'teacher','glm-test') for i in range(4)]
with ThreadPoolExecutor(max_workers=4) as pool:
results=list(pool.map(self.request,scopes))
self.assertTrue(all(code==200 for code,_ in results))
self.assertEqual(peak,1)
self.assertEqual(len(p.requests),4)
def test_cancel_active_request_interrupts_without_refund(self):
def delay(n,body):
time.sleep(.4);return ok(n,body)
p=self.provider(delay); b=self.broker(p); scope=b.issue_scope('e','teacher','glm-test')
with ThreadPoolExecutor(max_workers=1) as pool:
future=pool.submit(self.request,scope)
until=time.monotonic()+1
while not p.requests and time.monotonic()<until: time.sleep(.005)
b.cancel('e')
self.assertEqual(future.result()[0],503)
self.assertEqual(len(p.requests),1)
self.assertIsNone(b.snapshot()['attempts'][0]['ended_at'])
self.assertEqual(b.execution_status('e')['generations'][0]['reason'],'CANCELLED')
def test_auth_failure_pauses_group_and_no_new_id_bypasses_it(self):
p=self.provider(lambda n,body:(401,{},b'upstream-private-diagnostic'))
b=self.broker(p); scope=b.issue_scope('e','teacher','glm-test')
code,body=self.request(scope)
self.assertEqual(code,503);self.assertNotIn(b'upstream-private',body)
self.assertEqual(b.snapshot()['groups'][0]['state'],'PAUSED')
self.assertEqual(self.request(b.issue_scope('another','teacher','glm-test'))[0],503)
self.assertEqual(len(p.requests),1)
def test_independent_processes_share_RPM_ledger(self):
p=self.provider(); policy=self.policy(rpm=1,window_s=1,wait_budget_s=.12)
# Initialize the immutable group policy before simultaneous processes.
Broker(self.ledger,{'shared':policy},{'teacher':Upstream('shared',p.url,'upstream-secret',('glm-test',))})
context=multiprocessing.get_context('spawn')
ready,start,result=context.Queue(),context.Event(),context.Queue()
processes=[context.Process(target=_process_broker,args=(str(self.ledger),p.url,policy,str(i),ready,start,result)) for i in range(2)]
try:
for process in processes: process.start()
for _ in processes: self.assertTrue(ready.get(timeout=8))
start.set()
self.assertEqual(sorted(result.get(timeout=8) for _ in processes),[200,503])
for process in processes:
process.join(timeout=5);self.assertEqual(process.exitcode,0)
self.assertEqual(len(p.requests),1)
finally:
for process in processes:
if process.is_alive(): process.terminate();process.join(timeout=2)
for queue in (ready,result): queue.close();queue.join_thread()
def test_429_headers_fence_sends_before_error_body_finishes(self):
def stalled_body(out):
time.sleep(.2);out.write(b'{}')
p=self.provider(lambda n,body:(429,{'Retry-After':'2'},stalled_body))
b=self.broker(p,self.policy(wait_budget_s=.06,stream_idle_timeout_s=.3))
with ThreadPoolExecutor(max_workers=1) as pool:
first=pool.submit(self.request,b.issue_scope('first','teacher','glm-test'))
deadline=time.monotonic()+1
while b.snapshot()['groups'][0]['state']!='COOLING' and time.monotonic()<deadline:
time.sleep(.005)
self.assertEqual(b.snapshot()['groups'][0]['state'],'COOLING')
self.assertEqual(self.request(b.issue_scope('second','teacher','glm-test'))[0],503)
self.assertEqual(first.result()[0],503)
self.assertEqual(len(p.requests),1)
def test_wall_clock_rollback_cannot_extend_queue_budget(self):
p=self.provider(lambda n,body:(429,{'Retry-After':'10'},b'{}'))
b=self.broker(p,self.policy(wait_budget_s=.05))
scope=b.issue_scope('e','teacher','glm-test')
before=time.monotonic(); wall=time.time()
with patch('harness.quota.time.time',return_value=wall-3600):
self.assertEqual(self.request(scope)[0],503)
self.assertLess(time.monotonic()-before,.5)
self.assertGreaterEqual(b.execution_status('e')['wait_spent'],.05)
def test_dns_deadline_discards_late_resolution_without_sending(self):
p=self.provider();b=self.broker(p,self.policy(connect_timeout_s=.03,request_total_s=.1))
scope=b.issue_scope('dns','teacher','glm-test')
original=socket.getaddrinfo;release=threading.Event();entered=threading.Event()
def blocked(host,port,*args,**kwargs):
if port==p.server.server_port:
entered.set();release.wait(2)
return original(host,port,*args,**kwargs)
try:
with patch('harness.quota.socket.getaddrinfo',side_effect=blocked):
code,_=self.request(scope)
self.assertTrue(entered.is_set());self.assertEqual(code,503)
self.assertFalse(release.is_set())
self.assertEqual(b.execution_status('dns')['generations'][0]['reason'],'DNS_TIMEOUT')
self.assertEqual(p.requests,[])
finally:
release.set()
time.sleep(.06)
self.assertEqual(p.requests,[]) # Resolver has no transport capability.
def test_cancel_while_dns_blocked_does_not_wait_for_group_lock(self):
p=self.provider();b=self.broker(p,self.policy(connect_timeout_s=1))
scope=b.issue_scope('dns-cancel','teacher','glm-test')
original=socket.getaddrinfo;release=threading.Event();entered=threading.Event()
def blocked(host,port,*args,**kwargs):
if port==p.server.server_port:
entered.set();release.wait(2)
return original(host,port,*args,**kwargs)
try:
with patch('harness.quota.socket.getaddrinfo',side_effect=blocked),ThreadPoolExecutor(max_workers=1) as pool:
pending=pool.submit(self.request,scope)
self.assertTrue(entered.wait(1))
b.cancel('dns-cancel')
self.assertEqual(pending.result(timeout=.5)[0],409)
self.assertFalse(release.is_set());self.assertEqual(p.requests,[])
finally:
release.set()
def test_confirmed_end_allows_new_probe_but_never_old_generation_replay(self):
finished=threading.Event()
def slow_body(out):
time.sleep(.1);out.write(ok()[2]);out.flush();finished.set()
p=self.provider(lambda n,body:(200,{'Content-Type':'application/json'},slow_body) if n==1 else ok(n,body))
b=self.broker(p,self.policy(stream_idle_timeout_s=.03))
original=b.issue_scope('original','teacher','glm-test')
self.assertEqual(self.request(original,'original-g')[0],503)
self.assertTrue(finished.wait(1))
attempt=b.snapshot()['attempts'][0]
b.confirm_end(attempt['attempt_id'],'verified-completion-receipt')
self.assertEqual(b.snapshot()['groups'][0]['state'],'COOLING')
self.assertEqual(self.request(b.issue_scope('new','teacher','glm-test'))[0],200)
self.assertEqual(b.snapshot()['attempts'][-1]['state_at_send'],'HALF_OPEN')
self.assertEqual(b.execution_status('original')['generations'][0]['state'],'PAUSED')
self.assertEqual(self.request(original,'new-g')[0],409)
self.assertEqual(len(p.requests),2)
def test_stalled_429_body_retains_received_rate_limit_headers(self):
def slow(out): time.sleep(.1);out.write(b'{}')
p=self.provider(lambda n,body:(429,{'Retry-After':'8','X-Request-Id':'trace-123'},slow))
b=self.broker(p,self.policy(stream_idle_timeout_s=.03))
self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'))[0],503)
attempt=b.snapshot()['attempts'][0]
self.assertEqual(attempt['http_status'],429)
self.assertEqual(json.loads(attempt['response_headers']),{'retry-after':'8','x-request-id':'trace-123'})
def test_stream_integrity_validates_every_choice_and_tool_arguments(self):
def stream(*choices):
return b''.join(b'data: '+json.dumps({'choices':chunk}).encode()+b'\n\n' for chunk in choices)+b'data: [DONE]\n\n'
complete={'index':0,'delta':{'role':'assistant','content':'ok'},'finish_reason':'stop'}
after={'index':0,'delta':{'tool_calls':[{'index':0,'id':'c','function':{'name':'bash','arguments':'{broken'}}]}}
fixtures=[
(stream([complete],[after]),'INVALID_PROVIDER_RESPONSE'),
(stream([complete,{'index':1,'delta':{'content':'unfinished'}}]),'INCOMPLETE_GENERATION'),
(stream([{'index':0,'delta':{},'finish_reason':'stop'}]),'INVALID_PROVIDER_RESPONSE'),
(stream([dict(after,finish_reason='tool_calls')]),'INVALID_PROVIDER_RESPONSE'),
(stream([complete]),'COMPLETE'),
(stream([{'index':0,'delta':{'tool_calls':[{'index':0,'id':'c','type':'function','function':{'name':'bash','arguments':'{"command":'}}]}}],
[{'index':0,'delta':{'tool_calls':[{'index':0,'function':{'arguments':'"echo ok"}'}}]},'finish_reason':'tool_calls'}]),'COMPLETE'),
]
for payload,expected in fixtures:
with self.subTest(expected=expected,payload=payload):
self.assertEqual(validate_completion(payload,True),expected)
p=self.provider(lambda n,body:(200,{'Content-Type':'text/event-stream'},fixtures[0][0]))
b=self.broker(p)
self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'),body={'model':'glm-test','messages':[],'max_tokens':5,'stream':True})[0],503)
self.assertNotEqual(b.execution_status('e')['generations'][0]['state'],'COMPLETE')
def test_independent_snapshot_is_readonly_bounded_and_missing_is_unknown(self):
absent=Path(self.tmp.name)/'missing'/'ledger'
self.assertEqual(quota_snapshot(absent)['status'],'UNKNOWN')
self.assertFalse(absent.parent.exists())
p=self.provider();b=self.broker(p)
for i in range(3): self.request(b.issue_scope(str(i),'teacher','glm-test'))
before=hashlib.sha256(self.ledger.read_bytes()).hexdigest()
with patch.object(Broker,'__init__',side_effect=AssertionError('must not construct')):
snap=quota_snapshot(self.ledger,limit=2)
self.assertEqual(snap['status'],'OBSERVED')
self.assertLessEqual(len(snap['attempts']),2);self.assertLessEqual(len(snap['events']),2)
self.assertNotIn('upstream-secret',json.dumps(snap));self.assertNotIn('token_hash',json.dumps(snap))
self.assertEqual(hashlib.sha256(self.ledger.read_bytes()).hexdigest(),before)
self.assertEqual(snap['groups'][0]['inflight'],0)
def test_connect_failure_pause_resumes_after_cooldown(self):
p=self.provider();b=self.broker(p,self.policy())
scope=b.issue_scope('blip','teacher','glm-test')
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
def refused(connection):
if connection.port==p.server.server_port:
raise ConnectionRefusedError('injected Wi-Fi blip')
return original(connection)
with patch('harness.quota.http.client.HTTPConnection.connect',new=refused):
self.assertEqual(self.request(scope,'g')[0],503)
with b._db() as con:
cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0]
with patch.object(b,'_now',return_value=max(b._now(),cooldown)+1):
self.assertTrue(b.resume_readiness('blip')['eligible'])
b.resume('blip','g',checkpoint_complete=True,side_effects_known=True)
self.assertEqual(self.request(scope,'g')[0],200)
self.assertEqual(len(p.requests),1)
def test_connect_blip_is_resent_in_place_and_charged_to_the_wait_budget(self):
# 10-14: one refused connect paused a 10-hour rollout; four of them stranded it at 3/3 resumes.
p=self.provider();b=self.broker(p,self.policy())
scope=b.issue_scope('blip','teacher','glm-test')
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
refusals=[2]
def flaky(connection):
if connection.port==p.server.server_port and refusals[0]:
refusals[0]-=1
raise ConnectionRefusedError('injected Wi-Fi blip')
return original(connection)
with patch('harness.quota.http.client.HTTPConnection.connect',new=flaky):
self.assertEqual(self.request(scope)[0],200)
self.assertEqual(len(p.requests),1)
self.assertGreaterEqual(b.execution_status('blip')['wait_spent'],2*(.06+.2))
self.assertEqual(len([e for e in b.snapshot()['quota_events'] if e['event']=='UNSENT_RETRY']),2)
def test_pause_that_sent_nothing_does_not_spend_the_resume_budget(self):
p=self.provider();b=self.broker(p,self.policy())
scope=b.issue_scope('free','teacher','glm-test')
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
def refused(connection):
if connection.port==p.server.server_port:
raise ConnectionRefusedError('injected outage')
return original(connection)
with patch('harness.quota.http.client.HTTPConnection.connect',new=refused):
self.assertEqual(self.request(scope,'g')[0],503) # in-place retries spend the wait budget, then pause
cap=b.groups['shared'].max_auto_resumes
with b._db() as con:
con.execute('UPDATE executions SET resumes=? WHERE execution_id=?',(cap,'free'))
cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0]
with patch.object(b,'_now',return_value=max(b._now(),cooldown)+1):
ready=b.resume_readiness('free')
self.assertTrue(ready['eligible']);self.assertFalse(ready['counted'])
b.resume('free','g',checkpoint_complete=True,side_effects_known=True)
self.assertEqual(b.execution_status('free')['resumes'],cap)
self.assertEqual(p.requests,[])
def test_confirmed_connect_failure_does_not_spend_http_budget(self):
p=self.provider();b=self.broker(p,self.policy(rpm=1,window_s=1))
scope=b.issue_scope('connect-failure','teacher','glm-test')
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
def refused(connection):
if connection.port==p.server.server_port:
raise ConnectionRefusedError('injected TCP refusal before HTTP')
return original(connection)
with patch('harness.quota.http.client.HTTPConnection.connect',new=refused):
self.assertEqual(self.request(scope)[0],503)
generation=b.execution_status('connect-failure')['generations'][0]
self.assertEqual(generation['attempts'],0)
self.assertEqual(generation['reason'],'CONNECT_FAILED')
self.assertEqual(generation['state'],'PAUSED')
self.assertEqual(p.requests,[])
snapshot=quota_snapshot(self.ledger)
self.assertEqual(snapshot['groups'][0]['rpm_used'],0)
self.assertEqual(snapshot['groups'][0]['tpm_reserved'],0)
self.assertEqual(snapshot['groups'][0]['inflight'],0)
self.assertTrue(all(row['sent']==0 and row['state']=='NOT_SENT' for row in snapshot['attempts']))
self.assertIn('TRANSPORT_NOT_SENT',[event['event'] for event in snapshot['events']])
# The same account can serve a later request; no phantom RPM reservation.
with b._db() as con:
cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0]
with patch.object(b,'_now',return_value=max(b._now(),cooldown)):
self.assertEqual(self.request(b.issue_scope('next','teacher','glm-test'))[0],200)
self.assertEqual(len(p.requests),1)
def test_unknown_send_never_refunds_http_budget(self):
p=self.provider();b=self.broker(p,self.policy(window_s=1))
scope=b.issue_scope('unknown-send','teacher','glm-test')
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.request
def interrupted(connection,*args,**kwargs):
original(connection,*args,**kwargs)
if connection.port==p.server.server_port:
raise OSError('injected interruption after request bytes')
with patch('harness.quota.http.client.HTTPConnection.request',new=interrupted):
self.assertEqual(self.request(scope)[0],503)
generation=b.execution_status('unknown-send')['generations'][0]
self.assertEqual(generation['attempts'],1)
snapshot=quota_snapshot(self.ledger)
self.assertEqual(snapshot['groups'][0]['rpm_used'],1)
self.assertGreater(snapshot['groups'][0]['tpm_reserved'],0)
self.assertEqual(snapshot['groups'][0]['inflight'],1)
self.assertEqual(snapshot['attempts'][0]['state'],'UNKNOWN')
self.assertIsNone(snapshot['attempts'][0]['ended_at'])
self.assertNotIn('TRANSPORT_NOT_SENT',[event['event'] for event in snapshot['events']])
def test_invalid_budget_rejected(self):
with self.assertRaises(ValueError): self.policy(rpm=None)
def test_model_scope_cannot_change(self):
p=self.provider(); b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
self.assertEqual(self.request(s,body={'model':'other','messages':[],'max_tokens':2})[0],403)
self.assertEqual(p.requests,[])
if __name__=='__main__': unittest.main()