data-gen-bpu/scripts/test_quota.py
bench 6e1668c9a8 Quota: a request over the account's limit is a full context for Pi
Recorded FAILED CONTEXT_TOO_LONG like a provider overflow and answered with
the overflow body, so Pi compacts and continues instead of ending the rollout
on an unrecognised 413. Content review keeps its REQUEST_EXCEEDS_INTERNAL_TPM.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 16:52:42 +08:00

743 lines
45 KiB
Python

"""Quota acceptance uses real local HTTP, persistent ledgers and no provider account."""
from concurrent.futures import ThreadPoolExecutor
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import multiprocessing
import socket
import hashlib
from pathlib import Path
import tempfile
import threading
import time
import unittest
from unittest.mock import patch
import urllib.error
import urllib.request
from harness.quota import Broker, CONTEXT_OVERFLOW_BODY, QuotaPolicy, Upstream, QuotaError, quota_snapshot, validate_completion
class Provider:
def __init__(self, responder):
self.responder, self.requests = responder, []
outer = self
class Handler(BaseHTTPRequestHandler):
def log_message(self, *_): pass
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
outer.requests.append({'at': time.time(), 'body': body,
'authorization': self.headers.get('Authorization')})
code, headers, payload = outer.responder(len(outer.requests), body)
self.send_response(code)
for key, value in headers.items(): self.send_header(key, str(value))
self.end_headers()
try:
if callable(payload): payload(self.wfile)
else: self.wfile.write(payload)
except (BrokenPipeError, ConnectionResetError): pass
self.server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
self.thread.start()
self.url = 'http://127.0.0.1:%s/v1' % self.server.server_port
def close(self):
self.server.shutdown(); self.server.server_close(); self.thread.join()
def ok(_n=0, body=None):
if body and body.get('stream'):
return 200, {'Content-Type': 'text/event-stream'}, (b'data: {"choices":[{"delta":{"content":"ok"},"finish_reason":null}]}\n\n'
b'data: {"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}\n\n'
b'data: [DONE]\n\n')
return 200, {'Content-Type': 'application/json'}, b'{"choices":[{"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}'
def _process_broker(ledger, url, policy, execution, ready, start, result):
# Spawned interpreters do not inherit the parent's locks or server objects.
with Broker(Path(ledger), {'shared':policy}, {'teacher':Upstream('shared',url,'upstream-secret',('glm-test',))}) as broker:
scope=broker.issue_scope(execution,'teacher','glm-test')
ready.put(True)
if not start.wait(5):
result.put('START_TIMEOUT');return
request=urllib.request.Request(scope.base_url+'/chat/completions',
json.dumps({'model':'glm-test','messages':[],'max_tokens':5}).encode(),
{'Authorization':'Bearer '+scope.api_key,'Content-Type':'application/json'})
try:
with urllib.request.urlopen(request,timeout=5) as response:
response.read();result.put(response.status)
except urllib.error.HTTPError as error:
error.read();result.put(error.code)
class Quota(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
self.ledger = Path(self.tmp.name) / 'shared.sqlite'
def provider(self, responder=ok):
item = Provider(responder); self.addCleanup(item.close); return item
def policy(self, **kw):
return QuotaPolicy(**dict({'max_concurrency':2, 'rpm':1000, 'tpm':1000000,
'window_s':.2, 'cooldown_initial_s':.06, 'cooldown_local_max_s':.1,
'jitter_fraction':0, 'recovery_window_s':.12, 'recovery_min_successes':2,
'wait_budget_s':2, 'connect_timeout_s':.2, 'first_byte_timeout_s':.3,
'stream_idle_timeout_s':.15, 'request_total_s':.6, 'poll_s':.01}, **kw))
def broker(self, provider, policy=None, aliases=None):
b = Broker(self.ledger, {'shared':policy or self.policy()}, aliases or {
'teacher':Upstream('shared', provider.url, 'upstream-secret', ('glm-test',))})
b.start(); self.addCleanup(b.close); return b
def request(self, scope, generation=None, body=None):
headers = {'Authorization':'Bearer '+scope.api_key, 'Content-Type':'application/json'}
if generation: headers['X-Bench-Generation-Id'] = generation
request = urllib.request.Request(scope.base_url+'/chat/completions',
json.dumps(body or {'model':'glm-test', 'messages':[{'role':'user','content':'hi'}], 'max_tokens':5}).encode(), headers)
try:
with urllib.request.urlopen(request, timeout=5) as response:
return response.status, response.read()
except urllib.error.HTTPError as error:
return error.code, error.read()
def test_success_scope_and_stream_usage(self):
p=self.provider(); b=self.broker(p); scope=b.issue_scope('execution', 'teacher', 'glm-test')
code, data=self.request(scope, 'generation', {'model':'glm-test','messages':[],'max_tokens':4,'stream':True})
self.assertEqual(code, 200); self.assertIn(b'[DONE]', data)
self.assertEqual(p.requests[0]['authorization'], 'Bearer upstream-secret')
self.assertNotIn('upstream-secret', scope.api_key)
self.assertEqual(b.snapshot()['attempts'][0]['charged_tokens'], 3)
self.assertEqual(len(p.requests), 1)
self.assertEqual(self.request(scope, 'generation')[0], 409)
def _raw_request(self, scope):
from urllib.parse import urlsplit
url = urlsplit(scope.base_url)
sock = socket.create_connection((url.hostname, url.port), timeout=2)
body = json.dumps({'model':'glm-test', 'messages':[], 'max_tokens':5}).encode()
sock.sendall((f'POST /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\n'
f'Authorization: Bearer {scope.api_key}\r\nContent-Length: {len(body)}\r\n'
'Content-Type: application/json\r\n\r\n').encode() + body)
return sock
def test_disconnected_queued_client_never_reaches_upstream(self):
p = self.provider(); b = self.broker(p, self.policy(rpm=1, window_s=.4))
self.assertEqual(self.request(b.issue_scope('first', 'teacher', 'glm-test'))[0], 200)
scope = b.issue_scope('disconnected', 'teacher', 'glm-test')
sock = self._raw_request(scope)
deadline = time.monotonic() + 1
while not b.execution_status('disconnected')['generations'] and time.monotonic() < deadline:
time.sleep(.005)
self.assertTrue(b.execution_status('disconnected')['generations'])
sock.close()
deadline = time.monotonic() + 1
while not b.execution_status('disconnected')['cancelled'] and time.monotonic() < deadline:
time.sleep(.005)
self.assertTrue(b.execution_status('disconnected')['cancelled'])
time.sleep(.45) # RPM capacity becomes free after the client has gone.
self.assertEqual(len(p.requests), 1)
gen = b.execution_status('disconnected')['generations'][0]
self.assertEqual((gen['state'], gen['attempts']), ('CANCELLED', 0))
self.assertEqual(b.snapshot()['groups'][0]['state'], 'OPEN')
def test_disconnected_sent_client_keeps_unknown_reservation(self):
release = threading.Event()
self.addCleanup(release.set)
p = self.provider(lambda *_: (release.wait(2) or True) and ok())
b = self.broker(p)
sock = self._raw_request(b.issue_scope('disconnected', 'teacher', 'glm-test'))
deadline = time.monotonic() + 1
while not p.requests and time.monotonic() < deadline:
time.sleep(.005)
self.assertEqual(len(p.requests), 1)
sock.close()
deadline = time.monotonic() + 1
while b.snapshot()['attempts'][0]['state'] != 'UNKNOWN' and time.monotonic() < deadline:
time.sleep(.005)
attempt = b.snapshot()['attempts'][0]
self.assertEqual(attempt['state'], 'UNKNOWN')
self.assertIsNone(attempt['ended_at'])
self.assertGreater(attempt['charged_tokens'], 0)
self.assertTrue(b.execution_status('disconnected')['cancelled'])
self.assertEqual(len(p.requests), 1)
release.set()
def test_valid_sse_done_does_not_wait_for_http_close(self):
for crlf in (False, True):
with self.subTest(crlf=crlf), tempfile.TemporaryDirectory() as temp:
payload = ok(body={'stream':True})[2]
if crlf:
payload = payload.replace(b'\n', b'\r\n')
release = threading.Event()
def send(writer):
# Split inside the terminator to exercise real read boundaries.
cut = payload.index(b'[DONE]') + 3
writer.write(payload[:cut]); writer.flush()
time.sleep(.02)
writer.write(payload[cut:]); writer.flush()
release.wait(2)
provider = self.provider(lambda *_: (200, {'Content-Type':'text/event-stream'}, send))
with Broker(Path(temp)/'quota.sqlite', {'shared':self.policy()}, {
'teacher':Upstream('shared',provider.url,'fake-only',('glm-test',))}) as broker:
try:
scope = broker.issue_scope('execution','teacher','glm-test')
code, data = self.request(scope,'generation',{'model':'glm-test','messages':[], 'max_tokens':4,'stream':True})
self.assertEqual((code,data),(200,payload))
attempt = broker.snapshot()['attempts'][0]
self.assertEqual((attempt['state'],attempt['reason'],attempt['charged_tokens']),('FINISHED','COMPLETE',3))
self.assertEqual(broker.snapshot()['groups'][0]['state'],'OPEN')
self.assertEqual(len(provider.requests),1)
finally:
release.set()
def test_sse_terminator_cannot_complete_invalid_or_unframed_response(self):
complete = ok(body={'stream':True})[2]
variants = {
'missing_blank_line': complete[:-2],
'content_literal_only': b'data: {"choices":[{"delta":{"content":"[DONE]"},"finish_reason":"stop"}]}\n\n',
'missing_finish': complete.split(b'data: {"choices":[{"delta":{}')[0] + b'data: [DONE]\n\n',
'partial_tool': b'data: {"choices":[{"delta":{"tool_calls":[{"id":"x","type":"function","function":{"name":"write","arguments":"{"}}]},"finish_reason":"tool_calls"}]}\n\ndata: [DONE]\n\n',
'extra_data_after_done': complete + b'data: {"choices":[]}\n\n',
}
for name,payload in variants.items():
with self.subTest(name=name), tempfile.TemporaryDirectory() as temp:
release = threading.Event()
def send(writer):
writer.write(payload); writer.flush(); release.wait(2)
provider = self.provider(lambda *_: (200, {'Content-Type':'text/event-stream'},send))
with Broker(Path(temp)/'quota.sqlite',{'shared':self.policy()}, {
'teacher':Upstream('shared',provider.url,'fake-only',('glm-test',))}) as broker:
try:
scope = broker.issue_scope('execution','teacher','glm-test')
code,_ = self.request(scope,'generation',{'model':'glm-test','messages':[],'max_tokens':4,'stream':True})
self.assertNotEqual(code,200)
attempt = broker.snapshot()['attempts'][0]
self.assertEqual((attempt['state'],attempt['transport_reason']),('UNKNOWN','STREAM_IDLE_TIMEOUT'))
self.assertEqual(len(provider.requests),1)
finally:
release.set()
def test_JOINT001_persistent_429_budget_and_single_probe(self):
p=self.provider(lambda n, body:(429, {'Retry-After':'.06'}, b'{"error":{"code":"rate_limit"}}'))
b=self.broker(p, self.policy(max_attempts=4))
scopes=[b.issue_scope('e'+str(i),'teacher','glm-test') for i in range(4)]
with ThreadPoolExecutor(max_workers=4) as pool:
results=list(pool.map(lambda s:self.request(s),scopes))
snapshot=b.snapshot()
self.assertTrue(all(code==503 for code,_ in results))
self.assertTrue(all(g['attempts']<=4 for g in snapshot['generations']))
self.assertTrue(all(t['state_at_send'] in ('OPEN','HALF_OPEN','RECOVERING') for t in snapshot['attempts']))
self.assertEqual(len(p.requests), len(snapshot['attempts']))
probes=[a for a in snapshot['attempts'] if a['state_at_send']=='HALF_OPEN']
for before,after in zip(probes,probes[1:]):
self.assertGreaterEqual(after['started_at'],before['ended_at'])
self.assertEqual(len({a['attempt_id'] for a in snapshot['attempts']}),len(snapshot['attempts']))
def test_QUOTA001_shared_alias_group_independent_group(self):
p=self.provider(lambda n, body:(429, {'Retry-After':'1'}, b'{}') if body['model']=='limited' else ok(n,body))
b=Broker(self.ledger, {'shared':self.policy(wait_budget_s=.12), 'independent':self.policy()}, {
'a':Upstream('shared',p.url,'a',('limited',)), 'b':Upstream('shared',p.url,'b',('limited',)),
'other':Upstream('independent',p.url,'c',('free',))})
b.start(); self.addCleanup(b.close)
self.request(b.issue_scope('a','a','limited'),body={'model':'limited','messages':[],'max_tokens':1})
self.request(b.issue_scope('b','b','limited'),body={'model':'limited','messages':[],'max_tokens':1})
self.assertEqual(len(p.requests),1)
self.assertEqual(self.request(b.issue_scope('c','other','free'),body={'model':'free','messages':[],'max_tokens':1})[0],200)
def test_JOINT003_restart_keeps_budget_and_missing_usage(self):
p=self.provider(lambda n,body:(200,{'Content-Type':'application/json'},b'{"choices":[{"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}'))
b=self.broker(p,self.policy(rpm=1,window_s=1,wait_budget_s=.06))
self.assertEqual(self.request(b.issue_scope('e1','teacher','glm-test'))[0],200)
first=b.snapshot()['attempts'][0]
self.assertEqual(first['charged_tokens'],first['reserved_tokens'])
b.close()
b2=self.broker(p,self.policy(rpm=1,window_s=1,wait_budget_s=.06))
self.assertEqual(self.request(b2.issue_scope('e2','teacher','glm-test'))[0],503)
self.assertEqual(len(p.requests),1)
def test_cancel_blocks_transport_without_hitting_upstream(self):
p=self.provider(); b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
b.cancel('e'); self.assertEqual(self.request(s)[0],409); self.assertEqual(p.requests,[])
def test_QUOTA002_retry_after_and_auth_pause(self):
p=self.provider(lambda n,body:(429,{'Retry-After':'4'},b'{}'))
b=self.broker(p,self.policy(wait_budget_s=.1)); before=time.time()
self.request(b.issue_scope('e','teacher','glm-test'))
self.assertGreaterEqual(b.snapshot()['groups'][0]['cooldown_until'],before+4)
self.assertEqual(len(p.requests),1)
def test_JOINT004_nonempty_stream_interruption_does_not_retry(self):
p=self.provider(lambda n,body:(200,{'Content-Type':'text/event-stream'}, b'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'))
b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
self.assertEqual(self.request(s,body={'model':'glm-test','messages':[],'max_tokens':5,'stream':True})[0],503)
self.assertEqual(len(p.requests),1)
self.assertEqual(b.snapshot()['generations'][0]['reason'],'INCOMPLETE_GENERATION')
self.assertEqual(self.request(s,'new-id')[0],409)
def test_JOINT002_recovery_adds_at_most_one_per_window(self):
p=self.provider(lambda n,body:(429,{},b'{}') if n==1 else ok(n,body))
policy=self.policy(max_concurrency=4,recovery_window_s=.12)
b=self.broker(p,policy)
self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'))[0],200)
self.assertEqual(b.snapshot()['groups'][0]['state'],'RECOVERING')
self.assertEqual(b.snapshot()['groups'][0]['target'],1)
clock=[b.snapshot()['groups'][0]['recovery_at']]
# Only quota's durable clock is controlled. HTTP itself remains real.
with patch.object(b,'_now',side_effect=lambda:clock[0]):
for i in range(3):
self.assertEqual(self.request(b.issue_scope('e'+str(i),'teacher','glm-test'))[0],200)
self.assertEqual(b.snapshot()['groups'][0]['target'],1)
clock[0]+=.13
self.request(b.issue_scope('next','teacher','glm-test'))
self.assertEqual(b.snapshot()['groups'][0]['target'],1)
self.request(b.issue_scope('next2','teacher','glm-test'))
self.assertEqual(b.snapshot()['groups'][0]['target'],2)
self.request(b.issue_scope('after','teacher','glm-test'))
self.assertEqual(b.snapshot()['groups'][0]['target'],2)
def test_shared_brokers_enforce_actual_RPM(self):
p=self.provider(); policy=self.policy(rpm=1,window_s=.7,wait_budget_s=.12)
a=self.broker(p,policy); b=self.broker(p,policy)
scopes=[a.issue_scope('a','teacher','glm-test'),b.issue_scope('b','teacher','glm-test')]
with ThreadPoolExecutor(max_workers=2) as pool:
results=list(pool.map(self.request,scopes))
self.assertEqual(sorted(code for code,_ in results),[200,503])
self.assertEqual(len(p.requests),1)
self.assertEqual(len(a.snapshot()['attempts']),1)
def test_missing_usage_reservation_blocks_TPM_until_end_window(self):
p=self.provider(lambda n,body:(200,{},b'{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}]}'))
b=self.broker(p,self.policy(tpm=300,window_s=.7,wait_budget_s=.08))
self.assertEqual(self.request(b.issue_scope('a','teacher','glm-test'))[0],200)
self.assertEqual(self.request(b.issue_scope('b','teacher','glm-test'))[0],503)
self.assertEqual(len(p.requests),1)
def test_QUOTA003_attempts_never_reset_and_each_resume_gets_a_fresh_wait(self):
p=self.provider(lambda n,body:(429,{'Retry-After':'3'},b'{}'))
b=self.broker(p,self.policy(wait_budget_s=.07))
scope=b.issue_scope('e','teacher','glm-test')
self.assertEqual(self.request(scope,'stable')[0],503)
spent=b.execution_status('e')['wait_spent']
self.assertFalse(b.resume_readiness('e')['eligible'])
with self.assertRaises(QuotaError):
b.resume('e','stable',checkpoint_complete=True,side_effects_known=True)
self.assertEqual(b.execution_status('e')['resumes'],0)
clock=[b._now()]
def tick(): # time moves while the broker waits, so a fresh allowance runs out
clock[0]+=.01; return clock[0]
with patch.object(b,'_now',side_effect=tick):
for i in range(3):
clock[0]+=4
self.assertTrue(b.resume_readiness('e')['eligible'])
b.resume('e','stable',checkpoint_complete=True,side_effects_known=True)
# A lifetime wait total paused at the first busy moment after every resume, forever (10-19).
self.assertEqual(b.execution_status('e')['wait_spent'],0)
self.assertEqual(self.request(scope,'stable')[0],503)
self.assertGreater(spent,0)
self.assertEqual(len(p.requests),4)
self.assertEqual(b.execution_status('e')['resumes'],3)
with self.assertRaises(QuotaError):
b.resume('e','stable',checkpoint_complete=True,side_effects_known=True)
self.assertEqual(self.request(scope,'replacement')[0],409)
def test_first_byte_timeout_quarantines_unknown_request(self):
def delay(n,body):
time.sleep(.2);return ok(n,body)
p=self.provider(delay); b=self.broker(p,self.policy(first_byte_timeout_s=.04))
s=b.issue_scope('e','teacher','glm-test')
self.assertEqual(self.request(s)[0],503)
snap=b.snapshot(); self.assertEqual(len(p.requests),1)
self.assertEqual(snap['attempts'][0]['transport_reason'],'FIRST_BYTE_TIMEOUT')
self.assertIsNone(snap['attempts'][0]['ended_at'])
self.assertEqual(snap['groups'][0]['state'],'RECONCILING')
self.assertEqual(self.request(b.issue_scope('e2','teacher','glm-test'))[0],503)
self.assertEqual(len(p.requests),1)
def test_stream_idle_and_busy_total_timeout_are_bounded(self):
for mode in ('idle','busy'):
with self.subTest(mode=mode):
# Separate ledger avoids deliberately quarantined prior group.
self.ledger=Path(self.tmp.name)/(mode+'.sqlite')
def payload(out):
for i in range(30):
out.write(b'data: {"choices":[{"delta":{"content":"x"}}]}\n\n');out.flush()
time.sleep(.2 if mode=='idle' else .01)
p=self.provider(lambda n,body:(200,{'Content-Type':'text/event-stream'},payload))
b=self.broker(p,self.policy(stream_idle_timeout_s=.05,request_total_s=.12))
started=time.monotonic()
self.assertEqual(self.request(b.issue_scope(mode,'teacher','glm-test'),body={'model':'glm-test','messages':[],'stream':True,'max_tokens':4})[0],503)
self.assertLess(time.monotonic()-started,.3)
attempt=b.snapshot()['attempts'][0]
self.assertEqual(attempt['reason'],'INCOMPLETE_GENERATION')
self.assertEqual(attempt['transport_reason'],'STREAM_IDLE_TIMEOUT' if mode=='idle' else 'REQUEST_TOTAL_TIMEOUT')
self.assertEqual(len(p.requests),1)
def test_cancel_queued_execution_fences_later_send(self):
p=self.provider(lambda n,body:(429,{'Retry-After':'1'},b'{}'))
b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
with ThreadPoolExecutor(max_workers=1) as pool:
pending=pool.submit(self.request,s)
deadline=time.monotonic()+1
while not p.requests and time.monotonic()<deadline: time.sleep(.005)
b.cancel('e')
self.assertEqual(pending.result()[0],409)
self.assertEqual(len(p.requests),1)
def test_5xx_retries_and_auth_pauses_without_provider_body_leak(self):
p=self.provider(lambda n,body:(503,{},b'secret provider failure') if n==1 else ok(n,body))
b=self.broker(p); self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'))[0],200)
self.assertEqual(len(p.requests),2)
self.assertNotIn('upstream-secret',json.dumps(b.snapshot()))
self.assertNotIn('secret provider failure',json.dumps(b.snapshot()))
def test_concurrency_is_kept_until_response_end(self):
active=peak=0
lock=threading.Lock()
def responder(n,body):
nonlocal active,peak
with lock:
active+=1;peak=max(peak,active)
time.sleep(.05)
with lock: active-=1
return ok(n,body)
p=self.provider(responder); b=self.broker(p,self.policy(max_concurrency=1))
scopes=[b.issue_scope(str(i),'teacher','glm-test') for i in range(4)]
with ThreadPoolExecutor(max_workers=4) as pool:
results=list(pool.map(self.request,scopes))
self.assertTrue(all(code==200 for code,_ in results))
self.assertEqual(peak,1)
self.assertEqual(len(p.requests),4)
def test_cancel_active_request_interrupts_without_refund(self):
def delay(n,body):
time.sleep(.4);return ok(n,body)
p=self.provider(delay); b=self.broker(p); scope=b.issue_scope('e','teacher','glm-test')
with ThreadPoolExecutor(max_workers=1) as pool:
future=pool.submit(self.request,scope)
until=time.monotonic()+1
while not p.requests and time.monotonic()<until: time.sleep(.005)
b.cancel('e')
self.assertEqual(future.result()[0],503)
self.assertEqual(len(p.requests),1)
self.assertIsNone(b.snapshot()['attempts'][0]['ended_at'])
self.assertEqual(b.execution_status('e')['generations'][0]['reason'],'CANCELLED')
def test_auth_failure_pauses_group_and_no_new_id_bypasses_it(self):
p=self.provider(lambda n,body:(401,{},b'upstream-private-diagnostic'))
b=self.broker(p); scope=b.issue_scope('e','teacher','glm-test')
code,body=self.request(scope)
self.assertEqual(code,503);self.assertNotIn(b'upstream-private',body)
self.assertEqual(b.snapshot()['groups'][0]['state'],'PAUSED')
self.assertEqual(self.request(b.issue_scope('another','teacher','glm-test'))[0],503)
self.assertEqual(len(p.requests),1)
def test_independent_processes_share_RPM_ledger(self):
p=self.provider(); policy=self.policy(rpm=1,window_s=1,wait_budget_s=.12)
# Initialize the immutable group policy before simultaneous processes.
Broker(self.ledger,{'shared':policy},{'teacher':Upstream('shared',p.url,'upstream-secret',('glm-test',))})
context=multiprocessing.get_context('spawn')
ready,start,result=context.Queue(),context.Event(),context.Queue()
processes=[context.Process(target=_process_broker,args=(str(self.ledger),p.url,policy,str(i),ready,start,result)) for i in range(2)]
try:
for process in processes: process.start()
for _ in processes: self.assertTrue(ready.get(timeout=8))
start.set()
self.assertEqual(sorted(result.get(timeout=8) for _ in processes),[200,503])
for process in processes:
process.join(timeout=5);self.assertEqual(process.exitcode,0)
self.assertEqual(len(p.requests),1)
finally:
for process in processes:
if process.is_alive(): process.terminate();process.join(timeout=2)
for queue in (ready,result): queue.close();queue.join_thread()
def test_429_headers_fence_sends_before_error_body_finishes(self):
def stalled_body(out):
time.sleep(.2);out.write(b'{}')
p=self.provider(lambda n,body:(429,{'Retry-After':'2'},stalled_body))
b=self.broker(p,self.policy(wait_budget_s=.06,stream_idle_timeout_s=.3))
with ThreadPoolExecutor(max_workers=1) as pool:
first=pool.submit(self.request,b.issue_scope('first','teacher','glm-test'))
deadline=time.monotonic()+1
while b.snapshot()['groups'][0]['state']!='COOLING' and time.monotonic()<deadline:
time.sleep(.005)
self.assertEqual(b.snapshot()['groups'][0]['state'],'COOLING')
self.assertEqual(self.request(b.issue_scope('second','teacher','glm-test'))[0],503)
self.assertEqual(first.result()[0],503)
self.assertEqual(len(p.requests),1)
def test_wall_clock_rollback_cannot_extend_queue_budget(self):
p=self.provider(lambda n,body:(429,{'Retry-After':'10'},b'{}'))
b=self.broker(p,self.policy(wait_budget_s=.05))
scope=b.issue_scope('e','teacher','glm-test')
before=time.monotonic(); wall=time.time()
with patch('harness.quota.time.time',return_value=wall-3600):
self.assertEqual(self.request(scope)[0],503)
self.assertLess(time.monotonic()-before,.5)
self.assertGreaterEqual(b.execution_status('e')['wait_spent'],.05)
def test_dns_deadline_discards_late_resolution_without_sending(self):
p=self.provider();b=self.broker(p,self.policy(connect_timeout_s=.03,request_total_s=.1))
scope=b.issue_scope('dns','teacher','glm-test')
original=socket.getaddrinfo;release=threading.Event();entered=threading.Event()
def blocked(host,port,*args,**kwargs):
if port==p.server.server_port:
entered.set();release.wait(2)
return original(host,port,*args,**kwargs)
try:
with patch('harness.quota.socket.getaddrinfo',side_effect=blocked):
code,_=self.request(scope)
self.assertTrue(entered.is_set());self.assertEqual(code,503)
self.assertFalse(release.is_set())
self.assertEqual(b.execution_status('dns')['generations'][0]['reason'],'DNS_TIMEOUT')
self.assertEqual(p.requests,[])
finally:
release.set()
time.sleep(.06)
self.assertEqual(p.requests,[]) # Resolver has no transport capability.
def test_cancel_while_dns_blocked_does_not_wait_for_group_lock(self):
p=self.provider();b=self.broker(p,self.policy(connect_timeout_s=1))
scope=b.issue_scope('dns-cancel','teacher','glm-test')
original=socket.getaddrinfo;release=threading.Event();entered=threading.Event()
def blocked(host,port,*args,**kwargs):
if port==p.server.server_port:
entered.set();release.wait(2)
return original(host,port,*args,**kwargs)
try:
with patch('harness.quota.socket.getaddrinfo',side_effect=blocked),ThreadPoolExecutor(max_workers=1) as pool:
pending=pool.submit(self.request,scope)
self.assertTrue(entered.wait(1))
b.cancel('dns-cancel')
self.assertEqual(pending.result(timeout=.5)[0],409)
self.assertFalse(release.is_set());self.assertEqual(p.requests,[])
finally:
release.set()
def test_confirmed_end_allows_new_probe_but_never_old_generation_replay(self):
finished=threading.Event()
def slow_body(out):
time.sleep(.1);out.write(ok()[2]);out.flush();finished.set()
p=self.provider(lambda n,body:(200,{'Content-Type':'application/json'},slow_body) if n==1 else ok(n,body))
b=self.broker(p,self.policy(stream_idle_timeout_s=.03))
original=b.issue_scope('original','teacher','glm-test')
self.assertEqual(self.request(original,'original-g')[0],503)
self.assertTrue(finished.wait(1))
attempt=b.snapshot()['attempts'][0]
b.confirm_end(attempt['attempt_id'],'verified-completion-receipt')
self.assertEqual(b.snapshot()['groups'][0]['state'],'COOLING')
self.assertEqual(self.request(b.issue_scope('new','teacher','glm-test'))[0],200)
self.assertEqual(b.snapshot()['attempts'][-1]['state_at_send'],'HALF_OPEN')
self.assertEqual(b.execution_status('original')['generations'][0]['state'],'PAUSED')
self.assertEqual(self.request(original,'new-g')[0],409)
self.assertEqual(len(p.requests),2)
def test_stalled_429_body_retains_received_rate_limit_headers(self):
def slow(out): time.sleep(.1);out.write(b'{}')
p=self.provider(lambda n,body:(429,{'Retry-After':'8','X-Request-Id':'trace-123'},slow))
b=self.broker(p,self.policy(stream_idle_timeout_s=.03))
self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'))[0],503)
attempt=b.snapshot()['attempts'][0]
self.assertEqual(attempt['http_status'],429)
self.assertEqual(json.loads(attempt['response_headers']),{'retry-after':'8','x-request-id':'trace-123'})
def test_stream_integrity_validates_every_choice_and_tool_arguments(self):
def stream(*choices):
return b''.join(b'data: '+json.dumps({'choices':chunk}).encode()+b'\n\n' for chunk in choices)+b'data: [DONE]\n\n'
complete={'index':0,'delta':{'role':'assistant','content':'ok'},'finish_reason':'stop'}
after={'index':0,'delta':{'tool_calls':[{'index':0,'id':'c','function':{'name':'bash','arguments':'{broken'}}]}}
fixtures=[
(stream([complete],[after]),'INVALID_PROVIDER_RESPONSE'),
(stream([complete,{'index':1,'delta':{'content':'unfinished'}}]),'INCOMPLETE_GENERATION'),
(stream([{'index':0,'delta':{},'finish_reason':'stop'}]),'INVALID_PROVIDER_RESPONSE'),
(stream([dict(after,finish_reason='tool_calls')]),'INVALID_PROVIDER_RESPONSE'),
(stream([complete]),'COMPLETE'),
(stream([{'index':0,'delta':{'tool_calls':[{'index':0,'id':'c','type':'function','function':{'name':'bash','arguments':'{"command":'}}]}}],
[{'index':0,'delta':{'tool_calls':[{'index':0,'function':{'arguments':'"echo ok"}'}}]},'finish_reason':'tool_calls'}]),'COMPLETE'),
]
for payload,expected in fixtures:
with self.subTest(expected=expected,payload=payload):
self.assertEqual(validate_completion(payload,True),expected)
p=self.provider(lambda n,body:(200,{'Content-Type':'text/event-stream'},fixtures[0][0]))
b=self.broker(p)
self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'),body={'model':'glm-test','messages':[],'max_tokens':5,'stream':True})[0],503)
self.assertNotEqual(b.execution_status('e')['generations'][0]['state'],'COMPLETE')
def test_independent_snapshot_is_readonly_bounded_and_missing_is_unknown(self):
absent=Path(self.tmp.name)/'missing'/'ledger'
self.assertEqual(quota_snapshot(absent)['status'],'UNKNOWN')
self.assertFalse(absent.parent.exists())
p=self.provider();b=self.broker(p)
for i in range(3): self.request(b.issue_scope(str(i),'teacher','glm-test'))
before=hashlib.sha256(self.ledger.read_bytes()).hexdigest()
with patch.object(Broker,'__init__',side_effect=AssertionError('must not construct')):
snap=quota_snapshot(self.ledger,limit=2)
self.assertEqual(snap['status'],'OBSERVED')
self.assertLessEqual(len(snap['attempts']),2);self.assertLessEqual(len(snap['events']),2)
self.assertNotIn('upstream-secret',json.dumps(snap));self.assertNotIn('token_hash',json.dumps(snap))
self.assertEqual(hashlib.sha256(self.ledger.read_bytes()).hexdigest(),before)
self.assertEqual(snap['groups'][0]['inflight'],0)
def test_connect_failure_pause_resumes_after_cooldown(self):
p=self.provider();b=self.broker(p,self.policy())
scope=b.issue_scope('blip','teacher','glm-test')
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
def refused(connection):
if connection.port==p.server.server_port:
raise ConnectionRefusedError('injected Wi-Fi blip')
return original(connection)
with patch('harness.quota.http.client.HTTPConnection.connect',new=refused):
self.assertEqual(self.request(scope,'g')[0],503)
with b._db() as con:
cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0]
with patch.object(b,'_now',return_value=max(b._now(),cooldown)+1):
self.assertTrue(b.resume_readiness('blip')['eligible'])
b.resume('blip','g',checkpoint_complete=True,side_effects_known=True)
self.assertEqual(self.request(scope,'g')[0],200)
self.assertEqual(len(p.requests),1)
def test_connect_blip_is_resent_in_place_and_charged_to_the_wait_budget(self):
# 10-14: one refused connect paused a 10-hour rollout; four of them stranded it at 3/3 resumes.
p=self.provider();b=self.broker(p,self.policy())
scope=b.issue_scope('blip','teacher','glm-test')
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
refusals=[2]
def flaky(connection):
if connection.port==p.server.server_port and refusals[0]:
refusals[0]-=1
raise ConnectionRefusedError('injected Wi-Fi blip')
return original(connection)
with patch('harness.quota.http.client.HTTPConnection.connect',new=flaky):
self.assertEqual(self.request(scope)[0],200)
self.assertEqual(len(p.requests),1)
self.assertGreaterEqual(b.execution_status('blip')['wait_spent'],2*(.06+.2))
self.assertEqual(len([e for e in b.snapshot()['quota_events'] if e['event']=='UNSENT_RETRY']),2)
def test_pause_that_sent_nothing_does_not_spend_the_resume_budget(self):
p=self.provider();b=self.broker(p,self.policy())
scope=b.issue_scope('free','teacher','glm-test')
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
def refused(connection):
if connection.port==p.server.server_port:
raise ConnectionRefusedError('injected outage')
return original(connection)
with patch('harness.quota.http.client.HTTPConnection.connect',new=refused):
self.assertEqual(self.request(scope,'g')[0],503) # in-place retries spend the wait budget, then pause
cap=b.groups['shared'].max_auto_resumes
with b._db() as con:
con.execute('UPDATE executions SET resumes=? WHERE execution_id=?',(cap,'free'))
cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0]
with patch.object(b,'_now',return_value=max(b._now(),cooldown)+1):
ready=b.resume_readiness('free')
self.assertTrue(ready['eligible']);self.assertFalse(ready['counted'])
b.resume('free','g',checkpoint_complete=True,side_effects_known=True)
self.assertEqual(b.execution_status('free')['resumes'],cap)
self.assertEqual(p.requests,[])
def test_confirmed_connect_failure_does_not_spend_http_budget(self):
p=self.provider();b=self.broker(p,self.policy(rpm=1,window_s=1))
scope=b.issue_scope('connect-failure','teacher','glm-test')
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
def refused(connection):
if connection.port==p.server.server_port:
raise ConnectionRefusedError('injected TCP refusal before HTTP')
return original(connection)
with patch('harness.quota.http.client.HTTPConnection.connect',new=refused):
self.assertEqual(self.request(scope)[0],503)
generation=b.execution_status('connect-failure')['generations'][0]
self.assertEqual(generation['attempts'],0)
self.assertEqual(generation['reason'],'CONNECT_FAILED')
self.assertEqual(generation['state'],'PAUSED')
self.assertEqual(p.requests,[])
snapshot=quota_snapshot(self.ledger)
self.assertEqual(snapshot['groups'][0]['rpm_used'],0)
self.assertEqual(snapshot['groups'][0]['tpm_reserved'],0)
self.assertEqual(snapshot['groups'][0]['inflight'],0)
self.assertTrue(all(row['sent']==0 and row['state']=='NOT_SENT' for row in snapshot['attempts']))
self.assertIn('TRANSPORT_NOT_SENT',[event['event'] for event in snapshot['events']])
# The same account can serve a later request; no phantom RPM reservation.
with b._db() as con:
cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0]
with patch.object(b,'_now',return_value=max(b._now(),cooldown)):
self.assertEqual(self.request(b.issue_scope('next','teacher','glm-test'))[0],200)
self.assertEqual(len(p.requests),1)
def test_unknown_send_never_refunds_http_budget(self):
p=self.provider();b=self.broker(p,self.policy(window_s=1))
scope=b.issue_scope('unknown-send','teacher','glm-test')
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.request
def interrupted(connection,*args,**kwargs):
original(connection,*args,**kwargs)
if connection.port==p.server.server_port:
raise OSError('injected interruption after request bytes')
with patch('harness.quota.http.client.HTTPConnection.request',new=interrupted):
self.assertEqual(self.request(scope)[0],503)
generation=b.execution_status('unknown-send')['generations'][0]
self.assertEqual(generation['attempts'],1)
snapshot=quota_snapshot(self.ledger)
self.assertEqual(snapshot['groups'][0]['rpm_used'],1)
self.assertGreater(snapshot['groups'][0]['tpm_reserved'],0)
self.assertEqual(snapshot['groups'][0]['inflight'],1)
self.assertEqual(snapshot['attempts'][0]['state'],'UNKNOWN')
self.assertIsNone(snapshot['attempts'][0]['ended_at'])
self.assertNotIn('TRANSPORT_NOT_SENT',[event['event'] for event in snapshot['events']])
def test_full_context_400_goes_back_to_pi_and_fences_nothing(self):
# 10-16: seven Qwen rollouts ended on vLLM's "maximum context length" 400: the agent's context, not the platform.
# Pi compacts on it and sends again, so the generation must fail without fencing the execution.
full = b'{"error":{"message":"This model\'s maximum context length is 262144 tokens. However, you requested 270000 tokens."}}'
p=self.provider(lambda n,body:(400,{'Content-Type':'application/json'},full) if n==1 else ok()); b=self.broker(p)
s=b.issue_scope('full','teacher','glm-test')
status,body=self.request(s,'g')[:2]
self.assertEqual(status,400); self.assertIn(b'context_length_exceeded',body)
first=b.execution_status('full')['generations'][0]
self.assertEqual((first['state'],first['reason']),('FAILED','CONTEXT_TOO_LONG'))
self.assertFalse(b.resume_readiness('full')['eligible'])
self.assertEqual(self.request(s,'g2')[0],200)
def test_request_over_the_account_limit_is_a_full_context_for_pi(self):
# 10-19 staging: a 413 Pi did not recognise ended the rollout as an infrastructure error instead of compacting.
calls=[]; p=self.provider(lambda n,body:(calls.append(n), ok())[1]); b=self.broker(p,self.policy(tpm=2000))
s=b.issue_scope('big','teacher','glm-test')
big={'model':'glm-test','messages':[{'role':'user','content':'x'*3000}],'max_tokens':5}
status,body=self.request(s,'g',big)[:2]
self.assertEqual((status,body),(400,CONTEXT_OVERFLOW_BODY))
self.assertEqual(calls,[])
first=b.execution_status('big')['generations'][0]
self.assertEqual((first['state'],first['reason']),('FAILED','CONTEXT_TOO_LONG'))
self.assertEqual(self.request(s,'g2')[0],200) # the compacted retry goes through
with self.assertRaisesRegex(QuotaError,'REQUEST_EXCEEDS_INTERNAL_TPM'): # content review keeps its reason
b.generate(b.issue_scope('review','teacher','glm-test').api_key,json.dumps(big).encode(),'review:0')
def test_other_400_stays_upstream_rejected(self):
p=self.provider(lambda n,body:(400,{'Content-Type':'application/json'},b'{"error":"invalid tool schema"}')); b=self.broker(p)
s=b.issue_scope('bad','teacher','glm-test')
self.assertEqual(self.request(s,'g')[0],503)
self.assertEqual(b.execution_status('bad')['generations'][0]['reason'],'UPSTREAM_REJECTED')
def test_dropped_connection_holds_the_account_briefly(self):
# 10-16 14:35: one dropped stream held 05's account until the request's own deadline (~25 min).
import struct
def cut(wfile):
wfile.write(b'data: {"choices":[{"delta":{"content":"par"}}]}\n\n'); wfile.flush()
wfile._sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
wfile._sock.close() # RST: the client sees a reset, not an end
p=self.provider(lambda n,body:(200,{'Content-Type':'text/event-stream'},cut))
b=self.broker(p,self.policy(request_total_s=600,stream_idle_timeout_s=5,first_byte_timeout_s=5))
s=b.issue_scope('cut','teacher','glm-test')
self.assertEqual(self.request(s,'g',{'model':'glm-test','messages':[],'max_tokens':5,'stream':True})[0],503)
attempt=b.snapshot()['attempts'][0]
self.assertEqual((attempt['state'],attempt['transport_reason']),('UNKNOWN','REQUEST_CONNECTION_INTERRUPTED'))
self.assertLessEqual(attempt['deadline']-attempt['started_at'],__import__('harness.quota',fromlist=['x']).UNKNOWN_HOLD_S+1)
b.pause_unsent('g') # possibly sent: an internal error later must not turn it into a replayable pause
self.assertNotEqual(b.execution_status('cut')['generations'][0]['reason'],'BROKER_ERROR')
def test_internal_error_before_sending_pauses_resumably(self):
# 10-16 05 hrnet.w32: an exception inside the broker ended a 2.9 h rollout as INFRA_INCOMPLETE_GENERATION.
p=self.provider(); b=self.broker(p); s=b.issue_scope('boom','teacher','glm-test')
with patch.object(b,'_generate',side_effect=RuntimeError('injected')):
self.assertEqual(self.request(s,'g'),(503,b'{"error":{"code":"BROKER_UNAVAILABLE"}}'))
gen=b.execution_status('boom')['generations'][0]
self.assertEqual((gen['state'],gen['reason']),('PAUSED','BROKER_ERROR'))
self.assertTrue(b.resume_readiness('boom')['eligible'])
b.resume('boom','g',checkpoint_complete=True,side_effects_known=True)
self.assertEqual(self.request(s,'g')[0],200)
self.assertEqual(len(p.requests),1)
def test_invalid_budget_rejected(self):
with self.assertRaises(ValueError): self.policy(rpm=None)
def test_model_scope_cannot_change(self):
p=self.provider(); b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
self.assertEqual(self.request(s,body={'model':'other','messages':[],'max_tokens':2})[0],403)
self.assertEqual(p.requests,[])
if __name__=='__main__': unittest.main()