Pi requests could hit Node fetch's independent five-minute header deadline, and queued requests could outlive their local client or terminal rollout. Set header/body deadlines only for the loopback broker origin, cancel scopes when clients disconnect or executions terminate, and retain uncertain upstream reservations instead of retrying after cancellation. Resumable pauses remain open. Validation: real Pi SDK accepted a response delayed 310 seconds (314.139 s test); quota, Pi session, acceptance, cancellation, and lifecycle regression coverage.
621 lines
37 KiB
Python
621 lines
37 KiB
Python
"""Quota acceptance uses real local HTTP, persistent ledgers and no provider account."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
import json
|
|
import multiprocessing
|
|
import socket
|
|
import hashlib
|
|
from pathlib import Path
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
from harness.quota import Broker, QuotaPolicy, Upstream, QuotaError, quota_snapshot, validate_completion
|
|
|
|
|
|
class Provider:
|
|
def __init__(self, responder):
|
|
self.responder, self.requests = responder, []
|
|
outer = self
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *_): pass
|
|
def do_POST(self):
|
|
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
|
outer.requests.append({'at': time.time(), 'body': body,
|
|
'authorization': self.headers.get('Authorization')})
|
|
code, headers, payload = outer.responder(len(outer.requests), body)
|
|
self.send_response(code)
|
|
for key, value in headers.items(): self.send_header(key, str(value))
|
|
self.end_headers()
|
|
try:
|
|
if callable(payload): payload(self.wfile)
|
|
else: self.wfile.write(payload)
|
|
except (BrokenPipeError, ConnectionResetError): pass
|
|
self.server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
|
|
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
|
self.thread.start()
|
|
self.url = 'http://127.0.0.1:%s/v1' % self.server.server_port
|
|
def close(self):
|
|
self.server.shutdown(); self.server.server_close(); self.thread.join()
|
|
|
|
|
|
def ok(_n=0, body=None):
|
|
if body and body.get('stream'):
|
|
return 200, {'Content-Type': 'text/event-stream'}, (b'data: {"choices":[{"delta":{"content":"ok"},"finish_reason":null}]}\n\n'
|
|
b'data: {"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}\n\n'
|
|
b'data: [DONE]\n\n')
|
|
return 200, {'Content-Type': 'application/json'}, b'{"choices":[{"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}'
|
|
|
|
|
|
def _process_broker(ledger, url, policy, execution, ready, start, result):
|
|
# Spawned interpreters do not inherit the parent's locks or server objects.
|
|
with Broker(Path(ledger), {'shared':policy}, {'teacher':Upstream('shared',url,'upstream-secret',('glm-test',))}) as broker:
|
|
scope=broker.issue_scope(execution,'teacher','glm-test')
|
|
ready.put(True)
|
|
if not start.wait(5):
|
|
result.put('START_TIMEOUT');return
|
|
request=urllib.request.Request(scope.base_url+'/chat/completions',
|
|
json.dumps({'model':'glm-test','messages':[],'max_tokens':5}).encode(),
|
|
{'Authorization':'Bearer '+scope.api_key,'Content-Type':'application/json'})
|
|
try:
|
|
with urllib.request.urlopen(request,timeout=5) as response:
|
|
response.read();result.put(response.status)
|
|
except urllib.error.HTTPError as error:
|
|
error.read();result.put(error.code)
|
|
|
|
|
|
class Quota(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
|
|
self.ledger = Path(self.tmp.name) / 'shared.sqlite'
|
|
def provider(self, responder=ok):
|
|
item = Provider(responder); self.addCleanup(item.close); return item
|
|
def policy(self, **kw):
|
|
return QuotaPolicy(**dict({'max_concurrency':2, 'rpm':1000, 'tpm':1000000,
|
|
'window_s':.2, 'cooldown_initial_s':.06, 'cooldown_local_max_s':.1,
|
|
'jitter_fraction':0, 'recovery_window_s':.12, 'recovery_min_successes':2,
|
|
'wait_budget_s':2, 'connect_timeout_s':.2, 'first_byte_timeout_s':.3,
|
|
'stream_idle_timeout_s':.15, 'request_total_s':.6, 'poll_s':.01}, **kw))
|
|
def broker(self, provider, policy=None, aliases=None):
|
|
b = Broker(self.ledger, {'shared':policy or self.policy()}, aliases or {
|
|
'teacher':Upstream('shared', provider.url, 'upstream-secret', ('glm-test',))})
|
|
b.start(); self.addCleanup(b.close); return b
|
|
def request(self, scope, generation=None, body=None):
|
|
headers = {'Authorization':'Bearer '+scope.api_key, 'Content-Type':'application/json'}
|
|
if generation: headers['X-Bench-Generation-Id'] = generation
|
|
request = urllib.request.Request(scope.base_url+'/chat/completions',
|
|
json.dumps(body or {'model':'glm-test', 'messages':[{'role':'user','content':'hi'}], 'max_tokens':5}).encode(), headers)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=5) as response:
|
|
return response.status, response.read()
|
|
except urllib.error.HTTPError as error:
|
|
return error.code, error.read()
|
|
def test_success_scope_and_stream_usage(self):
|
|
p=self.provider(); b=self.broker(p); scope=b.issue_scope('execution', 'teacher', 'glm-test')
|
|
code, data=self.request(scope, 'generation', {'model':'glm-test','messages':[],'max_tokens':4,'stream':True})
|
|
self.assertEqual(code, 200); self.assertIn(b'[DONE]', data)
|
|
self.assertEqual(p.requests[0]['authorization'], 'Bearer upstream-secret')
|
|
self.assertNotIn('upstream-secret', scope.api_key)
|
|
self.assertEqual(b.snapshot()['attempts'][0]['charged_tokens'], 3)
|
|
self.assertEqual(len(p.requests), 1)
|
|
self.assertEqual(self.request(scope, 'generation')[0], 409)
|
|
|
|
def _raw_request(self, scope):
|
|
from urllib.parse import urlsplit
|
|
url = urlsplit(scope.base_url)
|
|
sock = socket.create_connection((url.hostname, url.port), timeout=2)
|
|
body = json.dumps({'model':'glm-test', 'messages':[], 'max_tokens':5}).encode()
|
|
sock.sendall((f'POST /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\n'
|
|
f'Authorization: Bearer {scope.api_key}\r\nContent-Length: {len(body)}\r\n'
|
|
'Content-Type: application/json\r\n\r\n').encode() + body)
|
|
return sock
|
|
|
|
def test_disconnected_queued_client_never_reaches_upstream(self):
|
|
p = self.provider(); b = self.broker(p, self.policy(rpm=1, window_s=.4))
|
|
self.assertEqual(self.request(b.issue_scope('first', 'teacher', 'glm-test'))[0], 200)
|
|
scope = b.issue_scope('disconnected', 'teacher', 'glm-test')
|
|
sock = self._raw_request(scope)
|
|
deadline = time.monotonic() + 1
|
|
while not b.execution_status('disconnected')['generations'] and time.monotonic() < deadline:
|
|
time.sleep(.005)
|
|
self.assertTrue(b.execution_status('disconnected')['generations'])
|
|
sock.close()
|
|
deadline = time.monotonic() + 1
|
|
while not b.execution_status('disconnected')['cancelled'] and time.monotonic() < deadline:
|
|
time.sleep(.005)
|
|
self.assertTrue(b.execution_status('disconnected')['cancelled'])
|
|
time.sleep(.45) # RPM capacity becomes free after the client has gone.
|
|
self.assertEqual(len(p.requests), 1)
|
|
gen = b.execution_status('disconnected')['generations'][0]
|
|
self.assertEqual((gen['state'], gen['attempts']), ('CANCELLED', 0))
|
|
self.assertEqual(b.snapshot()['groups'][0]['state'], 'OPEN')
|
|
|
|
def test_disconnected_sent_client_keeps_unknown_reservation(self):
|
|
release = threading.Event()
|
|
self.addCleanup(release.set)
|
|
p = self.provider(lambda *_: (release.wait(2) or True) and ok())
|
|
b = self.broker(p)
|
|
sock = self._raw_request(b.issue_scope('disconnected', 'teacher', 'glm-test'))
|
|
deadline = time.monotonic() + 1
|
|
while not p.requests and time.monotonic() < deadline:
|
|
time.sleep(.005)
|
|
self.assertEqual(len(p.requests), 1)
|
|
sock.close()
|
|
deadline = time.monotonic() + 1
|
|
while b.snapshot()['attempts'][0]['state'] != 'UNKNOWN' and time.monotonic() < deadline:
|
|
time.sleep(.005)
|
|
attempt = b.snapshot()['attempts'][0]
|
|
self.assertEqual(attempt['state'], 'UNKNOWN')
|
|
self.assertIsNone(attempt['ended_at'])
|
|
self.assertGreater(attempt['charged_tokens'], 0)
|
|
self.assertTrue(b.execution_status('disconnected')['cancelled'])
|
|
self.assertEqual(len(p.requests), 1)
|
|
release.set()
|
|
|
|
def test_valid_sse_done_does_not_wait_for_http_close(self):
|
|
for crlf in (False, True):
|
|
with self.subTest(crlf=crlf), tempfile.TemporaryDirectory() as temp:
|
|
payload = ok(body={'stream':True})[2]
|
|
if crlf:
|
|
payload = payload.replace(b'\n', b'\r\n')
|
|
release = threading.Event()
|
|
def send(writer):
|
|
# Split inside the terminator to exercise real read boundaries.
|
|
cut = payload.index(b'[DONE]') + 3
|
|
writer.write(payload[:cut]); writer.flush()
|
|
time.sleep(.02)
|
|
writer.write(payload[cut:]); writer.flush()
|
|
release.wait(2)
|
|
provider = self.provider(lambda *_: (200, {'Content-Type':'text/event-stream'}, send))
|
|
with Broker(Path(temp)/'quota.sqlite', {'shared':self.policy()}, {
|
|
'teacher':Upstream('shared',provider.url,'fake-only',('glm-test',))}) as broker:
|
|
try:
|
|
scope = broker.issue_scope('execution','teacher','glm-test')
|
|
code, data = self.request(scope,'generation',{'model':'glm-test','messages':[], 'max_tokens':4,'stream':True})
|
|
self.assertEqual((code,data),(200,payload))
|
|
attempt = broker.snapshot()['attempts'][0]
|
|
self.assertEqual((attempt['state'],attempt['reason'],attempt['charged_tokens']),('FINISHED','COMPLETE',3))
|
|
self.assertEqual(broker.snapshot()['groups'][0]['state'],'OPEN')
|
|
self.assertEqual(len(provider.requests),1)
|
|
finally:
|
|
release.set()
|
|
|
|
def test_sse_terminator_cannot_complete_invalid_or_unframed_response(self):
|
|
complete = ok(body={'stream':True})[2]
|
|
variants = {
|
|
'missing_blank_line': complete[:-2],
|
|
'content_literal_only': b'data: {"choices":[{"delta":{"content":"[DONE]"},"finish_reason":"stop"}]}\n\n',
|
|
'missing_finish': complete.split(b'data: {"choices":[{"delta":{}')[0] + b'data: [DONE]\n\n',
|
|
'partial_tool': b'data: {"choices":[{"delta":{"tool_calls":[{"id":"x","type":"function","function":{"name":"write","arguments":"{"}}]},"finish_reason":"tool_calls"}]}\n\ndata: [DONE]\n\n',
|
|
'extra_data_after_done': complete + b'data: {"choices":[]}\n\n',
|
|
}
|
|
for name,payload in variants.items():
|
|
with self.subTest(name=name), tempfile.TemporaryDirectory() as temp:
|
|
release = threading.Event()
|
|
def send(writer):
|
|
writer.write(payload); writer.flush(); release.wait(2)
|
|
provider = self.provider(lambda *_: (200, {'Content-Type':'text/event-stream'},send))
|
|
with Broker(Path(temp)/'quota.sqlite',{'shared':self.policy()}, {
|
|
'teacher':Upstream('shared',provider.url,'fake-only',('glm-test',))}) as broker:
|
|
try:
|
|
scope = broker.issue_scope('execution','teacher','glm-test')
|
|
code,_ = self.request(scope,'generation',{'model':'glm-test','messages':[],'max_tokens':4,'stream':True})
|
|
self.assertNotEqual(code,200)
|
|
attempt = broker.snapshot()['attempts'][0]
|
|
self.assertEqual((attempt['state'],attempt['transport_reason']),('UNKNOWN','STREAM_IDLE_TIMEOUT'))
|
|
self.assertEqual(len(provider.requests),1)
|
|
finally:
|
|
release.set()
|
|
def test_JOINT001_persistent_429_budget_and_single_probe(self):
|
|
p=self.provider(lambda n, body:(429, {'Retry-After':'.06'}, b'{"error":{"code":"rate_limit"}}'))
|
|
b=self.broker(p, self.policy(max_attempts=4))
|
|
scopes=[b.issue_scope('e'+str(i),'teacher','glm-test') for i in range(4)]
|
|
with ThreadPoolExecutor(max_workers=4) as pool:
|
|
results=list(pool.map(lambda s:self.request(s),scopes))
|
|
snapshot=b.snapshot()
|
|
self.assertTrue(all(code==503 for code,_ in results))
|
|
self.assertTrue(all(g['attempts']<=4 for g in snapshot['generations']))
|
|
self.assertTrue(all(t['state_at_send'] in ('OPEN','HALF_OPEN','RECOVERING') for t in snapshot['attempts']))
|
|
self.assertEqual(len(p.requests), len(snapshot['attempts']))
|
|
probes=[a for a in snapshot['attempts'] if a['state_at_send']=='HALF_OPEN']
|
|
for before,after in zip(probes,probes[1:]):
|
|
self.assertGreaterEqual(after['started_at'],before['ended_at'])
|
|
self.assertEqual(len({a['attempt_id'] for a in snapshot['attempts']}),len(snapshot['attempts']))
|
|
def test_QUOTA001_shared_alias_group_independent_group(self):
|
|
p=self.provider(lambda n, body:(429, {'Retry-After':'1'}, b'{}') if body['model']=='limited' else ok(n,body))
|
|
b=Broker(self.ledger, {'shared':self.policy(wait_budget_s=.12), 'independent':self.policy()}, {
|
|
'a':Upstream('shared',p.url,'a',('limited',)), 'b':Upstream('shared',p.url,'b',('limited',)),
|
|
'other':Upstream('independent',p.url,'c',('free',))})
|
|
b.start(); self.addCleanup(b.close)
|
|
self.request(b.issue_scope('a','a','limited'),body={'model':'limited','messages':[],'max_tokens':1})
|
|
self.request(b.issue_scope('b','b','limited'),body={'model':'limited','messages':[],'max_tokens':1})
|
|
self.assertEqual(len(p.requests),1)
|
|
self.assertEqual(self.request(b.issue_scope('c','other','free'),body={'model':'free','messages':[],'max_tokens':1})[0],200)
|
|
def test_JOINT003_restart_keeps_budget_and_missing_usage(self):
|
|
p=self.provider(lambda n,body:(200,{'Content-Type':'application/json'},b'{"choices":[{"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}'))
|
|
b=self.broker(p,self.policy(rpm=1,window_s=1,wait_budget_s=.06))
|
|
self.assertEqual(self.request(b.issue_scope('e1','teacher','glm-test'))[0],200)
|
|
first=b.snapshot()['attempts'][0]
|
|
self.assertEqual(first['charged_tokens'],first['reserved_tokens'])
|
|
b.close()
|
|
b2=self.broker(p,self.policy(rpm=1,window_s=1,wait_budget_s=.06))
|
|
self.assertEqual(self.request(b2.issue_scope('e2','teacher','glm-test'))[0],503)
|
|
self.assertEqual(len(p.requests),1)
|
|
def test_cancel_blocks_transport_without_hitting_upstream(self):
|
|
p=self.provider(); b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
|
|
b.cancel('e'); self.assertEqual(self.request(s)[0],409); self.assertEqual(p.requests,[])
|
|
def test_QUOTA002_retry_after_and_auth_pause(self):
|
|
p=self.provider(lambda n,body:(429,{'Retry-After':'4'},b'{}'))
|
|
b=self.broker(p,self.policy(wait_budget_s=.1)); before=time.time()
|
|
self.request(b.issue_scope('e','teacher','glm-test'))
|
|
self.assertGreaterEqual(b.snapshot()['groups'][0]['cooldown_until'],before+4)
|
|
self.assertEqual(len(p.requests),1)
|
|
def test_JOINT004_nonempty_stream_interruption_does_not_retry(self):
|
|
p=self.provider(lambda n,body:(200,{'Content-Type':'text/event-stream'}, b'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'))
|
|
b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
|
|
self.assertEqual(self.request(s,body={'model':'glm-test','messages':[],'max_tokens':5,'stream':True})[0],503)
|
|
self.assertEqual(len(p.requests),1)
|
|
self.assertEqual(b.snapshot()['generations'][0]['reason'],'INCOMPLETE_GENERATION')
|
|
self.assertEqual(self.request(s,'new-id')[0],409)
|
|
def test_JOINT002_recovery_adds_at_most_one_per_window(self):
|
|
p=self.provider(lambda n,body:(429,{},b'{}') if n==1 else ok(n,body))
|
|
policy=self.policy(max_concurrency=4,recovery_window_s=.12)
|
|
b=self.broker(p,policy)
|
|
self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'))[0],200)
|
|
self.assertEqual(b.snapshot()['groups'][0]['state'],'RECOVERING')
|
|
self.assertEqual(b.snapshot()['groups'][0]['target'],1)
|
|
clock=[b.snapshot()['groups'][0]['recovery_at']]
|
|
# Only quota's durable clock is controlled. HTTP itself remains real.
|
|
with patch.object(b,'_now',side_effect=lambda:clock[0]):
|
|
for i in range(3):
|
|
self.assertEqual(self.request(b.issue_scope('e'+str(i),'teacher','glm-test'))[0],200)
|
|
self.assertEqual(b.snapshot()['groups'][0]['target'],1)
|
|
clock[0]+=.13
|
|
self.request(b.issue_scope('next','teacher','glm-test'))
|
|
self.assertEqual(b.snapshot()['groups'][0]['target'],1)
|
|
self.request(b.issue_scope('next2','teacher','glm-test'))
|
|
self.assertEqual(b.snapshot()['groups'][0]['target'],2)
|
|
self.request(b.issue_scope('after','teacher','glm-test'))
|
|
self.assertEqual(b.snapshot()['groups'][0]['target'],2)
|
|
|
|
def test_shared_brokers_enforce_actual_RPM(self):
|
|
p=self.provider(); policy=self.policy(rpm=1,window_s=.7,wait_budget_s=.12)
|
|
a=self.broker(p,policy); b=self.broker(p,policy)
|
|
scopes=[a.issue_scope('a','teacher','glm-test'),b.issue_scope('b','teacher','glm-test')]
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
results=list(pool.map(self.request,scopes))
|
|
self.assertEqual(sorted(code for code,_ in results),[200,503])
|
|
self.assertEqual(len(p.requests),1)
|
|
self.assertEqual(len(a.snapshot()['attempts']),1)
|
|
|
|
def test_missing_usage_reservation_blocks_TPM_until_end_window(self):
|
|
p=self.provider(lambda n,body:(200,{},b'{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}]}'))
|
|
b=self.broker(p,self.policy(tpm=300,window_s=.7,wait_budget_s=.08))
|
|
self.assertEqual(self.request(b.issue_scope('a','teacher','glm-test'))[0],200)
|
|
self.assertEqual(self.request(b.issue_scope('b','teacher','glm-test'))[0],503)
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_QUOTA003_wait_and_attempts_never_reset_on_resume(self):
|
|
p=self.provider(lambda n,body:(429,{'Retry-After':'3'},b'{}'))
|
|
b=self.broker(p,self.policy(wait_budget_s=.07))
|
|
scope=b.issue_scope('e','teacher','glm-test')
|
|
self.assertEqual(self.request(scope,'stable')[0],503)
|
|
spent=b.execution_status('e')['wait_spent']
|
|
self.assertFalse(b.resume_readiness('e')['eligible'])
|
|
with self.assertRaises(QuotaError):
|
|
b.resume('e','stable',checkpoint_complete=True,side_effects_known=True)
|
|
self.assertEqual(b.execution_status('e')['resumes'],0)
|
|
clock=[b._now()]
|
|
with patch.object(b,'_now',side_effect=lambda:clock[0]):
|
|
for i in range(3):
|
|
clock[0]+=4
|
|
self.assertTrue(b.resume_readiness('e')['eligible'])
|
|
b.resume('e','stable',checkpoint_complete=True,side_effects_known=True)
|
|
self.assertEqual(self.request(scope,'stable')[0],503)
|
|
self.assertGreaterEqual(b.execution_status('e')['wait_spent'],spent)
|
|
self.assertEqual(len(p.requests),4)
|
|
self.assertEqual(b.execution_status('e')['resumes'],3)
|
|
with self.assertRaises(QuotaError):
|
|
b.resume('e','stable',checkpoint_complete=True,side_effects_known=True)
|
|
self.assertEqual(self.request(scope,'replacement')[0],409)
|
|
|
|
def test_first_byte_timeout_quarantines_unknown_request(self):
|
|
def delay(n,body):
|
|
time.sleep(.2);return ok(n,body)
|
|
p=self.provider(delay); b=self.broker(p,self.policy(first_byte_timeout_s=.04))
|
|
s=b.issue_scope('e','teacher','glm-test')
|
|
self.assertEqual(self.request(s)[0],503)
|
|
snap=b.snapshot(); self.assertEqual(len(p.requests),1)
|
|
self.assertEqual(snap['attempts'][0]['transport_reason'],'FIRST_BYTE_TIMEOUT')
|
|
self.assertIsNone(snap['attempts'][0]['ended_at'])
|
|
self.assertEqual(snap['groups'][0]['state'],'RECONCILING')
|
|
self.assertEqual(self.request(b.issue_scope('e2','teacher','glm-test'))[0],503)
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_stream_idle_and_busy_total_timeout_are_bounded(self):
|
|
for mode in ('idle','busy'):
|
|
with self.subTest(mode=mode):
|
|
# Separate ledger avoids deliberately quarantined prior group.
|
|
self.ledger=Path(self.tmp.name)/(mode+'.sqlite')
|
|
def payload(out):
|
|
for i in range(30):
|
|
out.write(b'data: {"choices":[{"delta":{"content":"x"}}]}\n\n');out.flush()
|
|
time.sleep(.2 if mode=='idle' else .01)
|
|
p=self.provider(lambda n,body:(200,{'Content-Type':'text/event-stream'},payload))
|
|
b=self.broker(p,self.policy(stream_idle_timeout_s=.05,request_total_s=.12))
|
|
started=time.monotonic()
|
|
self.assertEqual(self.request(b.issue_scope(mode,'teacher','glm-test'),body={'model':'glm-test','messages':[],'stream':True,'max_tokens':4})[0],503)
|
|
self.assertLess(time.monotonic()-started,.3)
|
|
attempt=b.snapshot()['attempts'][0]
|
|
self.assertEqual(attempt['reason'],'INCOMPLETE_GENERATION')
|
|
self.assertEqual(attempt['transport_reason'],'STREAM_IDLE_TIMEOUT' if mode=='idle' else 'REQUEST_TOTAL_TIMEOUT')
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_cancel_queued_execution_fences_later_send(self):
|
|
p=self.provider(lambda n,body:(429,{'Retry-After':'1'},b'{}'))
|
|
b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
pending=pool.submit(self.request,s)
|
|
deadline=time.monotonic()+1
|
|
while not p.requests and time.monotonic()<deadline: time.sleep(.005)
|
|
b.cancel('e')
|
|
self.assertEqual(pending.result()[0],409)
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_5xx_retries_and_auth_pauses_without_provider_body_leak(self):
|
|
p=self.provider(lambda n,body:(503,{},b'secret provider failure') if n==1 else ok(n,body))
|
|
b=self.broker(p); self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'))[0],200)
|
|
self.assertEqual(len(p.requests),2)
|
|
self.assertNotIn('upstream-secret',json.dumps(b.snapshot()))
|
|
self.assertNotIn('secret provider failure',json.dumps(b.snapshot()))
|
|
|
|
def test_concurrency_is_kept_until_response_end(self):
|
|
active=peak=0
|
|
lock=threading.Lock()
|
|
def responder(n,body):
|
|
nonlocal active,peak
|
|
with lock:
|
|
active+=1;peak=max(peak,active)
|
|
time.sleep(.05)
|
|
with lock: active-=1
|
|
return ok(n,body)
|
|
p=self.provider(responder); b=self.broker(p,self.policy(max_concurrency=1))
|
|
scopes=[b.issue_scope(str(i),'teacher','glm-test') for i in range(4)]
|
|
with ThreadPoolExecutor(max_workers=4) as pool:
|
|
results=list(pool.map(self.request,scopes))
|
|
self.assertTrue(all(code==200 for code,_ in results))
|
|
self.assertEqual(peak,1)
|
|
self.assertEqual(len(p.requests),4)
|
|
|
|
def test_cancel_active_request_interrupts_without_refund(self):
|
|
def delay(n,body):
|
|
time.sleep(.4);return ok(n,body)
|
|
p=self.provider(delay); b=self.broker(p); scope=b.issue_scope('e','teacher','glm-test')
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
future=pool.submit(self.request,scope)
|
|
until=time.monotonic()+1
|
|
while not p.requests and time.monotonic()<until: time.sleep(.005)
|
|
b.cancel('e')
|
|
self.assertEqual(future.result()[0],503)
|
|
self.assertEqual(len(p.requests),1)
|
|
self.assertIsNone(b.snapshot()['attempts'][0]['ended_at'])
|
|
self.assertEqual(b.execution_status('e')['generations'][0]['reason'],'CANCELLED')
|
|
|
|
def test_auth_failure_pauses_group_and_no_new_id_bypasses_it(self):
|
|
p=self.provider(lambda n,body:(401,{},b'upstream-private-diagnostic'))
|
|
b=self.broker(p); scope=b.issue_scope('e','teacher','glm-test')
|
|
code,body=self.request(scope)
|
|
self.assertEqual(code,503);self.assertNotIn(b'upstream-private',body)
|
|
self.assertEqual(b.snapshot()['groups'][0]['state'],'PAUSED')
|
|
self.assertEqual(self.request(b.issue_scope('another','teacher','glm-test'))[0],503)
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_independent_processes_share_RPM_ledger(self):
|
|
p=self.provider(); policy=self.policy(rpm=1,window_s=1,wait_budget_s=.12)
|
|
# Initialize the immutable group policy before simultaneous processes.
|
|
Broker(self.ledger,{'shared':policy},{'teacher':Upstream('shared',p.url,'upstream-secret',('glm-test',))})
|
|
context=multiprocessing.get_context('spawn')
|
|
ready,start,result=context.Queue(),context.Event(),context.Queue()
|
|
processes=[context.Process(target=_process_broker,args=(str(self.ledger),p.url,policy,str(i),ready,start,result)) for i in range(2)]
|
|
try:
|
|
for process in processes: process.start()
|
|
for _ in processes: self.assertTrue(ready.get(timeout=8))
|
|
start.set()
|
|
self.assertEqual(sorted(result.get(timeout=8) for _ in processes),[200,503])
|
|
for process in processes:
|
|
process.join(timeout=5);self.assertEqual(process.exitcode,0)
|
|
self.assertEqual(len(p.requests),1)
|
|
finally:
|
|
for process in processes:
|
|
if process.is_alive(): process.terminate();process.join(timeout=2)
|
|
for queue in (ready,result): queue.close();queue.join_thread()
|
|
|
|
def test_429_headers_fence_sends_before_error_body_finishes(self):
|
|
def stalled_body(out):
|
|
time.sleep(.2);out.write(b'{}')
|
|
p=self.provider(lambda n,body:(429,{'Retry-After':'2'},stalled_body))
|
|
b=self.broker(p,self.policy(wait_budget_s=.06,stream_idle_timeout_s=.3))
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
first=pool.submit(self.request,b.issue_scope('first','teacher','glm-test'))
|
|
deadline=time.monotonic()+1
|
|
while b.snapshot()['groups'][0]['state']!='COOLING' and time.monotonic()<deadline:
|
|
time.sleep(.005)
|
|
self.assertEqual(b.snapshot()['groups'][0]['state'],'COOLING')
|
|
self.assertEqual(self.request(b.issue_scope('second','teacher','glm-test'))[0],503)
|
|
self.assertEqual(first.result()[0],503)
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_wall_clock_rollback_cannot_extend_queue_budget(self):
|
|
p=self.provider(lambda n,body:(429,{'Retry-After':'10'},b'{}'))
|
|
b=self.broker(p,self.policy(wait_budget_s=.05))
|
|
scope=b.issue_scope('e','teacher','glm-test')
|
|
before=time.monotonic(); wall=time.time()
|
|
with patch('harness.quota.time.time',return_value=wall-3600):
|
|
self.assertEqual(self.request(scope)[0],503)
|
|
self.assertLess(time.monotonic()-before,.5)
|
|
self.assertGreaterEqual(b.execution_status('e')['wait_spent'],.05)
|
|
|
|
def test_dns_deadline_discards_late_resolution_without_sending(self):
|
|
p=self.provider();b=self.broker(p,self.policy(connect_timeout_s=.03,request_total_s=.1))
|
|
scope=b.issue_scope('dns','teacher','glm-test')
|
|
original=socket.getaddrinfo;release=threading.Event();entered=threading.Event()
|
|
def blocked(host,port,*args,**kwargs):
|
|
if port==p.server.server_port:
|
|
entered.set();release.wait(2)
|
|
return original(host,port,*args,**kwargs)
|
|
try:
|
|
with patch('harness.quota.socket.getaddrinfo',side_effect=blocked):
|
|
code,_=self.request(scope)
|
|
self.assertTrue(entered.is_set());self.assertEqual(code,503)
|
|
self.assertFalse(release.is_set())
|
|
self.assertEqual(b.execution_status('dns')['generations'][0]['reason'],'DNS_TIMEOUT')
|
|
self.assertEqual(p.requests,[])
|
|
finally:
|
|
release.set()
|
|
time.sleep(.06)
|
|
self.assertEqual(p.requests,[]) # Resolver has no transport capability.
|
|
|
|
def test_cancel_while_dns_blocked_does_not_wait_for_group_lock(self):
|
|
p=self.provider();b=self.broker(p,self.policy(connect_timeout_s=1))
|
|
scope=b.issue_scope('dns-cancel','teacher','glm-test')
|
|
original=socket.getaddrinfo;release=threading.Event();entered=threading.Event()
|
|
def blocked(host,port,*args,**kwargs):
|
|
if port==p.server.server_port:
|
|
entered.set();release.wait(2)
|
|
return original(host,port,*args,**kwargs)
|
|
try:
|
|
with patch('harness.quota.socket.getaddrinfo',side_effect=blocked),ThreadPoolExecutor(max_workers=1) as pool:
|
|
pending=pool.submit(self.request,scope)
|
|
self.assertTrue(entered.wait(1))
|
|
b.cancel('dns-cancel')
|
|
self.assertEqual(pending.result(timeout=.5)[0],409)
|
|
self.assertFalse(release.is_set());self.assertEqual(p.requests,[])
|
|
finally:
|
|
release.set()
|
|
|
|
def test_confirmed_end_allows_new_probe_but_never_old_generation_replay(self):
|
|
finished=threading.Event()
|
|
def slow_body(out):
|
|
time.sleep(.1);out.write(ok()[2]);out.flush();finished.set()
|
|
p=self.provider(lambda n,body:(200,{'Content-Type':'application/json'},slow_body) if n==1 else ok(n,body))
|
|
b=self.broker(p,self.policy(stream_idle_timeout_s=.03))
|
|
original=b.issue_scope('original','teacher','glm-test')
|
|
self.assertEqual(self.request(original,'original-g')[0],503)
|
|
self.assertTrue(finished.wait(1))
|
|
attempt=b.snapshot()['attempts'][0]
|
|
b.confirm_end(attempt['attempt_id'],'verified-completion-receipt')
|
|
self.assertEqual(b.snapshot()['groups'][0]['state'],'COOLING')
|
|
self.assertEqual(self.request(b.issue_scope('new','teacher','glm-test'))[0],200)
|
|
self.assertEqual(b.snapshot()['attempts'][-1]['state_at_send'],'HALF_OPEN')
|
|
self.assertEqual(b.execution_status('original')['generations'][0]['state'],'PAUSED')
|
|
self.assertEqual(self.request(original,'new-g')[0],409)
|
|
self.assertEqual(len(p.requests),2)
|
|
|
|
def test_stalled_429_body_retains_received_rate_limit_headers(self):
|
|
def slow(out): time.sleep(.1);out.write(b'{}')
|
|
p=self.provider(lambda n,body:(429,{'Retry-After':'8','X-Request-Id':'trace-123'},slow))
|
|
b=self.broker(p,self.policy(stream_idle_timeout_s=.03))
|
|
self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'))[0],503)
|
|
attempt=b.snapshot()['attempts'][0]
|
|
self.assertEqual(attempt['http_status'],429)
|
|
self.assertEqual(json.loads(attempt['response_headers']),{'retry-after':'8','x-request-id':'trace-123'})
|
|
|
|
def test_stream_integrity_validates_every_choice_and_tool_arguments(self):
|
|
def stream(*choices):
|
|
return b''.join(b'data: '+json.dumps({'choices':chunk}).encode()+b'\n\n' for chunk in choices)+b'data: [DONE]\n\n'
|
|
complete={'index':0,'delta':{'role':'assistant','content':'ok'},'finish_reason':'stop'}
|
|
after={'index':0,'delta':{'tool_calls':[{'index':0,'id':'c','function':{'name':'bash','arguments':'{broken'}}]}}
|
|
fixtures=[
|
|
(stream([complete],[after]),'INVALID_PROVIDER_RESPONSE'),
|
|
(stream([complete,{'index':1,'delta':{'content':'unfinished'}}]),'INCOMPLETE_GENERATION'),
|
|
(stream([{'index':0,'delta':{},'finish_reason':'stop'}]),'INVALID_PROVIDER_RESPONSE'),
|
|
(stream([dict(after,finish_reason='tool_calls')]),'INVALID_PROVIDER_RESPONSE'),
|
|
(stream([complete]),'COMPLETE'),
|
|
(stream([{'index':0,'delta':{'tool_calls':[{'index':0,'id':'c','type':'function','function':{'name':'bash','arguments':'{"command":'}}]}}],
|
|
[{'index':0,'delta':{'tool_calls':[{'index':0,'function':{'arguments':'"echo ok"}'}}]},'finish_reason':'tool_calls'}]),'COMPLETE'),
|
|
]
|
|
for payload,expected in fixtures:
|
|
with self.subTest(expected=expected,payload=payload):
|
|
self.assertEqual(validate_completion(payload,True),expected)
|
|
p=self.provider(lambda n,body:(200,{'Content-Type':'text/event-stream'},fixtures[0][0]))
|
|
b=self.broker(p)
|
|
self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'),body={'model':'glm-test','messages':[],'max_tokens':5,'stream':True})[0],503)
|
|
self.assertNotEqual(b.execution_status('e')['generations'][0]['state'],'COMPLETE')
|
|
|
|
def test_independent_snapshot_is_readonly_bounded_and_missing_is_unknown(self):
|
|
absent=Path(self.tmp.name)/'missing'/'ledger'
|
|
self.assertEqual(quota_snapshot(absent)['status'],'UNKNOWN')
|
|
self.assertFalse(absent.parent.exists())
|
|
p=self.provider();b=self.broker(p)
|
|
for i in range(3): self.request(b.issue_scope(str(i),'teacher','glm-test'))
|
|
before=hashlib.sha256(self.ledger.read_bytes()).hexdigest()
|
|
with patch.object(Broker,'__init__',side_effect=AssertionError('must not construct')):
|
|
snap=quota_snapshot(self.ledger,limit=2)
|
|
self.assertEqual(snap['status'],'OBSERVED')
|
|
self.assertLessEqual(len(snap['attempts']),2);self.assertLessEqual(len(snap['events']),2)
|
|
self.assertNotIn('upstream-secret',json.dumps(snap));self.assertNotIn('token_hash',json.dumps(snap))
|
|
self.assertEqual(hashlib.sha256(self.ledger.read_bytes()).hexdigest(),before)
|
|
self.assertEqual(snap['groups'][0]['inflight'],0)
|
|
|
|
def test_confirmed_connect_failure_does_not_spend_http_budget(self):
|
|
p=self.provider();b=self.broker(p,self.policy(rpm=1,window_s=1))
|
|
scope=b.issue_scope('connect-failure','teacher','glm-test')
|
|
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
|
|
def refused(connection):
|
|
if connection.port==p.server.server_port:
|
|
raise ConnectionRefusedError('injected TCP refusal before HTTP')
|
|
return original(connection)
|
|
with patch('harness.quota.http.client.HTTPConnection.connect',new=refused):
|
|
self.assertEqual(self.request(scope)[0],503)
|
|
generation=b.execution_status('connect-failure')['generations'][0]
|
|
self.assertEqual(generation['attempts'],0)
|
|
self.assertEqual(generation['reason'],'CONNECT_FAILED')
|
|
self.assertEqual(generation['state'],'PAUSED')
|
|
self.assertEqual(p.requests,[])
|
|
snapshot=quota_snapshot(self.ledger)
|
|
self.assertEqual(snapshot['groups'][0]['rpm_used'],0)
|
|
self.assertEqual(snapshot['groups'][0]['tpm_reserved'],0)
|
|
self.assertEqual(snapshot['groups'][0]['inflight'],0)
|
|
self.assertTrue(all(row['sent']==0 and row['state']=='NOT_SENT' for row in snapshot['attempts']))
|
|
self.assertIn('TRANSPORT_NOT_SENT',[event['event'] for event in snapshot['events']])
|
|
# The same account can serve a later request; no phantom RPM reservation.
|
|
with b._db() as con:
|
|
cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0]
|
|
with patch.object(b,'_now',return_value=max(b._now(),cooldown)):
|
|
self.assertEqual(self.request(b.issue_scope('next','teacher','glm-test'))[0],200)
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_unknown_send_never_refunds_http_budget(self):
|
|
p=self.provider();b=self.broker(p,self.policy(window_s=1))
|
|
scope=b.issue_scope('unknown-send','teacher','glm-test')
|
|
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.request
|
|
def interrupted(connection,*args,**kwargs):
|
|
original(connection,*args,**kwargs)
|
|
if connection.port==p.server.server_port:
|
|
raise OSError('injected interruption after request bytes')
|
|
with patch('harness.quota.http.client.HTTPConnection.request',new=interrupted):
|
|
self.assertEqual(self.request(scope)[0],503)
|
|
generation=b.execution_status('unknown-send')['generations'][0]
|
|
self.assertEqual(generation['attempts'],1)
|
|
snapshot=quota_snapshot(self.ledger)
|
|
self.assertEqual(snapshot['groups'][0]['rpm_used'],1)
|
|
self.assertGreater(snapshot['groups'][0]['tpm_reserved'],0)
|
|
self.assertEqual(snapshot['groups'][0]['inflight'],1)
|
|
self.assertEqual(snapshot['attempts'][0]['state'],'UNKNOWN')
|
|
self.assertIsNone(snapshot['attempts'][0]['ended_at'])
|
|
self.assertNotIn('TRANSPORT_NOT_SENT',[event['event'] for event in snapshot['events']])
|
|
|
|
def test_invalid_budget_rejected(self):
|
|
with self.assertRaises(ValueError): self.policy(rpm=None)
|
|
def test_model_scope_cannot_change(self):
|
|
p=self.provider(); b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
|
|
self.assertEqual(self.request(s,body={'model':'other','messages':[],'max_tokens':2})[0],403)
|
|
self.assertEqual(p.requests,[])
|
|
|
|
|
|
if __name__=='__main__': unittest.main()
|