Recorded FAILED CONTEXT_TOO_LONG like a provider overflow and answered with the overflow body, so Pi compacts and continues instead of ending the rollout on an unrecognised 413. Content review keeps its REQUEST_EXCEEDS_INTERNAL_TPM. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
743 lines
45 KiB
Python
743 lines
45 KiB
Python
"""Quota acceptance uses real local HTTP, persistent ledgers and no provider account."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
import json
|
|
import multiprocessing
|
|
import socket
|
|
import hashlib
|
|
from pathlib import Path
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
from harness.quota import Broker, CONTEXT_OVERFLOW_BODY, QuotaPolicy, Upstream, QuotaError, quota_snapshot, validate_completion
|
|
|
|
|
|
class Provider:
|
|
def __init__(self, responder):
|
|
self.responder, self.requests = responder, []
|
|
outer = self
|
|
class Handler(BaseHTTPRequestHandler):
|
|
def log_message(self, *_): pass
|
|
def do_POST(self):
|
|
body = json.loads(self.rfile.read(int(self.headers['Content-Length'])))
|
|
outer.requests.append({'at': time.time(), 'body': body,
|
|
'authorization': self.headers.get('Authorization')})
|
|
code, headers, payload = outer.responder(len(outer.requests), body)
|
|
self.send_response(code)
|
|
for key, value in headers.items(): self.send_header(key, str(value))
|
|
self.end_headers()
|
|
try:
|
|
if callable(payload): payload(self.wfile)
|
|
else: self.wfile.write(payload)
|
|
except (BrokenPipeError, ConnectionResetError): pass
|
|
self.server = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
|
|
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
|
self.thread.start()
|
|
self.url = 'http://127.0.0.1:%s/v1' % self.server.server_port
|
|
def close(self):
|
|
self.server.shutdown(); self.server.server_close(); self.thread.join()
|
|
|
|
|
|
def ok(_n=0, body=None):
|
|
if body and body.get('stream'):
|
|
return 200, {'Content-Type': 'text/event-stream'}, (b'data: {"choices":[{"delta":{"content":"ok"},"finish_reason":null}]}\n\n'
|
|
b'data: {"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}\n\n'
|
|
b'data: [DONE]\n\n')
|
|
return 200, {'Content-Type': 'application/json'}, b'{"choices":[{"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}}'
|
|
|
|
|
|
def _process_broker(ledger, url, policy, execution, ready, start, result):
|
|
# Spawned interpreters do not inherit the parent's locks or server objects.
|
|
with Broker(Path(ledger), {'shared':policy}, {'teacher':Upstream('shared',url,'upstream-secret',('glm-test',))}) as broker:
|
|
scope=broker.issue_scope(execution,'teacher','glm-test')
|
|
ready.put(True)
|
|
if not start.wait(5):
|
|
result.put('START_TIMEOUT');return
|
|
request=urllib.request.Request(scope.base_url+'/chat/completions',
|
|
json.dumps({'model':'glm-test','messages':[],'max_tokens':5}).encode(),
|
|
{'Authorization':'Bearer '+scope.api_key,'Content-Type':'application/json'})
|
|
try:
|
|
with urllib.request.urlopen(request,timeout=5) as response:
|
|
response.read();result.put(response.status)
|
|
except urllib.error.HTTPError as error:
|
|
error.read();result.put(error.code)
|
|
|
|
|
|
class Quota(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
|
|
self.ledger = Path(self.tmp.name) / 'shared.sqlite'
|
|
def provider(self, responder=ok):
|
|
item = Provider(responder); self.addCleanup(item.close); return item
|
|
def policy(self, **kw):
|
|
return QuotaPolicy(**dict({'max_concurrency':2, 'rpm':1000, 'tpm':1000000,
|
|
'window_s':.2, 'cooldown_initial_s':.06, 'cooldown_local_max_s':.1,
|
|
'jitter_fraction':0, 'recovery_window_s':.12, 'recovery_min_successes':2,
|
|
'wait_budget_s':2, 'connect_timeout_s':.2, 'first_byte_timeout_s':.3,
|
|
'stream_idle_timeout_s':.15, 'request_total_s':.6, 'poll_s':.01}, **kw))
|
|
def broker(self, provider, policy=None, aliases=None):
|
|
b = Broker(self.ledger, {'shared':policy or self.policy()}, aliases or {
|
|
'teacher':Upstream('shared', provider.url, 'upstream-secret', ('glm-test',))})
|
|
b.start(); self.addCleanup(b.close); return b
|
|
def request(self, scope, generation=None, body=None):
|
|
headers = {'Authorization':'Bearer '+scope.api_key, 'Content-Type':'application/json'}
|
|
if generation: headers['X-Bench-Generation-Id'] = generation
|
|
request = urllib.request.Request(scope.base_url+'/chat/completions',
|
|
json.dumps(body or {'model':'glm-test', 'messages':[{'role':'user','content':'hi'}], 'max_tokens':5}).encode(), headers)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=5) as response:
|
|
return response.status, response.read()
|
|
except urllib.error.HTTPError as error:
|
|
return error.code, error.read()
|
|
def test_success_scope_and_stream_usage(self):
|
|
p=self.provider(); b=self.broker(p); scope=b.issue_scope('execution', 'teacher', 'glm-test')
|
|
code, data=self.request(scope, 'generation', {'model':'glm-test','messages':[],'max_tokens':4,'stream':True})
|
|
self.assertEqual(code, 200); self.assertIn(b'[DONE]', data)
|
|
self.assertEqual(p.requests[0]['authorization'], 'Bearer upstream-secret')
|
|
self.assertNotIn('upstream-secret', scope.api_key)
|
|
self.assertEqual(b.snapshot()['attempts'][0]['charged_tokens'], 3)
|
|
self.assertEqual(len(p.requests), 1)
|
|
self.assertEqual(self.request(scope, 'generation')[0], 409)
|
|
|
|
def _raw_request(self, scope):
|
|
from urllib.parse import urlsplit
|
|
url = urlsplit(scope.base_url)
|
|
sock = socket.create_connection((url.hostname, url.port), timeout=2)
|
|
body = json.dumps({'model':'glm-test', 'messages':[], 'max_tokens':5}).encode()
|
|
sock.sendall((f'POST /v1/chat/completions HTTP/1.1\r\nHost: localhost\r\n'
|
|
f'Authorization: Bearer {scope.api_key}\r\nContent-Length: {len(body)}\r\n'
|
|
'Content-Type: application/json\r\n\r\n').encode() + body)
|
|
return sock
|
|
|
|
def test_disconnected_queued_client_never_reaches_upstream(self):
|
|
p = self.provider(); b = self.broker(p, self.policy(rpm=1, window_s=.4))
|
|
self.assertEqual(self.request(b.issue_scope('first', 'teacher', 'glm-test'))[0], 200)
|
|
scope = b.issue_scope('disconnected', 'teacher', 'glm-test')
|
|
sock = self._raw_request(scope)
|
|
deadline = time.monotonic() + 1
|
|
while not b.execution_status('disconnected')['generations'] and time.monotonic() < deadline:
|
|
time.sleep(.005)
|
|
self.assertTrue(b.execution_status('disconnected')['generations'])
|
|
sock.close()
|
|
deadline = time.monotonic() + 1
|
|
while not b.execution_status('disconnected')['cancelled'] and time.monotonic() < deadline:
|
|
time.sleep(.005)
|
|
self.assertTrue(b.execution_status('disconnected')['cancelled'])
|
|
time.sleep(.45) # RPM capacity becomes free after the client has gone.
|
|
self.assertEqual(len(p.requests), 1)
|
|
gen = b.execution_status('disconnected')['generations'][0]
|
|
self.assertEqual((gen['state'], gen['attempts']), ('CANCELLED', 0))
|
|
self.assertEqual(b.snapshot()['groups'][0]['state'], 'OPEN')
|
|
|
|
def test_disconnected_sent_client_keeps_unknown_reservation(self):
|
|
release = threading.Event()
|
|
self.addCleanup(release.set)
|
|
p = self.provider(lambda *_: (release.wait(2) or True) and ok())
|
|
b = self.broker(p)
|
|
sock = self._raw_request(b.issue_scope('disconnected', 'teacher', 'glm-test'))
|
|
deadline = time.monotonic() + 1
|
|
while not p.requests and time.monotonic() < deadline:
|
|
time.sleep(.005)
|
|
self.assertEqual(len(p.requests), 1)
|
|
sock.close()
|
|
deadline = time.monotonic() + 1
|
|
while b.snapshot()['attempts'][0]['state'] != 'UNKNOWN' and time.monotonic() < deadline:
|
|
time.sleep(.005)
|
|
attempt = b.snapshot()['attempts'][0]
|
|
self.assertEqual(attempt['state'], 'UNKNOWN')
|
|
self.assertIsNone(attempt['ended_at'])
|
|
self.assertGreater(attempt['charged_tokens'], 0)
|
|
self.assertTrue(b.execution_status('disconnected')['cancelled'])
|
|
self.assertEqual(len(p.requests), 1)
|
|
release.set()
|
|
|
|
def test_valid_sse_done_does_not_wait_for_http_close(self):
|
|
for crlf in (False, True):
|
|
with self.subTest(crlf=crlf), tempfile.TemporaryDirectory() as temp:
|
|
payload = ok(body={'stream':True})[2]
|
|
if crlf:
|
|
payload = payload.replace(b'\n', b'\r\n')
|
|
release = threading.Event()
|
|
def send(writer):
|
|
# Split inside the terminator to exercise real read boundaries.
|
|
cut = payload.index(b'[DONE]') + 3
|
|
writer.write(payload[:cut]); writer.flush()
|
|
time.sleep(.02)
|
|
writer.write(payload[cut:]); writer.flush()
|
|
release.wait(2)
|
|
provider = self.provider(lambda *_: (200, {'Content-Type':'text/event-stream'}, send))
|
|
with Broker(Path(temp)/'quota.sqlite', {'shared':self.policy()}, {
|
|
'teacher':Upstream('shared',provider.url,'fake-only',('glm-test',))}) as broker:
|
|
try:
|
|
scope = broker.issue_scope('execution','teacher','glm-test')
|
|
code, data = self.request(scope,'generation',{'model':'glm-test','messages':[], 'max_tokens':4,'stream':True})
|
|
self.assertEqual((code,data),(200,payload))
|
|
attempt = broker.snapshot()['attempts'][0]
|
|
self.assertEqual((attempt['state'],attempt['reason'],attempt['charged_tokens']),('FINISHED','COMPLETE',3))
|
|
self.assertEqual(broker.snapshot()['groups'][0]['state'],'OPEN')
|
|
self.assertEqual(len(provider.requests),1)
|
|
finally:
|
|
release.set()
|
|
|
|
def test_sse_terminator_cannot_complete_invalid_or_unframed_response(self):
|
|
complete = ok(body={'stream':True})[2]
|
|
variants = {
|
|
'missing_blank_line': complete[:-2],
|
|
'content_literal_only': b'data: {"choices":[{"delta":{"content":"[DONE]"},"finish_reason":"stop"}]}\n\n',
|
|
'missing_finish': complete.split(b'data: {"choices":[{"delta":{}')[0] + b'data: [DONE]\n\n',
|
|
'partial_tool': b'data: {"choices":[{"delta":{"tool_calls":[{"id":"x","type":"function","function":{"name":"write","arguments":"{"}}]},"finish_reason":"tool_calls"}]}\n\ndata: [DONE]\n\n',
|
|
'extra_data_after_done': complete + b'data: {"choices":[]}\n\n',
|
|
}
|
|
for name,payload in variants.items():
|
|
with self.subTest(name=name), tempfile.TemporaryDirectory() as temp:
|
|
release = threading.Event()
|
|
def send(writer):
|
|
writer.write(payload); writer.flush(); release.wait(2)
|
|
provider = self.provider(lambda *_: (200, {'Content-Type':'text/event-stream'},send))
|
|
with Broker(Path(temp)/'quota.sqlite',{'shared':self.policy()}, {
|
|
'teacher':Upstream('shared',provider.url,'fake-only',('glm-test',))}) as broker:
|
|
try:
|
|
scope = broker.issue_scope('execution','teacher','glm-test')
|
|
code,_ = self.request(scope,'generation',{'model':'glm-test','messages':[],'max_tokens':4,'stream':True})
|
|
self.assertNotEqual(code,200)
|
|
attempt = broker.snapshot()['attempts'][0]
|
|
self.assertEqual((attempt['state'],attempt['transport_reason']),('UNKNOWN','STREAM_IDLE_TIMEOUT'))
|
|
self.assertEqual(len(provider.requests),1)
|
|
finally:
|
|
release.set()
|
|
def test_JOINT001_persistent_429_budget_and_single_probe(self):
|
|
p=self.provider(lambda n, body:(429, {'Retry-After':'.06'}, b'{"error":{"code":"rate_limit"}}'))
|
|
b=self.broker(p, self.policy(max_attempts=4))
|
|
scopes=[b.issue_scope('e'+str(i),'teacher','glm-test') for i in range(4)]
|
|
with ThreadPoolExecutor(max_workers=4) as pool:
|
|
results=list(pool.map(lambda s:self.request(s),scopes))
|
|
snapshot=b.snapshot()
|
|
self.assertTrue(all(code==503 for code,_ in results))
|
|
self.assertTrue(all(g['attempts']<=4 for g in snapshot['generations']))
|
|
self.assertTrue(all(t['state_at_send'] in ('OPEN','HALF_OPEN','RECOVERING') for t in snapshot['attempts']))
|
|
self.assertEqual(len(p.requests), len(snapshot['attempts']))
|
|
probes=[a for a in snapshot['attempts'] if a['state_at_send']=='HALF_OPEN']
|
|
for before,after in zip(probes,probes[1:]):
|
|
self.assertGreaterEqual(after['started_at'],before['ended_at'])
|
|
self.assertEqual(len({a['attempt_id'] for a in snapshot['attempts']}),len(snapshot['attempts']))
|
|
def test_QUOTA001_shared_alias_group_independent_group(self):
|
|
p=self.provider(lambda n, body:(429, {'Retry-After':'1'}, b'{}') if body['model']=='limited' else ok(n,body))
|
|
b=Broker(self.ledger, {'shared':self.policy(wait_budget_s=.12), 'independent':self.policy()}, {
|
|
'a':Upstream('shared',p.url,'a',('limited',)), 'b':Upstream('shared',p.url,'b',('limited',)),
|
|
'other':Upstream('independent',p.url,'c',('free',))})
|
|
b.start(); self.addCleanup(b.close)
|
|
self.request(b.issue_scope('a','a','limited'),body={'model':'limited','messages':[],'max_tokens':1})
|
|
self.request(b.issue_scope('b','b','limited'),body={'model':'limited','messages':[],'max_tokens':1})
|
|
self.assertEqual(len(p.requests),1)
|
|
self.assertEqual(self.request(b.issue_scope('c','other','free'),body={'model':'free','messages':[],'max_tokens':1})[0],200)
|
|
def test_JOINT003_restart_keeps_budget_and_missing_usage(self):
|
|
p=self.provider(lambda n,body:(200,{'Content-Type':'application/json'},b'{"choices":[{"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}]}'))
|
|
b=self.broker(p,self.policy(rpm=1,window_s=1,wait_budget_s=.06))
|
|
self.assertEqual(self.request(b.issue_scope('e1','teacher','glm-test'))[0],200)
|
|
first=b.snapshot()['attempts'][0]
|
|
self.assertEqual(first['charged_tokens'],first['reserved_tokens'])
|
|
b.close()
|
|
b2=self.broker(p,self.policy(rpm=1,window_s=1,wait_budget_s=.06))
|
|
self.assertEqual(self.request(b2.issue_scope('e2','teacher','glm-test'))[0],503)
|
|
self.assertEqual(len(p.requests),1)
|
|
def test_cancel_blocks_transport_without_hitting_upstream(self):
|
|
p=self.provider(); b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
|
|
b.cancel('e'); self.assertEqual(self.request(s)[0],409); self.assertEqual(p.requests,[])
|
|
def test_QUOTA002_retry_after_and_auth_pause(self):
|
|
p=self.provider(lambda n,body:(429,{'Retry-After':'4'},b'{}'))
|
|
b=self.broker(p,self.policy(wait_budget_s=.1)); before=time.time()
|
|
self.request(b.issue_scope('e','teacher','glm-test'))
|
|
self.assertGreaterEqual(b.snapshot()['groups'][0]['cooldown_until'],before+4)
|
|
self.assertEqual(len(p.requests),1)
|
|
def test_JOINT004_nonempty_stream_interruption_does_not_retry(self):
|
|
p=self.provider(lambda n,body:(200,{'Content-Type':'text/event-stream'}, b'data: {"choices":[{"delta":{"content":"partial"}}]}\n\n'))
|
|
b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
|
|
self.assertEqual(self.request(s,body={'model':'glm-test','messages':[],'max_tokens':5,'stream':True})[0],503)
|
|
self.assertEqual(len(p.requests),1)
|
|
self.assertEqual(b.snapshot()['generations'][0]['reason'],'INCOMPLETE_GENERATION')
|
|
self.assertEqual(self.request(s,'new-id')[0],409)
|
|
def test_JOINT002_recovery_adds_at_most_one_per_window(self):
|
|
p=self.provider(lambda n,body:(429,{},b'{}') if n==1 else ok(n,body))
|
|
policy=self.policy(max_concurrency=4,recovery_window_s=.12)
|
|
b=self.broker(p,policy)
|
|
self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'))[0],200)
|
|
self.assertEqual(b.snapshot()['groups'][0]['state'],'RECOVERING')
|
|
self.assertEqual(b.snapshot()['groups'][0]['target'],1)
|
|
clock=[b.snapshot()['groups'][0]['recovery_at']]
|
|
# Only quota's durable clock is controlled. HTTP itself remains real.
|
|
with patch.object(b,'_now',side_effect=lambda:clock[0]):
|
|
for i in range(3):
|
|
self.assertEqual(self.request(b.issue_scope('e'+str(i),'teacher','glm-test'))[0],200)
|
|
self.assertEqual(b.snapshot()['groups'][0]['target'],1)
|
|
clock[0]+=.13
|
|
self.request(b.issue_scope('next','teacher','glm-test'))
|
|
self.assertEqual(b.snapshot()['groups'][0]['target'],1)
|
|
self.request(b.issue_scope('next2','teacher','glm-test'))
|
|
self.assertEqual(b.snapshot()['groups'][0]['target'],2)
|
|
self.request(b.issue_scope('after','teacher','glm-test'))
|
|
self.assertEqual(b.snapshot()['groups'][0]['target'],2)
|
|
|
|
def test_shared_brokers_enforce_actual_RPM(self):
|
|
p=self.provider(); policy=self.policy(rpm=1,window_s=.7,wait_budget_s=.12)
|
|
a=self.broker(p,policy); b=self.broker(p,policy)
|
|
scopes=[a.issue_scope('a','teacher','glm-test'),b.issue_scope('b','teacher','glm-test')]
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
results=list(pool.map(self.request,scopes))
|
|
self.assertEqual(sorted(code for code,_ in results),[200,503])
|
|
self.assertEqual(len(p.requests),1)
|
|
self.assertEqual(len(a.snapshot()['attempts']),1)
|
|
|
|
def test_missing_usage_reservation_blocks_TPM_until_end_window(self):
|
|
p=self.provider(lambda n,body:(200,{},b'{"choices":[{"message":{"content":"ok"},"finish_reason":"stop"}]}'))
|
|
b=self.broker(p,self.policy(tpm=300,window_s=.7,wait_budget_s=.08))
|
|
self.assertEqual(self.request(b.issue_scope('a','teacher','glm-test'))[0],200)
|
|
self.assertEqual(self.request(b.issue_scope('b','teacher','glm-test'))[0],503)
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_QUOTA003_attempts_never_reset_and_each_resume_gets_a_fresh_wait(self):
|
|
p=self.provider(lambda n,body:(429,{'Retry-After':'3'},b'{}'))
|
|
b=self.broker(p,self.policy(wait_budget_s=.07))
|
|
scope=b.issue_scope('e','teacher','glm-test')
|
|
self.assertEqual(self.request(scope,'stable')[0],503)
|
|
spent=b.execution_status('e')['wait_spent']
|
|
self.assertFalse(b.resume_readiness('e')['eligible'])
|
|
with self.assertRaises(QuotaError):
|
|
b.resume('e','stable',checkpoint_complete=True,side_effects_known=True)
|
|
self.assertEqual(b.execution_status('e')['resumes'],0)
|
|
clock=[b._now()]
|
|
def tick(): # time moves while the broker waits, so a fresh allowance runs out
|
|
clock[0]+=.01; return clock[0]
|
|
with patch.object(b,'_now',side_effect=tick):
|
|
for i in range(3):
|
|
clock[0]+=4
|
|
self.assertTrue(b.resume_readiness('e')['eligible'])
|
|
b.resume('e','stable',checkpoint_complete=True,side_effects_known=True)
|
|
# A lifetime wait total paused at the first busy moment after every resume, forever (10-19).
|
|
self.assertEqual(b.execution_status('e')['wait_spent'],0)
|
|
self.assertEqual(self.request(scope,'stable')[0],503)
|
|
self.assertGreater(spent,0)
|
|
self.assertEqual(len(p.requests),4)
|
|
self.assertEqual(b.execution_status('e')['resumes'],3)
|
|
with self.assertRaises(QuotaError):
|
|
b.resume('e','stable',checkpoint_complete=True,side_effects_known=True)
|
|
self.assertEqual(self.request(scope,'replacement')[0],409)
|
|
|
|
def test_first_byte_timeout_quarantines_unknown_request(self):
|
|
def delay(n,body):
|
|
time.sleep(.2);return ok(n,body)
|
|
p=self.provider(delay); b=self.broker(p,self.policy(first_byte_timeout_s=.04))
|
|
s=b.issue_scope('e','teacher','glm-test')
|
|
self.assertEqual(self.request(s)[0],503)
|
|
snap=b.snapshot(); self.assertEqual(len(p.requests),1)
|
|
self.assertEqual(snap['attempts'][0]['transport_reason'],'FIRST_BYTE_TIMEOUT')
|
|
self.assertIsNone(snap['attempts'][0]['ended_at'])
|
|
self.assertEqual(snap['groups'][0]['state'],'RECONCILING')
|
|
self.assertEqual(self.request(b.issue_scope('e2','teacher','glm-test'))[0],503)
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_stream_idle_and_busy_total_timeout_are_bounded(self):
|
|
for mode in ('idle','busy'):
|
|
with self.subTest(mode=mode):
|
|
# Separate ledger avoids deliberately quarantined prior group.
|
|
self.ledger=Path(self.tmp.name)/(mode+'.sqlite')
|
|
def payload(out):
|
|
for i in range(30):
|
|
out.write(b'data: {"choices":[{"delta":{"content":"x"}}]}\n\n');out.flush()
|
|
time.sleep(.2 if mode=='idle' else .01)
|
|
p=self.provider(lambda n,body:(200,{'Content-Type':'text/event-stream'},payload))
|
|
b=self.broker(p,self.policy(stream_idle_timeout_s=.05,request_total_s=.12))
|
|
started=time.monotonic()
|
|
self.assertEqual(self.request(b.issue_scope(mode,'teacher','glm-test'),body={'model':'glm-test','messages':[],'stream':True,'max_tokens':4})[0],503)
|
|
self.assertLess(time.monotonic()-started,.3)
|
|
attempt=b.snapshot()['attempts'][0]
|
|
self.assertEqual(attempt['reason'],'INCOMPLETE_GENERATION')
|
|
self.assertEqual(attempt['transport_reason'],'STREAM_IDLE_TIMEOUT' if mode=='idle' else 'REQUEST_TOTAL_TIMEOUT')
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_cancel_queued_execution_fences_later_send(self):
|
|
p=self.provider(lambda n,body:(429,{'Retry-After':'1'},b'{}'))
|
|
b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
pending=pool.submit(self.request,s)
|
|
deadline=time.monotonic()+1
|
|
while not p.requests and time.monotonic()<deadline: time.sleep(.005)
|
|
b.cancel('e')
|
|
self.assertEqual(pending.result()[0],409)
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_5xx_retries_and_auth_pauses_without_provider_body_leak(self):
|
|
p=self.provider(lambda n,body:(503,{},b'secret provider failure') if n==1 else ok(n,body))
|
|
b=self.broker(p); self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'))[0],200)
|
|
self.assertEqual(len(p.requests),2)
|
|
self.assertNotIn('upstream-secret',json.dumps(b.snapshot()))
|
|
self.assertNotIn('secret provider failure',json.dumps(b.snapshot()))
|
|
|
|
def test_concurrency_is_kept_until_response_end(self):
|
|
active=peak=0
|
|
lock=threading.Lock()
|
|
def responder(n,body):
|
|
nonlocal active,peak
|
|
with lock:
|
|
active+=1;peak=max(peak,active)
|
|
time.sleep(.05)
|
|
with lock: active-=1
|
|
return ok(n,body)
|
|
p=self.provider(responder); b=self.broker(p,self.policy(max_concurrency=1))
|
|
scopes=[b.issue_scope(str(i),'teacher','glm-test') for i in range(4)]
|
|
with ThreadPoolExecutor(max_workers=4) as pool:
|
|
results=list(pool.map(self.request,scopes))
|
|
self.assertTrue(all(code==200 for code,_ in results))
|
|
self.assertEqual(peak,1)
|
|
self.assertEqual(len(p.requests),4)
|
|
|
|
def test_cancel_active_request_interrupts_without_refund(self):
|
|
def delay(n,body):
|
|
time.sleep(.4);return ok(n,body)
|
|
p=self.provider(delay); b=self.broker(p); scope=b.issue_scope('e','teacher','glm-test')
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
future=pool.submit(self.request,scope)
|
|
until=time.monotonic()+1
|
|
while not p.requests and time.monotonic()<until: time.sleep(.005)
|
|
b.cancel('e')
|
|
self.assertEqual(future.result()[0],503)
|
|
self.assertEqual(len(p.requests),1)
|
|
self.assertIsNone(b.snapshot()['attempts'][0]['ended_at'])
|
|
self.assertEqual(b.execution_status('e')['generations'][0]['reason'],'CANCELLED')
|
|
|
|
def test_auth_failure_pauses_group_and_no_new_id_bypasses_it(self):
|
|
p=self.provider(lambda n,body:(401,{},b'upstream-private-diagnostic'))
|
|
b=self.broker(p); scope=b.issue_scope('e','teacher','glm-test')
|
|
code,body=self.request(scope)
|
|
self.assertEqual(code,503);self.assertNotIn(b'upstream-private',body)
|
|
self.assertEqual(b.snapshot()['groups'][0]['state'],'PAUSED')
|
|
self.assertEqual(self.request(b.issue_scope('another','teacher','glm-test'))[0],503)
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_independent_processes_share_RPM_ledger(self):
|
|
p=self.provider(); policy=self.policy(rpm=1,window_s=1,wait_budget_s=.12)
|
|
# Initialize the immutable group policy before simultaneous processes.
|
|
Broker(self.ledger,{'shared':policy},{'teacher':Upstream('shared',p.url,'upstream-secret',('glm-test',))})
|
|
context=multiprocessing.get_context('spawn')
|
|
ready,start,result=context.Queue(),context.Event(),context.Queue()
|
|
processes=[context.Process(target=_process_broker,args=(str(self.ledger),p.url,policy,str(i),ready,start,result)) for i in range(2)]
|
|
try:
|
|
for process in processes: process.start()
|
|
for _ in processes: self.assertTrue(ready.get(timeout=8))
|
|
start.set()
|
|
self.assertEqual(sorted(result.get(timeout=8) for _ in processes),[200,503])
|
|
for process in processes:
|
|
process.join(timeout=5);self.assertEqual(process.exitcode,0)
|
|
self.assertEqual(len(p.requests),1)
|
|
finally:
|
|
for process in processes:
|
|
if process.is_alive(): process.terminate();process.join(timeout=2)
|
|
for queue in (ready,result): queue.close();queue.join_thread()
|
|
|
|
def test_429_headers_fence_sends_before_error_body_finishes(self):
|
|
def stalled_body(out):
|
|
time.sleep(.2);out.write(b'{}')
|
|
p=self.provider(lambda n,body:(429,{'Retry-After':'2'},stalled_body))
|
|
b=self.broker(p,self.policy(wait_budget_s=.06,stream_idle_timeout_s=.3))
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
first=pool.submit(self.request,b.issue_scope('first','teacher','glm-test'))
|
|
deadline=time.monotonic()+1
|
|
while b.snapshot()['groups'][0]['state']!='COOLING' and time.monotonic()<deadline:
|
|
time.sleep(.005)
|
|
self.assertEqual(b.snapshot()['groups'][0]['state'],'COOLING')
|
|
self.assertEqual(self.request(b.issue_scope('second','teacher','glm-test'))[0],503)
|
|
self.assertEqual(first.result()[0],503)
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_wall_clock_rollback_cannot_extend_queue_budget(self):
|
|
p=self.provider(lambda n,body:(429,{'Retry-After':'10'},b'{}'))
|
|
b=self.broker(p,self.policy(wait_budget_s=.05))
|
|
scope=b.issue_scope('e','teacher','glm-test')
|
|
before=time.monotonic(); wall=time.time()
|
|
with patch('harness.quota.time.time',return_value=wall-3600):
|
|
self.assertEqual(self.request(scope)[0],503)
|
|
self.assertLess(time.monotonic()-before,.5)
|
|
self.assertGreaterEqual(b.execution_status('e')['wait_spent'],.05)
|
|
|
|
def test_dns_deadline_discards_late_resolution_without_sending(self):
|
|
p=self.provider();b=self.broker(p,self.policy(connect_timeout_s=.03,request_total_s=.1))
|
|
scope=b.issue_scope('dns','teacher','glm-test')
|
|
original=socket.getaddrinfo;release=threading.Event();entered=threading.Event()
|
|
def blocked(host,port,*args,**kwargs):
|
|
if port==p.server.server_port:
|
|
entered.set();release.wait(2)
|
|
return original(host,port,*args,**kwargs)
|
|
try:
|
|
with patch('harness.quota.socket.getaddrinfo',side_effect=blocked):
|
|
code,_=self.request(scope)
|
|
self.assertTrue(entered.is_set());self.assertEqual(code,503)
|
|
self.assertFalse(release.is_set())
|
|
self.assertEqual(b.execution_status('dns')['generations'][0]['reason'],'DNS_TIMEOUT')
|
|
self.assertEqual(p.requests,[])
|
|
finally:
|
|
release.set()
|
|
time.sleep(.06)
|
|
self.assertEqual(p.requests,[]) # Resolver has no transport capability.
|
|
|
|
def test_cancel_while_dns_blocked_does_not_wait_for_group_lock(self):
|
|
p=self.provider();b=self.broker(p,self.policy(connect_timeout_s=1))
|
|
scope=b.issue_scope('dns-cancel','teacher','glm-test')
|
|
original=socket.getaddrinfo;release=threading.Event();entered=threading.Event()
|
|
def blocked(host,port,*args,**kwargs):
|
|
if port==p.server.server_port:
|
|
entered.set();release.wait(2)
|
|
return original(host,port,*args,**kwargs)
|
|
try:
|
|
with patch('harness.quota.socket.getaddrinfo',side_effect=blocked),ThreadPoolExecutor(max_workers=1) as pool:
|
|
pending=pool.submit(self.request,scope)
|
|
self.assertTrue(entered.wait(1))
|
|
b.cancel('dns-cancel')
|
|
self.assertEqual(pending.result(timeout=.5)[0],409)
|
|
self.assertFalse(release.is_set());self.assertEqual(p.requests,[])
|
|
finally:
|
|
release.set()
|
|
|
|
def test_confirmed_end_allows_new_probe_but_never_old_generation_replay(self):
|
|
finished=threading.Event()
|
|
def slow_body(out):
|
|
time.sleep(.1);out.write(ok()[2]);out.flush();finished.set()
|
|
p=self.provider(lambda n,body:(200,{'Content-Type':'application/json'},slow_body) if n==1 else ok(n,body))
|
|
b=self.broker(p,self.policy(stream_idle_timeout_s=.03))
|
|
original=b.issue_scope('original','teacher','glm-test')
|
|
self.assertEqual(self.request(original,'original-g')[0],503)
|
|
self.assertTrue(finished.wait(1))
|
|
attempt=b.snapshot()['attempts'][0]
|
|
b.confirm_end(attempt['attempt_id'],'verified-completion-receipt')
|
|
self.assertEqual(b.snapshot()['groups'][0]['state'],'COOLING')
|
|
self.assertEqual(self.request(b.issue_scope('new','teacher','glm-test'))[0],200)
|
|
self.assertEqual(b.snapshot()['attempts'][-1]['state_at_send'],'HALF_OPEN')
|
|
self.assertEqual(b.execution_status('original')['generations'][0]['state'],'PAUSED')
|
|
self.assertEqual(self.request(original,'new-g')[0],409)
|
|
self.assertEqual(len(p.requests),2)
|
|
|
|
def test_stalled_429_body_retains_received_rate_limit_headers(self):
|
|
def slow(out): time.sleep(.1);out.write(b'{}')
|
|
p=self.provider(lambda n,body:(429,{'Retry-After':'8','X-Request-Id':'trace-123'},slow))
|
|
b=self.broker(p,self.policy(stream_idle_timeout_s=.03))
|
|
self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'))[0],503)
|
|
attempt=b.snapshot()['attempts'][0]
|
|
self.assertEqual(attempt['http_status'],429)
|
|
self.assertEqual(json.loads(attempt['response_headers']),{'retry-after':'8','x-request-id':'trace-123'})
|
|
|
|
def test_stream_integrity_validates_every_choice_and_tool_arguments(self):
|
|
def stream(*choices):
|
|
return b''.join(b'data: '+json.dumps({'choices':chunk}).encode()+b'\n\n' for chunk in choices)+b'data: [DONE]\n\n'
|
|
complete={'index':0,'delta':{'role':'assistant','content':'ok'},'finish_reason':'stop'}
|
|
after={'index':0,'delta':{'tool_calls':[{'index':0,'id':'c','function':{'name':'bash','arguments':'{broken'}}]}}
|
|
fixtures=[
|
|
(stream([complete],[after]),'INVALID_PROVIDER_RESPONSE'),
|
|
(stream([complete,{'index':1,'delta':{'content':'unfinished'}}]),'INCOMPLETE_GENERATION'),
|
|
(stream([{'index':0,'delta':{},'finish_reason':'stop'}]),'INVALID_PROVIDER_RESPONSE'),
|
|
(stream([dict(after,finish_reason='tool_calls')]),'INVALID_PROVIDER_RESPONSE'),
|
|
(stream([complete]),'COMPLETE'),
|
|
(stream([{'index':0,'delta':{'tool_calls':[{'index':0,'id':'c','type':'function','function':{'name':'bash','arguments':'{"command":'}}]}}],
|
|
[{'index':0,'delta':{'tool_calls':[{'index':0,'function':{'arguments':'"echo ok"}'}}]},'finish_reason':'tool_calls'}]),'COMPLETE'),
|
|
]
|
|
for payload,expected in fixtures:
|
|
with self.subTest(expected=expected,payload=payload):
|
|
self.assertEqual(validate_completion(payload,True),expected)
|
|
p=self.provider(lambda n,body:(200,{'Content-Type':'text/event-stream'},fixtures[0][0]))
|
|
b=self.broker(p)
|
|
self.assertEqual(self.request(b.issue_scope('e','teacher','glm-test'),body={'model':'glm-test','messages':[],'max_tokens':5,'stream':True})[0],503)
|
|
self.assertNotEqual(b.execution_status('e')['generations'][0]['state'],'COMPLETE')
|
|
|
|
def test_independent_snapshot_is_readonly_bounded_and_missing_is_unknown(self):
|
|
absent=Path(self.tmp.name)/'missing'/'ledger'
|
|
self.assertEqual(quota_snapshot(absent)['status'],'UNKNOWN')
|
|
self.assertFalse(absent.parent.exists())
|
|
p=self.provider();b=self.broker(p)
|
|
for i in range(3): self.request(b.issue_scope(str(i),'teacher','glm-test'))
|
|
before=hashlib.sha256(self.ledger.read_bytes()).hexdigest()
|
|
with patch.object(Broker,'__init__',side_effect=AssertionError('must not construct')):
|
|
snap=quota_snapshot(self.ledger,limit=2)
|
|
self.assertEqual(snap['status'],'OBSERVED')
|
|
self.assertLessEqual(len(snap['attempts']),2);self.assertLessEqual(len(snap['events']),2)
|
|
self.assertNotIn('upstream-secret',json.dumps(snap));self.assertNotIn('token_hash',json.dumps(snap))
|
|
self.assertEqual(hashlib.sha256(self.ledger.read_bytes()).hexdigest(),before)
|
|
self.assertEqual(snap['groups'][0]['inflight'],0)
|
|
|
|
def test_connect_failure_pause_resumes_after_cooldown(self):
|
|
p=self.provider();b=self.broker(p,self.policy())
|
|
scope=b.issue_scope('blip','teacher','glm-test')
|
|
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
|
|
def refused(connection):
|
|
if connection.port==p.server.server_port:
|
|
raise ConnectionRefusedError('injected Wi-Fi blip')
|
|
return original(connection)
|
|
with patch('harness.quota.http.client.HTTPConnection.connect',new=refused):
|
|
self.assertEqual(self.request(scope,'g')[0],503)
|
|
with b._db() as con:
|
|
cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0]
|
|
with patch.object(b,'_now',return_value=max(b._now(),cooldown)+1):
|
|
self.assertTrue(b.resume_readiness('blip')['eligible'])
|
|
b.resume('blip','g',checkpoint_complete=True,side_effects_known=True)
|
|
self.assertEqual(self.request(scope,'g')[0],200)
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_connect_blip_is_resent_in_place_and_charged_to_the_wait_budget(self):
|
|
# 10-14: one refused connect paused a 10-hour rollout; four of them stranded it at 3/3 resumes.
|
|
p=self.provider();b=self.broker(p,self.policy())
|
|
scope=b.issue_scope('blip','teacher','glm-test')
|
|
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
|
|
refusals=[2]
|
|
def flaky(connection):
|
|
if connection.port==p.server.server_port and refusals[0]:
|
|
refusals[0]-=1
|
|
raise ConnectionRefusedError('injected Wi-Fi blip')
|
|
return original(connection)
|
|
with patch('harness.quota.http.client.HTTPConnection.connect',new=flaky):
|
|
self.assertEqual(self.request(scope)[0],200)
|
|
self.assertEqual(len(p.requests),1)
|
|
self.assertGreaterEqual(b.execution_status('blip')['wait_spent'],2*(.06+.2))
|
|
self.assertEqual(len([e for e in b.snapshot()['quota_events'] if e['event']=='UNSENT_RETRY']),2)
|
|
|
|
def test_pause_that_sent_nothing_does_not_spend_the_resume_budget(self):
|
|
p=self.provider();b=self.broker(p,self.policy())
|
|
scope=b.issue_scope('free','teacher','glm-test')
|
|
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
|
|
def refused(connection):
|
|
if connection.port==p.server.server_port:
|
|
raise ConnectionRefusedError('injected outage')
|
|
return original(connection)
|
|
with patch('harness.quota.http.client.HTTPConnection.connect',new=refused):
|
|
self.assertEqual(self.request(scope,'g')[0],503) # in-place retries spend the wait budget, then pause
|
|
cap=b.groups['shared'].max_auto_resumes
|
|
with b._db() as con:
|
|
con.execute('UPDATE executions SET resumes=? WHERE execution_id=?',(cap,'free'))
|
|
cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0]
|
|
with patch.object(b,'_now',return_value=max(b._now(),cooldown)+1):
|
|
ready=b.resume_readiness('free')
|
|
self.assertTrue(ready['eligible']);self.assertFalse(ready['counted'])
|
|
b.resume('free','g',checkpoint_complete=True,side_effects_known=True)
|
|
self.assertEqual(b.execution_status('free')['resumes'],cap)
|
|
self.assertEqual(p.requests,[])
|
|
|
|
def test_confirmed_connect_failure_does_not_spend_http_budget(self):
|
|
p=self.provider();b=self.broker(p,self.policy(rpm=1,window_s=1))
|
|
scope=b.issue_scope('connect-failure','teacher','glm-test')
|
|
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.connect
|
|
def refused(connection):
|
|
if connection.port==p.server.server_port:
|
|
raise ConnectionRefusedError('injected TCP refusal before HTTP')
|
|
return original(connection)
|
|
with patch('harness.quota.http.client.HTTPConnection.connect',new=refused):
|
|
self.assertEqual(self.request(scope)[0],503)
|
|
generation=b.execution_status('connect-failure')['generations'][0]
|
|
self.assertEqual(generation['attempts'],0)
|
|
self.assertEqual(generation['reason'],'CONNECT_FAILED')
|
|
self.assertEqual(generation['state'],'PAUSED')
|
|
self.assertEqual(p.requests,[])
|
|
snapshot=quota_snapshot(self.ledger)
|
|
self.assertEqual(snapshot['groups'][0]['rpm_used'],0)
|
|
self.assertEqual(snapshot['groups'][0]['tpm_reserved'],0)
|
|
self.assertEqual(snapshot['groups'][0]['inflight'],0)
|
|
self.assertTrue(all(row['sent']==0 and row['state']=='NOT_SENT' for row in snapshot['attempts']))
|
|
self.assertIn('TRANSPORT_NOT_SENT',[event['event'] for event in snapshot['events']])
|
|
# The same account can serve a later request; no phantom RPM reservation.
|
|
with b._db() as con:
|
|
cooldown=con.execute('SELECT cooldown_until FROM quota_groups').fetchone()[0]
|
|
with patch.object(b,'_now',return_value=max(b._now(),cooldown)):
|
|
self.assertEqual(self.request(b.issue_scope('next','teacher','glm-test'))[0],200)
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_unknown_send_never_refunds_http_budget(self):
|
|
p=self.provider();b=self.broker(p,self.policy(window_s=1))
|
|
scope=b.issue_scope('unknown-send','teacher','glm-test')
|
|
original=__import__('http.client',fromlist=['HTTPConnection']).HTTPConnection.request
|
|
def interrupted(connection,*args,**kwargs):
|
|
original(connection,*args,**kwargs)
|
|
if connection.port==p.server.server_port:
|
|
raise OSError('injected interruption after request bytes')
|
|
with patch('harness.quota.http.client.HTTPConnection.request',new=interrupted):
|
|
self.assertEqual(self.request(scope)[0],503)
|
|
generation=b.execution_status('unknown-send')['generations'][0]
|
|
self.assertEqual(generation['attempts'],1)
|
|
snapshot=quota_snapshot(self.ledger)
|
|
self.assertEqual(snapshot['groups'][0]['rpm_used'],1)
|
|
self.assertGreater(snapshot['groups'][0]['tpm_reserved'],0)
|
|
self.assertEqual(snapshot['groups'][0]['inflight'],1)
|
|
self.assertEqual(snapshot['attempts'][0]['state'],'UNKNOWN')
|
|
self.assertIsNone(snapshot['attempts'][0]['ended_at'])
|
|
self.assertNotIn('TRANSPORT_NOT_SENT',[event['event'] for event in snapshot['events']])
|
|
|
|
def test_full_context_400_goes_back_to_pi_and_fences_nothing(self):
|
|
# 10-16: seven Qwen rollouts ended on vLLM's "maximum context length" 400: the agent's context, not the platform.
|
|
# Pi compacts on it and sends again, so the generation must fail without fencing the execution.
|
|
full = b'{"error":{"message":"This model\'s maximum context length is 262144 tokens. However, you requested 270000 tokens."}}'
|
|
p=self.provider(lambda n,body:(400,{'Content-Type':'application/json'},full) if n==1 else ok()); b=self.broker(p)
|
|
s=b.issue_scope('full','teacher','glm-test')
|
|
status,body=self.request(s,'g')[:2]
|
|
self.assertEqual(status,400); self.assertIn(b'context_length_exceeded',body)
|
|
first=b.execution_status('full')['generations'][0]
|
|
self.assertEqual((first['state'],first['reason']),('FAILED','CONTEXT_TOO_LONG'))
|
|
self.assertFalse(b.resume_readiness('full')['eligible'])
|
|
self.assertEqual(self.request(s,'g2')[0],200)
|
|
|
|
def test_request_over_the_account_limit_is_a_full_context_for_pi(self):
|
|
# 10-19 staging: a 413 Pi did not recognise ended the rollout as an infrastructure error instead of compacting.
|
|
calls=[]; p=self.provider(lambda n,body:(calls.append(n), ok())[1]); b=self.broker(p,self.policy(tpm=2000))
|
|
s=b.issue_scope('big','teacher','glm-test')
|
|
big={'model':'glm-test','messages':[{'role':'user','content':'x'*3000}],'max_tokens':5}
|
|
status,body=self.request(s,'g',big)[:2]
|
|
self.assertEqual((status,body),(400,CONTEXT_OVERFLOW_BODY))
|
|
self.assertEqual(calls,[])
|
|
first=b.execution_status('big')['generations'][0]
|
|
self.assertEqual((first['state'],first['reason']),('FAILED','CONTEXT_TOO_LONG'))
|
|
self.assertEqual(self.request(s,'g2')[0],200) # the compacted retry goes through
|
|
with self.assertRaisesRegex(QuotaError,'REQUEST_EXCEEDS_INTERNAL_TPM'): # content review keeps its reason
|
|
b.generate(b.issue_scope('review','teacher','glm-test').api_key,json.dumps(big).encode(),'review:0')
|
|
|
|
def test_other_400_stays_upstream_rejected(self):
|
|
p=self.provider(lambda n,body:(400,{'Content-Type':'application/json'},b'{"error":"invalid tool schema"}')); b=self.broker(p)
|
|
s=b.issue_scope('bad','teacher','glm-test')
|
|
self.assertEqual(self.request(s,'g')[0],503)
|
|
self.assertEqual(b.execution_status('bad')['generations'][0]['reason'],'UPSTREAM_REJECTED')
|
|
|
|
def test_dropped_connection_holds_the_account_briefly(self):
|
|
# 10-16 14:35: one dropped stream held 05's account until the request's own deadline (~25 min).
|
|
import struct
|
|
def cut(wfile):
|
|
wfile.write(b'data: {"choices":[{"delta":{"content":"par"}}]}\n\n'); wfile.flush()
|
|
wfile._sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
|
|
wfile._sock.close() # RST: the client sees a reset, not an end
|
|
p=self.provider(lambda n,body:(200,{'Content-Type':'text/event-stream'},cut))
|
|
b=self.broker(p,self.policy(request_total_s=600,stream_idle_timeout_s=5,first_byte_timeout_s=5))
|
|
s=b.issue_scope('cut','teacher','glm-test')
|
|
self.assertEqual(self.request(s,'g',{'model':'glm-test','messages':[],'max_tokens':5,'stream':True})[0],503)
|
|
attempt=b.snapshot()['attempts'][0]
|
|
self.assertEqual((attempt['state'],attempt['transport_reason']),('UNKNOWN','REQUEST_CONNECTION_INTERRUPTED'))
|
|
self.assertLessEqual(attempt['deadline']-attempt['started_at'],__import__('harness.quota',fromlist=['x']).UNKNOWN_HOLD_S+1)
|
|
b.pause_unsent('g') # possibly sent: an internal error later must not turn it into a replayable pause
|
|
self.assertNotEqual(b.execution_status('cut')['generations'][0]['reason'],'BROKER_ERROR')
|
|
|
|
def test_internal_error_before_sending_pauses_resumably(self):
|
|
# 10-16 05 hrnet.w32: an exception inside the broker ended a 2.9 h rollout as INFRA_INCOMPLETE_GENERATION.
|
|
p=self.provider(); b=self.broker(p); s=b.issue_scope('boom','teacher','glm-test')
|
|
with patch.object(b,'_generate',side_effect=RuntimeError('injected')):
|
|
self.assertEqual(self.request(s,'g'),(503,b'{"error":{"code":"BROKER_UNAVAILABLE"}}'))
|
|
gen=b.execution_status('boom')['generations'][0]
|
|
self.assertEqual((gen['state'],gen['reason']),('PAUSED','BROKER_ERROR'))
|
|
self.assertTrue(b.resume_readiness('boom')['eligible'])
|
|
b.resume('boom','g',checkpoint_complete=True,side_effects_known=True)
|
|
self.assertEqual(self.request(s,'g')[0],200)
|
|
self.assertEqual(len(p.requests),1)
|
|
|
|
def test_invalid_budget_rejected(self):
|
|
with self.assertRaises(ValueError): self.policy(rpm=None)
|
|
def test_model_scope_cannot_change(self):
|
|
p=self.provider(); b=self.broker(p); s=b.issue_scope('e','teacher','glm-test')
|
|
self.assertEqual(self.request(s,body={'model':'other','messages':[],'max_tokens':2})[0],403)
|
|
self.assertEqual(p.requests,[])
|
|
|
|
|
|
if __name__=='__main__': unittest.main()
|