The Qwen service runs four identical instances on 114.111.18.115:18080-18083. A credential may now carry upstream_port, limited to the ports the route lists, so rollout.batch's least-busy key choice spreads attempts across instances. The broker log records the upstream each request used. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
435 lines
24 KiB
Python
435 lines
24 KiB
Python
import hashlib
|
|
import http.client
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
import ipaddress
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from unittest.mock import Mock, patch
|
|
from urllib.parse import urlsplit
|
|
|
|
from rollout import model_proxy
|
|
from rollout.model_proxy import MAX_BODY_BYTES, ModelProxy, MODEL, MODELS, QWEN_MODEL, UPSTREAM_IDLE_SECONDS
|
|
from rollout.model_proxy import _credential
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
FAST = dict(STALL_SECONDS=0.4, KEEPALIVE_SECONDS=0.1, RETRY_DELAYS=(0, 0))
|
|
|
|
|
|
def sse(*events):
|
|
return b"".join(b"data: " + json.dumps(event).encode() + b"\n\n" for event in events)
|
|
|
|
|
|
PREAMBLE = sse(dict(type="response.created"), dict(type="response.in_progress"))
|
|
REPLY = sse(dict(type="response.output_text.delta", delta="ok"),
|
|
dict(type="response.completed", response=dict(model=QWEN_MODEL, status="completed")))
|
|
|
|
|
|
class ModelProxyTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.client_type = http.client.HTTPConnection
|
|
self.temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temporary.cleanup)
|
|
self.root = Path(self.temporary.name).resolve()
|
|
self.key = "test-provider-credential-do-not-forward"
|
|
self.credential = self.root / "credential.json"
|
|
self.credential.write_text(json.dumps({"model_api_key": self.key}))
|
|
self.credential.chmod(0o600)
|
|
self.log = self.root / "proxy.jsonl"
|
|
|
|
def proxy(self, model=MODEL):
|
|
return ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log, model=model)
|
|
|
|
def request(self, proxy, body=None, method="POST", path="/v1/chat/completions", headers=None):
|
|
endpoint = urlsplit(proxy.base_url)
|
|
client = http.client.HTTPConnection(endpoint.hostname, endpoint.port, timeout=5)
|
|
try:
|
|
client.request(method, path, body=body, headers=headers or {})
|
|
response = client.getresponse()
|
|
return response.status, dict(response.getheaders()), response.read()
|
|
finally:
|
|
client.close()
|
|
|
|
def upstream(self, status=200, chunks=None, content_type="application/json"):
|
|
response = Mock(status=status)
|
|
response.getheader.return_value = content_type
|
|
response.read1.side_effect = (chunks or [b'{"choices":[]}']) + [b""]
|
|
connection = Mock()
|
|
connection.getresponse.return_value = response
|
|
return connection, response
|
|
|
|
def test_credential_and_bind_trust_boundaries(self):
|
|
self.credential.chmod(0o640)
|
|
with self.assertRaises(ValueError):
|
|
self.proxy()
|
|
self.credential.chmod(0o600)
|
|
linked = self.root / "linked.json"
|
|
linked.symlink_to(self.credential)
|
|
with self.assertRaises(ValueError):
|
|
ModelProxy(linked, "127.0.0.1", "127.0.0.0/8", self.log)
|
|
self.credential.write_text('{"model_api_key":"bad\\nheader"}')
|
|
with self.assertRaises(ValueError):
|
|
self.proxy()
|
|
for bind, network in (("0.0.0.0", "0.0.0.0/8"), ("8.8.8.8", "8.0.0.0/8"),
|
|
("172.19.0.1", "172.18.0.0/16")):
|
|
with self.assertRaises(ValueError):
|
|
ModelProxy(self.credential, bind, network, self.log)
|
|
|
|
def test_source_model_path_method_and_body_are_rejected_without_upstream(self):
|
|
with patch("rollout.model_proxy.http.client.HTTPSConnection") as upstream:
|
|
with self.proxy() as proxy:
|
|
cases = [(b'{"model":"other"}', "POST", "/v1/chat/completions", 400),
|
|
(b'{"model":"glm-5.3","model":"other"}', "POST", "/v1/chat/completions", 400),
|
|
(b'[]', "POST", "/v1/chat/completions", 400),
|
|
(b'bad json', "POST", "/v1/chat/completions", 400),
|
|
(b'{"model":"glm-5.3"}', "POST", "/other", 404),
|
|
(None, "GET", "/v1/chat/completions", 405)]
|
|
for body, method, path, status in cases:
|
|
self.assertEqual(self.request(proxy, body, method, path)[0], status)
|
|
self.assertEqual(self.request(proxy, b"", headers={"Content-Length": str(MAX_BODY_BYTES + 1)})[0], 413)
|
|
self.assertEqual(self.request(proxy, b"", headers={"Transfer-Encoding": "chunked"})[0], 400)
|
|
proxy.network = ipaddress.ip_network("127.0.0.2/32")
|
|
self.assertEqual(self.request(proxy, b'{"model":"glm-5.3"}')[0], 403)
|
|
upstream.assert_not_called()
|
|
|
|
def test_token_proxy_rejects_missing_or_wrong_bearer_before_upstream(self):
|
|
token = "a" * 64
|
|
with self.assertRaises(ValueError):
|
|
ModelProxy(self.credential, "127.0.0.1", "10.0.0.0/8", self.log)
|
|
with self.assertRaises(ValueError):
|
|
ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log, token="short")
|
|
connection, _ = self.upstream()
|
|
body = b'{"model":"glm-5.3","messages":[]}'
|
|
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream:
|
|
with ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log, token=token) as proxy:
|
|
self.assertEqual(self.request(proxy, body)[0], 401)
|
|
self.assertEqual(self.request(proxy, body, headers={"Authorization": "Bearer " + "b" * 64})[0], 401)
|
|
upstream.assert_not_called()
|
|
self.assertEqual(self.request(proxy, body, headers={"Authorization": "Bearer " + token})[0], 200)
|
|
self.assertEqual(self.request(proxy, None, "GET", "/health",
|
|
headers={"Authorization": "Bearer " + token})[0], 204)
|
|
self.assertEqual(self.request(proxy, None, "GET", "/health")[0], 401)
|
|
self.assertEqual([json.loads(line)["status"] for line in self.log.read_text().splitlines()],
|
|
[401, 401, 200, 401])
|
|
self.assertNotIn(token, self.log.read_text())
|
|
|
|
def test_fixed_upstream_authentication_and_json_transfer(self):
|
|
connection, response = self.upstream()
|
|
body = b'{"model":"glm-5.3","messages":[]}'
|
|
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream:
|
|
with self.proxy() as proxy:
|
|
status, headers, result = self.request(proxy, body, headers={"Authorization": "Bearer dummy"})
|
|
self.assertIsNone(proxy._key)
|
|
self.assertEqual((status, result), (200, b'{"choices":[]}'))
|
|
self.assertEqual(headers["Connection"], "close")
|
|
upstream.assert_called_once_with("open.bigmodel.cn", timeout=30)
|
|
connection.request.assert_called_once_with("POST", "/api/coding/paas/v4/chat/completions", body=body,
|
|
headers={"Authorization": "Bearer " + self.key,
|
|
"Content-Type": "application/json", "Connection": "close"})
|
|
connection.close.assert_called_once()
|
|
response.read1.assert_called_with(64 * 1024)
|
|
facts = json.loads(self.log.read_text())
|
|
self.assertEqual(facts["request_body_sha256"], hashlib.sha256(body).hexdigest())
|
|
self.assertEqual(facts["status"], 200)
|
|
self.assertEqual(facts["request_bytes"], len(body))
|
|
self.assertEqual(facts["response_bytes"], len(result))
|
|
self.assertNotIn(self.key, self.log.read_text())
|
|
self.assertNotIn("Authorization", self.log.read_text())
|
|
self.assertEqual(self.log.stat().st_mode & 0o777, 0o600)
|
|
|
|
def test_sse_transfer_preserves_chunks(self):
|
|
chunks = [b'data: {"choices":[]}\n\n', b'data: [DONE]\n\n']
|
|
connection, response = self.upstream(chunks=chunks, content_type="text/event-stream; charset=utf-8")
|
|
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection):
|
|
with self.proxy() as proxy:
|
|
status, headers, body = self.request(proxy, b'{"model":"glm-5.3","stream":true}')
|
|
self.assertEqual(status, 200)
|
|
self.assertEqual(headers["Content-Type"], "text/event-stream; charset=utf-8")
|
|
self.assertEqual(body, b"".join(chunks))
|
|
self.assertEqual(response.read1.call_count, 3)
|
|
self.assertEqual(UPSTREAM_IDLE_SECONDS, 900)
|
|
connection.sock.settimeout.assert_called_once_with(UPSTREAM_IDLE_SECONDS)
|
|
|
|
def test_qwen_responses_preserves_split_stream_and_confirms_terminal(self):
|
|
terminal = json.dumps(dict(type="response.completed", response=dict(model=QWEN_MODEL, status="completed")))
|
|
chunks = [b'data: {"type":"response.created"}\n\nda',
|
|
("ta: " + terminal[:30]).encode(), (terminal[30:] + "\n\n").encode()]
|
|
connection, response = self.upstream(chunks=chunks, content_type="text/event-stream")
|
|
body = json.dumps(dict(model=QWEN_MODEL, input="fix", stream=True)).encode()
|
|
with patch("rollout.model_proxy.http.client.HTTPConnection", return_value=connection) as upstream:
|
|
# Keep the caller's HTTPConnection real while only mocking the fixed upstream.
|
|
upstream.side_effect = lambda host, *args, **kwargs: connection if host == "114.111.18.115:18080" else self.client_type(host, *args, **kwargs)
|
|
with self.proxy(QWEN_MODEL) as proxy:
|
|
status, headers, result = self.request(proxy, body, path="/v1/responses")
|
|
self.assertEqual((status, result), (200, b"".join(chunks)))
|
|
connection.sock.settimeout.assert_called_once_with(UPSTREAM_IDLE_SECONDS)
|
|
connection.request.assert_called_once_with("POST", "/v1/responses", body=body,
|
|
headers={"Authorization": "Bearer " + self.key, "Content-Type": "application/json", "Connection": "close"})
|
|
facts = json.loads(self.log.read_text())
|
|
self.assertEqual(facts["responses_terminal"], "response.completed")
|
|
self.assertEqual(facts["responses_terminal_count"], 1)
|
|
self.assertEqual(facts["response_model"], QWEN_MODEL)
|
|
self.assertEqual(facts["response_status"], "completed")
|
|
self.assertTrue(facts["responses_stream_valid"] and facts["responses_stream_finished"])
|
|
self.assertNotIn(self.key, self.log.read_text())
|
|
|
|
def qwen(self, *streams):
|
|
"""Run one Qwen request against scripted upstream attempts; a stream step is bytes, a pause or 'hang'."""
|
|
released = threading.Event()
|
|
connections = []
|
|
for steps in streams:
|
|
steps = iter(list(steps) + [b""])
|
|
|
|
def read1(size, steps=steps):
|
|
step = next(steps)
|
|
while isinstance(step, float):
|
|
time.sleep(step)
|
|
step = next(steps)
|
|
if step == "hang":
|
|
released.wait(5)
|
|
raise OSError("closed")
|
|
return step
|
|
|
|
connection, response = self.upstream(content_type="text/event-stream")
|
|
response.read1.side_effect = read1
|
|
connection.sock.shutdown.side_effect = lambda how: released.set()
|
|
connections.append(connection)
|
|
attempts = iter(connections)
|
|
body = json.dumps(dict(model=QWEN_MODEL, input="fix", stream=True)).encode()
|
|
with patch("rollout.model_proxy.http.client.HTTPConnection") as upstream, \
|
|
patch.multiple(model_proxy, **FAST):
|
|
upstream.side_effect = lambda host, *args, **kwargs: (
|
|
next(attempts) if host == "114.111.18.115:18080" else self.client_type(host, *args, **kwargs))
|
|
with self.proxy(QWEN_MODEL) as proxy:
|
|
try:
|
|
result = self.request(proxy, body, path="/v1/responses")[2]
|
|
except (OSError, http.client.HTTPException):
|
|
result = None
|
|
return result, json.loads(self.log.read_text())
|
|
|
|
def test_stall_before_generation_is_retried_and_pi_sees_one_response(self):
|
|
started = time.monotonic()
|
|
result, facts = self.qwen([PREAMBLE, "hang"], [PREAMBLE, REPLY])
|
|
self.assertLess(time.monotonic() - started, 3) # The stall timer fired, not the 5 s test fallback.
|
|
self.assertEqual(result.replace(b": keepalive\n\n", b""), PREAMBLE + REPLY)
|
|
self.assertIn(b": keepalive\n\n", result)
|
|
self.assertEqual(facts["upstream_statuses"], ["stalled", 200])
|
|
self.assertEqual((facts["status"], facts["responses_terminal"], facts["responses_terminal_count"]),
|
|
(200, "response.completed", 1))
|
|
self.assertTrue(facts["responses_stream_finished"])
|
|
|
|
def test_slow_first_event_gets_keepalives_and_no_retry(self):
|
|
result, facts = self.qwen([PREAMBLE, 0.3, REPLY])
|
|
self.assertEqual(result.replace(b": keepalive\n\n", b""), PREAMBLE + REPLY)
|
|
self.assertIn(b": keepalive\n\n", result)
|
|
self.assertEqual(facts["upstream_statuses"], [200])
|
|
self.assertGreaterEqual(facts["first_event_seconds"], 0.3)
|
|
|
|
def test_generation_already_forwarded_is_never_retried(self):
|
|
started = sse(dict(type="response.output_text.delta", delta="o"))
|
|
result, facts = self.qwen([PREAMBLE, started, "hang"], [PREAMBLE, REPLY])
|
|
self.assertEqual(facts["upstream_statuses"], [200])
|
|
self.assertIsNone(facts["responses_terminal"])
|
|
self.assertFalse(facts["responses_stream_finished"])
|
|
|
|
def test_every_attempt_stalling_ends_without_forwarding_a_preamble(self):
|
|
result, facts = self.qwen([PREAMBLE, "hang"], [PREAMBLE, "hang"], [PREAMBLE, "hang"])
|
|
self.assertEqual(facts["upstream_statuses"], ["stalled"] * 3)
|
|
self.assertNotIn(b"response.created", result or b"")
|
|
self.assertIsNone(facts["responses_terminal"])
|
|
|
|
@unittest.skipUnless(shutil.which("node") and (ROOT / "agent/node_modules/undici").is_dir(),
|
|
"needs Node and agent/node_modules with the pinned Pi SDK")
|
|
def test_real_pi_sdk_survives_a_stall_only_because_of_keepalives(self):
|
|
"""Pi's fetch gives up after 0.6 s here; the proxy waits 1 s, retries, and the retry takes 0.9 s."""
|
|
posts = []
|
|
|
|
class Upstream(BaseHTTPRequestHandler):
|
|
protocol_version = "HTTP/1.1"
|
|
|
|
def log_message(self, *args):
|
|
pass
|
|
|
|
def do_POST(self):
|
|
self.rfile.read(int(self.headers["Content-Length"]))
|
|
posts.append(time.monotonic())
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "text/event-stream")
|
|
self.send_header("Connection", "close")
|
|
self.end_headers()
|
|
self.wfile.write(sse(dict(type="response.created", response=dict(id="r")),
|
|
dict(type="response.in_progress", response=dict(id="r"))))
|
|
self.wfile.flush()
|
|
time.sleep(3 if len(posts) == 1 else 0.9)
|
|
item = dict(type="message", id="m", role="assistant", status="completed",
|
|
content=[dict(type="output_text", text="Done.", annotations=[])])
|
|
self.wfile.write(sse(
|
|
dict(type="response.output_item.added", output_index=0, item=dict(item, content=[])),
|
|
dict(type="response.output_text.delta", item_id="m", output_index=0, content_index=0,
|
|
delta="Done."),
|
|
dict(type="response.output_item.done", output_index=0, item=item),
|
|
dict(type="response.completed", response=dict(id="r", model=QWEN_MODEL, status="completed",
|
|
usage=dict(input_tokens=3, output_tokens=1,
|
|
total_tokens=4)))))
|
|
|
|
script = """
|
|
import {Agent, setGlobalDispatcher} from 'undici';
|
|
import fs from 'node:fs';
|
|
import {run, loadSdk} from './pi.mjs';
|
|
setGlobalDispatcher(new Agent({headersTimeout: 600, bodyTimeout: 600}));
|
|
const [baseUrl, directory] = process.argv.slice(1);
|
|
const summary = await run('Say done.', {cwd: directory, traceFile: directory + '/native.jsonl', baseUrl,
|
|
modelId: 'qwen3.8-27b-nvfp4', maxTokens: 64, mirror: () => {}}, await loadSdk());
|
|
const last = fs.readFileSync(directory + '/native.jsonl', 'utf8').trim().split('\\n').map(JSON.parse)
|
|
.filter(event => event.type === 'message_end').at(-1);
|
|
console.log(JSON.stringify({complete: summary.complete, error: last?.message?.errorMessage ?? null}));
|
|
"""
|
|
upstream = ThreadingHTTPServer(("127.0.0.1", 0), Upstream)
|
|
threading.Thread(target=upstream.serve_forever, daemon=True).start()
|
|
self.addCleanup(upstream.server_close)
|
|
self.addCleanup(upstream.shutdown)
|
|
outcomes = []
|
|
for keepalive in (0.2, 30):
|
|
posts.clear()
|
|
self.log.unlink(missing_ok=True)
|
|
work = Path(tempfile.mkdtemp(dir=self.root))
|
|
route = dict(MODELS[QWEN_MODEL], upstream="127.0.0.1:%d" % upstream.server_port)
|
|
with patch.dict(MODELS, {QWEN_MODEL: route}), patch.multiple(
|
|
model_proxy, STALL_SECONDS=1, KEEPALIVE_SECONDS=keepalive, RETRY_DELAYS=(0, 0)):
|
|
with self.proxy(QWEN_MODEL) as proxy:
|
|
done = subprocess.run(["node", "--input-type=module", "-e", script, proxy.base_url, str(work)],
|
|
cwd=ROOT / "agent", capture_output=True, text=True, timeout=60,
|
|
env=dict(os.environ, NODE_NO_WARNINGS="1"))
|
|
self.assertEqual(done.returncode, 0, done.stderr)
|
|
outcomes.append((json.loads(done.stdout.splitlines()[-1]), len(posts),
|
|
json.loads(self.log.read_text().splitlines()[0])["upstream_statuses"]))
|
|
(kept, kept_posts, kept_statuses), (bare, _, _) = outcomes
|
|
self.assertEqual(kept, dict(complete=True, error=None))
|
|
self.assertEqual((kept_posts, kept_statuses), (2, ["stalled", 200]))
|
|
# Control: without keepalives the same stall kills the session the way batch 6 lost 26 attempts.
|
|
self.assertEqual(bare, dict(complete=False, error="terminated"))
|
|
|
|
def test_qwen_rejects_other_model_chat_path_or_nonstream_before_upstream(self):
|
|
with patch("rollout.model_proxy.http.client.HTTPConnection") as upstream:
|
|
upstream.side_effect = lambda host, *args, **kwargs: self.client_type(host, *args, **kwargs)
|
|
with self.proxy(QWEN_MODEL) as proxy:
|
|
for payload, path, expected in ((dict(model=MODEL, stream=True), "/v1/responses", 400),
|
|
(dict(model=QWEN_MODEL, stream=True), "/v1/chat/completions", 404),
|
|
(dict(model=QWEN_MODEL, stream=False), "/v1/responses", 400)):
|
|
self.assertEqual(self.request(proxy, json.dumps(payload).encode(), path=path)[0], expected)
|
|
self.assertFalse(any(call.args[0] == "114.111.18.115:18080" for call in upstream.call_args_list))
|
|
with self.assertRaises(ValueError):
|
|
self.proxy("unknown")
|
|
|
|
def test_provider_errors_are_sanitized_and_redirect_is_not_followed(self):
|
|
for status, expected in ((401, 401), (429, 429), (302, 502)):
|
|
self.log.unlink(missing_ok=True)
|
|
connection, response = self.upstream(status=status)
|
|
response.reason = self.key
|
|
response.read.return_value = self.key.encode()
|
|
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream, \
|
|
patch("rollout.model_proxy.time.sleep") as sleep:
|
|
with self.proxy() as proxy:
|
|
actual, headers, body = self.request(proxy, b'{"model":"glm-5.3"}')
|
|
self.assertEqual(actual, expected)
|
|
self.assertNotIn(self.key.encode(), body)
|
|
self.assertNotIn(self.key, self.log.read_text())
|
|
self.assertNotIn("Location", headers)
|
|
self.assertEqual(upstream.call_count, 3 if status == 429 else 1)
|
|
self.assertEqual(sleep.call_count, 2 if status == 429 else 0)
|
|
response.read.assert_not_called()
|
|
response.read1.assert_not_called()
|
|
|
|
def test_transient_failures_before_the_response_are_retried_and_recorded(self):
|
|
body = b'{"model":"glm-5.3","messages":[]}'
|
|
broken = Mock()
|
|
broken.connect.side_effect = ConnectionResetError("reset")
|
|
for attempts, expected, statuses in (([self.upstream(500)[0], self.upstream()[0]], 200, [500, 200]),
|
|
([broken, self.upstream()[0]], 200, ["connection_error", 200]),
|
|
([self.upstream(500)[0] for _ in range(3)], 500, [500, 500, 500])):
|
|
self.log.unlink(missing_ok=True)
|
|
with patch("rollout.model_proxy.http.client.HTTPSConnection", side_effect=attempts), \
|
|
patch("rollout.model_proxy.time.sleep") as sleep:
|
|
with self.proxy() as proxy:
|
|
status, _, result = self.request(proxy, body)
|
|
facts = json.loads(self.log.read_text())
|
|
self.assertEqual((status, facts["status"], facts["upstream_statuses"]), (expected, expected, statuses))
|
|
self.assertEqual([call.args[0] for call in sleep.call_args_list], [2, 5][:len(statuses) - 1])
|
|
if expected == 200:
|
|
self.assertEqual(result, b'{"choices":[]}')
|
|
|
|
def test_context_exit_interrupts_active_transfer_and_closes_listener(self):
|
|
connection, response = self.upstream()
|
|
connected = threading.Event()
|
|
stopped = threading.Event()
|
|
connection.sock.shutdown.side_effect = lambda how: stopped.set()
|
|
|
|
def waiting_response():
|
|
connected.set()
|
|
if not stopped.wait(5):
|
|
raise RuntimeError("proxy did not interrupt the upstream socket")
|
|
raise OSError("closed")
|
|
|
|
connection.getresponse.side_effect = waiting_response
|
|
errors = []
|
|
|
|
def call(proxy):
|
|
try:
|
|
self.request(proxy, b'{"model":"glm-5.3"}')
|
|
except (OSError, http.client.HTTPException) as error:
|
|
errors.append(type(error).__name__)
|
|
|
|
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection):
|
|
with self.proxy() as proxy:
|
|
worker = threading.Thread(target=call, args=(proxy,))
|
|
worker.start()
|
|
self.assertTrue(connected.wait(5))
|
|
worker.join(5)
|
|
self.assertFalse(worker.is_alive())
|
|
self.assertTrue(stopped.is_set())
|
|
self.assertEqual(len(self.log.read_text().splitlines()), 1)
|
|
with self.assertRaises(OSError):
|
|
self.request(proxy)
|
|
|
|
def test_server_start_failure_cleans_up_without_waiting_for_an_unstarted_thread(self):
|
|
thread = Mock()
|
|
thread.start.side_effect = RuntimeError("thread unavailable")
|
|
thread.is_alive.return_value = False
|
|
proxy = self.proxy()
|
|
with patch("rollout.model_proxy.threading.Thread", return_value=thread):
|
|
with self.assertRaises(RuntimeError):
|
|
proxy.__enter__()
|
|
self.assertIsNone(proxy._key)
|
|
self.assertIsNone(proxy._log)
|
|
self.assertEqual(proxy._server.socket.fileno(), -1)
|
|
thread.join.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|
|
|
|
|
|
class CredentialInstanceTests(unittest.TestCase):
|
|
def write(self, data):
|
|
folder = os.path.realpath(tempfile.mkdtemp()) # macOS /var is a symlink, which credentials refuse
|
|
self.addCleanup(shutil.rmtree, folder)
|
|
path = Path(folder, "key.json")
|
|
path.write_text(json.dumps(data))
|
|
path.chmod(0o600)
|
|
return path
|
|
|
|
def test_a_credential_may_pick_only_a_listed_instance_of_its_model(self):
|
|
qwen, glm = MODELS[QWEN_MODEL], MODELS["glm-5.3"]
|
|
self.assertEqual(_credential(self.write(dict(model_api_key="k")), qwen), ("k", "114.111.18.115:18080"))
|
|
self.assertEqual(_credential(self.write(dict(model_api_key="k", upstream_port=18083)), qwen),
|
|
("k", "114.111.18.115:18083"))
|
|
for port, route in ((18084, qwen), ("18081", qwen), (18081, glm)):
|
|
with self.assertRaises(ValueError):
|
|
_credential(self.write(dict(model_api_key="k", upstream_port=port)), route)
|