240 lines
13 KiB
Python
240 lines
13 KiB
Python
import hashlib
|
|
import http.client
|
|
import ipaddress
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from unittest.mock import Mock, patch
|
|
from urllib.parse import urlsplit
|
|
|
|
from rollout.model_proxy import MAX_BODY_BYTES, ModelProxy, MODEL, QWEN_MODEL, UPSTREAM_IDLE_SECONDS
|
|
|
|
|
|
class ModelProxyTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.client_type = http.client.HTTPConnection
|
|
self.temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temporary.cleanup)
|
|
self.root = Path(self.temporary.name).resolve()
|
|
self.key = "test-provider-credential-do-not-forward"
|
|
self.credential = self.root / "credential.json"
|
|
self.credential.write_text(json.dumps({"model_api_key": self.key}))
|
|
self.credential.chmod(0o600)
|
|
self.log = self.root / "proxy.jsonl"
|
|
|
|
def proxy(self, model=MODEL):
|
|
return ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log, model=model)
|
|
|
|
def request(self, proxy, body=None, method="POST", path="/v1/chat/completions", headers=None):
|
|
endpoint = urlsplit(proxy.base_url)
|
|
client = http.client.HTTPConnection(endpoint.hostname, endpoint.port, timeout=5)
|
|
try:
|
|
client.request(method, path, body=body, headers=headers or {})
|
|
response = client.getresponse()
|
|
return response.status, dict(response.getheaders()), response.read()
|
|
finally:
|
|
client.close()
|
|
|
|
def upstream(self, status=200, chunks=None, content_type="application/json"):
|
|
response = Mock(status=status)
|
|
response.getheader.return_value = content_type
|
|
response.read1.side_effect = (chunks or [b'{"choices":[]}']) + [b""]
|
|
connection = Mock()
|
|
connection.getresponse.return_value = response
|
|
return connection, response
|
|
|
|
def test_credential_and_bind_trust_boundaries(self):
|
|
self.credential.chmod(0o640)
|
|
with self.assertRaises(ValueError):
|
|
self.proxy()
|
|
self.credential.chmod(0o600)
|
|
linked = self.root / "linked.json"
|
|
linked.symlink_to(self.credential)
|
|
with self.assertRaises(ValueError):
|
|
ModelProxy(linked, "127.0.0.1", "127.0.0.0/8", self.log)
|
|
self.credential.write_text('{"model_api_key":"bad\\nheader"}')
|
|
with self.assertRaises(ValueError):
|
|
self.proxy()
|
|
for bind, network in (("0.0.0.0", "0.0.0.0/8"), ("8.8.8.8", "8.0.0.0/8"),
|
|
("172.19.0.1", "172.18.0.0/16")):
|
|
with self.assertRaises(ValueError):
|
|
ModelProxy(self.credential, bind, network, self.log)
|
|
|
|
def test_source_model_path_method_and_body_are_rejected_without_upstream(self):
|
|
with patch("rollout.model_proxy.http.client.HTTPSConnection") as upstream:
|
|
with self.proxy() as proxy:
|
|
cases = [(b'{"model":"other"}', "POST", "/v1/chat/completions", 400),
|
|
(b'{"model":"glm-5.3","model":"other"}', "POST", "/v1/chat/completions", 400),
|
|
(b'[]', "POST", "/v1/chat/completions", 400),
|
|
(b'bad json', "POST", "/v1/chat/completions", 400),
|
|
(b'{"model":"glm-5.3"}', "POST", "/other", 404),
|
|
(None, "GET", "/v1/chat/completions", 405)]
|
|
for body, method, path, status in cases:
|
|
self.assertEqual(self.request(proxy, body, method, path)[0], status)
|
|
self.assertEqual(self.request(proxy, b"", headers={"Content-Length": str(MAX_BODY_BYTES + 1)})[0], 413)
|
|
self.assertEqual(self.request(proxy, b"", headers={"Transfer-Encoding": "chunked"})[0], 400)
|
|
proxy.network = ipaddress.ip_network("127.0.0.2/32")
|
|
self.assertEqual(self.request(proxy, b'{"model":"glm-5.3"}')[0], 403)
|
|
upstream.assert_not_called()
|
|
|
|
def test_fixed_upstream_authentication_and_json_transfer(self):
|
|
connection, response = self.upstream()
|
|
body = b'{"model":"glm-5.3","messages":[]}'
|
|
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream:
|
|
with self.proxy() as proxy:
|
|
status, headers, result = self.request(proxy, body, headers={"Authorization": "Bearer dummy"})
|
|
self.assertIsNone(proxy._key)
|
|
self.assertEqual((status, result), (200, b'{"choices":[]}'))
|
|
self.assertEqual(headers["Connection"], "close")
|
|
upstream.assert_called_once_with("open.bigmodel.cn", timeout=30)
|
|
connection.request.assert_called_once_with("POST", "/api/coding/paas/v4/chat/completions", body=body,
|
|
headers={"Authorization": "Bearer " + self.key,
|
|
"Content-Type": "application/json", "Connection": "close"})
|
|
connection.close.assert_called_once()
|
|
response.read1.assert_called_with(64 * 1024)
|
|
facts = json.loads(self.log.read_text())
|
|
self.assertEqual(facts["request_body_sha256"], hashlib.sha256(body).hexdigest())
|
|
self.assertEqual(facts["status"], 200)
|
|
self.assertEqual(facts["request_bytes"], len(body))
|
|
self.assertEqual(facts["response_bytes"], len(result))
|
|
self.assertNotIn(self.key, self.log.read_text())
|
|
self.assertNotIn("Authorization", self.log.read_text())
|
|
self.assertEqual(self.log.stat().st_mode & 0o777, 0o600)
|
|
|
|
def test_sse_transfer_preserves_chunks(self):
|
|
chunks = [b'data: {"choices":[]}\n\n', b'data: [DONE]\n\n']
|
|
connection, response = self.upstream(chunks=chunks, content_type="text/event-stream; charset=utf-8")
|
|
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection):
|
|
with self.proxy() as proxy:
|
|
status, headers, body = self.request(proxy, b'{"model":"glm-5.3","stream":true}')
|
|
self.assertEqual(status, 200)
|
|
self.assertEqual(headers["Content-Type"], "text/event-stream; charset=utf-8")
|
|
self.assertEqual(body, b"".join(chunks))
|
|
self.assertEqual(response.read1.call_count, 3)
|
|
self.assertEqual(UPSTREAM_IDLE_SECONDS, 900)
|
|
connection.sock.settimeout.assert_called_once_with(UPSTREAM_IDLE_SECONDS)
|
|
|
|
def test_qwen_responses_preserves_split_stream_and_confirms_terminal(self):
|
|
terminal = json.dumps(dict(type="response.completed", response=dict(model=QWEN_MODEL, status="completed")))
|
|
chunks = [b'data: {"type":"response.created"}\n\nda',
|
|
("ta: " + terminal[:30]).encode(), (terminal[30:] + "\n\n").encode()]
|
|
connection, response = self.upstream(chunks=chunks, content_type="text/event-stream")
|
|
body = json.dumps(dict(model=QWEN_MODEL, input="fix", stream=True)).encode()
|
|
with patch("rollout.model_proxy.http.client.HTTPConnection", return_value=connection) as upstream:
|
|
# Keep the caller's HTTPConnection real while only mocking the fixed upstream.
|
|
upstream.side_effect = lambda host, *args, **kwargs: connection if host == "114.111.18.115:18080" else self.client_type(host, *args, **kwargs)
|
|
with self.proxy(QWEN_MODEL) as proxy:
|
|
status, headers, result = self.request(proxy, body, path="/v1/responses")
|
|
self.assertEqual((status, result), (200, b"".join(chunks)))
|
|
connection.sock.settimeout.assert_called_once_with(UPSTREAM_IDLE_SECONDS)
|
|
connection.request.assert_called_once_with("POST", "/v1/responses", body=body,
|
|
headers={"Authorization": "Bearer " + self.key, "Content-Type": "application/json", "Connection": "close"})
|
|
facts = json.loads(self.log.read_text())
|
|
self.assertEqual(facts["responses_terminal"], "response.completed")
|
|
self.assertEqual(facts["responses_terminal_count"], 1)
|
|
self.assertEqual(facts["response_model"], QWEN_MODEL)
|
|
self.assertEqual(facts["response_status"], "completed")
|
|
self.assertTrue(facts["responses_stream_valid"] and facts["responses_stream_finished"])
|
|
self.assertNotIn(self.key, self.log.read_text())
|
|
|
|
def test_qwen_rejects_other_model_chat_path_or_nonstream_before_upstream(self):
|
|
with patch("rollout.model_proxy.http.client.HTTPConnection") as upstream:
|
|
upstream.side_effect = lambda host, *args, **kwargs: self.client_type(host, *args, **kwargs)
|
|
with self.proxy(QWEN_MODEL) as proxy:
|
|
for payload, path, expected in ((dict(model=MODEL, stream=True), "/v1/responses", 400),
|
|
(dict(model=QWEN_MODEL, stream=True), "/v1/chat/completions", 404),
|
|
(dict(model=QWEN_MODEL, stream=False), "/v1/responses", 400)):
|
|
self.assertEqual(self.request(proxy, json.dumps(payload).encode(), path=path)[0], expected)
|
|
self.assertFalse(any(call.args[0] == "114.111.18.115:18080" for call in upstream.call_args_list))
|
|
with self.assertRaises(ValueError):
|
|
self.proxy("unknown")
|
|
|
|
def test_provider_errors_are_sanitized_and_redirect_is_not_followed(self):
|
|
for status, expected in ((401, 401), (429, 429), (302, 502)):
|
|
self.log.unlink(missing_ok=True)
|
|
connection, response = self.upstream(status=status)
|
|
response.reason = self.key
|
|
response.read.return_value = self.key.encode()
|
|
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream, \
|
|
patch("rollout.model_proxy.time.sleep") as sleep:
|
|
with self.proxy() as proxy:
|
|
actual, headers, body = self.request(proxy, b'{"model":"glm-5.3"}')
|
|
self.assertEqual(actual, expected)
|
|
self.assertNotIn(self.key.encode(), body)
|
|
self.assertNotIn(self.key, self.log.read_text())
|
|
self.assertNotIn("Location", headers)
|
|
self.assertEqual(upstream.call_count, 3 if status == 429 else 1)
|
|
self.assertEqual(sleep.call_count, 2 if status == 429 else 0)
|
|
response.read.assert_not_called()
|
|
response.read1.assert_not_called()
|
|
|
|
def test_transient_failures_before_the_response_are_retried_and_recorded(self):
|
|
body = b'{"model":"glm-5.3","messages":[]}'
|
|
broken = Mock()
|
|
broken.connect.side_effect = ConnectionResetError("reset")
|
|
for attempts, expected, statuses in (([self.upstream(500)[0], self.upstream()[0]], 200, [500, 200]),
|
|
([broken, self.upstream()[0]], 200, ["connection_error", 200]),
|
|
([self.upstream(500)[0] for _ in range(3)], 500, [500, 500, 500])):
|
|
self.log.unlink(missing_ok=True)
|
|
with patch("rollout.model_proxy.http.client.HTTPSConnection", side_effect=attempts), \
|
|
patch("rollout.model_proxy.time.sleep") as sleep:
|
|
with self.proxy() as proxy:
|
|
status, _, result = self.request(proxy, body)
|
|
facts = json.loads(self.log.read_text())
|
|
self.assertEqual((status, facts["status"], facts["upstream_statuses"]), (expected, expected, statuses))
|
|
self.assertEqual([call.args[0] for call in sleep.call_args_list], [2, 5][:len(statuses) - 1])
|
|
if expected == 200:
|
|
self.assertEqual(result, b'{"choices":[]}')
|
|
|
|
def test_context_exit_interrupts_active_transfer_and_closes_listener(self):
|
|
connection, response = self.upstream()
|
|
connected = threading.Event()
|
|
stopped = threading.Event()
|
|
connection.sock.shutdown.side_effect = lambda how: stopped.set()
|
|
|
|
def waiting_response():
|
|
connected.set()
|
|
if not stopped.wait(5):
|
|
raise RuntimeError("proxy did not interrupt the upstream socket")
|
|
raise OSError("closed")
|
|
|
|
connection.getresponse.side_effect = waiting_response
|
|
errors = []
|
|
|
|
def call(proxy):
|
|
try:
|
|
self.request(proxy, b'{"model":"glm-5.3"}')
|
|
except (OSError, http.client.HTTPException) as error:
|
|
errors.append(type(error).__name__)
|
|
|
|
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection):
|
|
with self.proxy() as proxy:
|
|
worker = threading.Thread(target=call, args=(proxy,))
|
|
worker.start()
|
|
self.assertTrue(connected.wait(5))
|
|
worker.join(5)
|
|
self.assertFalse(worker.is_alive())
|
|
self.assertTrue(stopped.is_set())
|
|
self.assertEqual(len(self.log.read_text().splitlines()), 1)
|
|
with self.assertRaises(OSError):
|
|
self.request(proxy)
|
|
|
|
def test_server_start_failure_cleans_up_without_waiting_for_an_unstarted_thread(self):
|
|
thread = Mock()
|
|
thread.start.side_effect = RuntimeError("thread unavailable")
|
|
thread.is_alive.return_value = False
|
|
proxy = self.proxy()
|
|
with patch("rollout.model_proxy.threading.Thread", return_value=thread):
|
|
with self.assertRaises(RuntimeError):
|
|
proxy.__enter__()
|
|
self.assertIsNone(proxy._key)
|
|
self.assertIsNone(proxy._log)
|
|
self.assertEqual(proxy._server.socket.fileno(), -1)
|
|
thread.join.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|