data-gen-ebd/tests/test_model_proxy.py

240 lines
13 KiB
Python

import hashlib
import http.client
import ipaddress
import json
from pathlib import Path
import tempfile
import threading
import unittest
from unittest.mock import Mock, patch
from urllib.parse import urlsplit
from rollout.model_proxy import MAX_BODY_BYTES, ModelProxy, MODEL, QWEN_MODEL, UPSTREAM_IDLE_SECONDS
class ModelProxyTests(unittest.TestCase):
def setUp(self):
self.client_type = http.client.HTTPConnection
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name).resolve()
self.key = "test-provider-credential-do-not-forward"
self.credential = self.root / "credential.json"
self.credential.write_text(json.dumps({"model_api_key": self.key}))
self.credential.chmod(0o600)
self.log = self.root / "proxy.jsonl"
def proxy(self, model=MODEL):
return ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log, model=model)
def request(self, proxy, body=None, method="POST", path="/v1/chat/completions", headers=None):
endpoint = urlsplit(proxy.base_url)
client = http.client.HTTPConnection(endpoint.hostname, endpoint.port, timeout=5)
try:
client.request(method, path, body=body, headers=headers or {})
response = client.getresponse()
return response.status, dict(response.getheaders()), response.read()
finally:
client.close()
def upstream(self, status=200, chunks=None, content_type="application/json"):
response = Mock(status=status)
response.getheader.return_value = content_type
response.read1.side_effect = (chunks or [b'{"choices":[]}']) + [b""]
connection = Mock()
connection.getresponse.return_value = response
return connection, response
def test_credential_and_bind_trust_boundaries(self):
self.credential.chmod(0o640)
with self.assertRaises(ValueError):
self.proxy()
self.credential.chmod(0o600)
linked = self.root / "linked.json"
linked.symlink_to(self.credential)
with self.assertRaises(ValueError):
ModelProxy(linked, "127.0.0.1", "127.0.0.0/8", self.log)
self.credential.write_text('{"model_api_key":"bad\\nheader"}')
with self.assertRaises(ValueError):
self.proxy()
for bind, network in (("0.0.0.0", "0.0.0.0/8"), ("8.8.8.8", "8.0.0.0/8"),
("172.19.0.1", "172.18.0.0/16")):
with self.assertRaises(ValueError):
ModelProxy(self.credential, bind, network, self.log)
def test_source_model_path_method_and_body_are_rejected_without_upstream(self):
with patch("rollout.model_proxy.http.client.HTTPSConnection") as upstream:
with self.proxy() as proxy:
cases = [(b'{"model":"other"}', "POST", "/v1/chat/completions", 400),
(b'{"model":"glm-5.3","model":"other"}', "POST", "/v1/chat/completions", 400),
(b'[]', "POST", "/v1/chat/completions", 400),
(b'bad json', "POST", "/v1/chat/completions", 400),
(b'{"model":"glm-5.3"}', "POST", "/other", 404),
(None, "GET", "/v1/chat/completions", 405)]
for body, method, path, status in cases:
self.assertEqual(self.request(proxy, body, method, path)[0], status)
self.assertEqual(self.request(proxy, b"", headers={"Content-Length": str(MAX_BODY_BYTES + 1)})[0], 413)
self.assertEqual(self.request(proxy, b"", headers={"Transfer-Encoding": "chunked"})[0], 400)
proxy.network = ipaddress.ip_network("127.0.0.2/32")
self.assertEqual(self.request(proxy, b'{"model":"glm-5.3"}')[0], 403)
upstream.assert_not_called()
def test_fixed_upstream_authentication_and_json_transfer(self):
connection, response = self.upstream()
body = b'{"model":"glm-5.3","messages":[]}'
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream:
with self.proxy() as proxy:
status, headers, result = self.request(proxy, body, headers={"Authorization": "Bearer dummy"})
self.assertIsNone(proxy._key)
self.assertEqual((status, result), (200, b'{"choices":[]}'))
self.assertEqual(headers["Connection"], "close")
upstream.assert_called_once_with("open.bigmodel.cn", timeout=30)
connection.request.assert_called_once_with("POST", "/api/coding/paas/v4/chat/completions", body=body,
headers={"Authorization": "Bearer " + self.key,
"Content-Type": "application/json", "Connection": "close"})
connection.close.assert_called_once()
response.read1.assert_called_with(64 * 1024)
facts = json.loads(self.log.read_text())
self.assertEqual(facts["request_body_sha256"], hashlib.sha256(body).hexdigest())
self.assertEqual(facts["status"], 200)
self.assertEqual(facts["request_bytes"], len(body))
self.assertEqual(facts["response_bytes"], len(result))
self.assertNotIn(self.key, self.log.read_text())
self.assertNotIn("Authorization", self.log.read_text())
self.assertEqual(self.log.stat().st_mode & 0o777, 0o600)
def test_sse_transfer_preserves_chunks(self):
chunks = [b'data: {"choices":[]}\n\n', b'data: [DONE]\n\n']
connection, response = self.upstream(chunks=chunks, content_type="text/event-stream; charset=utf-8")
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection):
with self.proxy() as proxy:
status, headers, body = self.request(proxy, b'{"model":"glm-5.3","stream":true}')
self.assertEqual(status, 200)
self.assertEqual(headers["Content-Type"], "text/event-stream; charset=utf-8")
self.assertEqual(body, b"".join(chunks))
self.assertEqual(response.read1.call_count, 3)
self.assertEqual(UPSTREAM_IDLE_SECONDS, 900)
connection.sock.settimeout.assert_called_once_with(UPSTREAM_IDLE_SECONDS)
def test_qwen_responses_preserves_split_stream_and_confirms_terminal(self):
terminal = json.dumps(dict(type="response.completed", response=dict(model=QWEN_MODEL, status="completed")))
chunks = [b'data: {"type":"response.created"}\n\nda',
("ta: " + terminal[:30]).encode(), (terminal[30:] + "\n\n").encode()]
connection, response = self.upstream(chunks=chunks, content_type="text/event-stream")
body = json.dumps(dict(model=QWEN_MODEL, input="fix", stream=True)).encode()
with patch("rollout.model_proxy.http.client.HTTPConnection", return_value=connection) as upstream:
# Keep the caller's HTTPConnection real while only mocking the fixed upstream.
upstream.side_effect = lambda host, *args, **kwargs: connection if host == "114.111.18.115:18080" else self.client_type(host, *args, **kwargs)
with self.proxy(QWEN_MODEL) as proxy:
status, headers, result = self.request(proxy, body, path="/v1/responses")
self.assertEqual((status, result), (200, b"".join(chunks)))
connection.sock.settimeout.assert_called_once_with(UPSTREAM_IDLE_SECONDS)
connection.request.assert_called_once_with("POST", "/v1/responses", body=body,
headers={"Authorization": "Bearer " + self.key, "Content-Type": "application/json", "Connection": "close"})
facts = json.loads(self.log.read_text())
self.assertEqual(facts["responses_terminal"], "response.completed")
self.assertEqual(facts["responses_terminal_count"], 1)
self.assertEqual(facts["response_model"], QWEN_MODEL)
self.assertEqual(facts["response_status"], "completed")
self.assertTrue(facts["responses_stream_valid"] and facts["responses_stream_finished"])
self.assertNotIn(self.key, self.log.read_text())
def test_qwen_rejects_other_model_chat_path_or_nonstream_before_upstream(self):
with patch("rollout.model_proxy.http.client.HTTPConnection") as upstream:
upstream.side_effect = lambda host, *args, **kwargs: self.client_type(host, *args, **kwargs)
with self.proxy(QWEN_MODEL) as proxy:
for payload, path, expected in ((dict(model=MODEL, stream=True), "/v1/responses", 400),
(dict(model=QWEN_MODEL, stream=True), "/v1/chat/completions", 404),
(dict(model=QWEN_MODEL, stream=False), "/v1/responses", 400)):
self.assertEqual(self.request(proxy, json.dumps(payload).encode(), path=path)[0], expected)
self.assertFalse(any(call.args[0] == "114.111.18.115:18080" for call in upstream.call_args_list))
with self.assertRaises(ValueError):
self.proxy("unknown")
def test_provider_errors_are_sanitized_and_redirect_is_not_followed(self):
for status, expected in ((401, 401), (429, 429), (302, 502)):
self.log.unlink(missing_ok=True)
connection, response = self.upstream(status=status)
response.reason = self.key
response.read.return_value = self.key.encode()
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream, \
patch("rollout.model_proxy.time.sleep") as sleep:
with self.proxy() as proxy:
actual, headers, body = self.request(proxy, b'{"model":"glm-5.3"}')
self.assertEqual(actual, expected)
self.assertNotIn(self.key.encode(), body)
self.assertNotIn(self.key, self.log.read_text())
self.assertNotIn("Location", headers)
self.assertEqual(upstream.call_count, 3 if status == 429 else 1)
self.assertEqual(sleep.call_count, 2 if status == 429 else 0)
response.read.assert_not_called()
response.read1.assert_not_called()
def test_transient_failures_before_the_response_are_retried_and_recorded(self):
body = b'{"model":"glm-5.3","messages":[]}'
broken = Mock()
broken.connect.side_effect = ConnectionResetError("reset")
for attempts, expected, statuses in (([self.upstream(500)[0], self.upstream()[0]], 200, [500, 200]),
([broken, self.upstream()[0]], 200, ["connection_error", 200]),
([self.upstream(500)[0] for _ in range(3)], 500, [500, 500, 500])):
self.log.unlink(missing_ok=True)
with patch("rollout.model_proxy.http.client.HTTPSConnection", side_effect=attempts), \
patch("rollout.model_proxy.time.sleep") as sleep:
with self.proxy() as proxy:
status, _, result = self.request(proxy, body)
facts = json.loads(self.log.read_text())
self.assertEqual((status, facts["status"], facts["upstream_statuses"]), (expected, expected, statuses))
self.assertEqual([call.args[0] for call in sleep.call_args_list], [2, 5][:len(statuses) - 1])
if expected == 200:
self.assertEqual(result, b'{"choices":[]}')
def test_context_exit_interrupts_active_transfer_and_closes_listener(self):
connection, response = self.upstream()
connected = threading.Event()
stopped = threading.Event()
connection.sock.shutdown.side_effect = lambda how: stopped.set()
def waiting_response():
connected.set()
if not stopped.wait(5):
raise RuntimeError("proxy did not interrupt the upstream socket")
raise OSError("closed")
connection.getresponse.side_effect = waiting_response
errors = []
def call(proxy):
try:
self.request(proxy, b'{"model":"glm-5.3"}')
except (OSError, http.client.HTTPException) as error:
errors.append(type(error).__name__)
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection):
with self.proxy() as proxy:
worker = threading.Thread(target=call, args=(proxy,))
worker.start()
self.assertTrue(connected.wait(5))
worker.join(5)
self.assertFalse(worker.is_alive())
self.assertTrue(stopped.is_set())
self.assertEqual(len(self.log.read_text().splitlines()), 1)
with self.assertRaises(OSError):
self.request(proxy)
def test_server_start_failure_cleans_up_without_waiting_for_an_unstarted_thread(self):
thread = Mock()
thread.start.side_effect = RuntimeError("thread unavailable")
thread.is_alive.return_value = False
proxy = self.proxy()
with patch("rollout.model_proxy.threading.Thread", return_value=thread):
with self.assertRaises(RuntimeError):
proxy.__enter__()
self.assertIsNone(proxy._key)
self.assertIsNone(proxy._log)
self.assertEqual(proxy._server.socket.fileno(), -1)
thread.join.assert_not_called()
if __name__ == "__main__":
unittest.main()