data-gen-ebd/tests/test_model_proxy.py

182 lines
8.7 KiB
Python

import hashlib
import http.client
import ipaddress
import json
from pathlib import Path
import tempfile
import threading
import unittest
from unittest.mock import Mock, patch
from urllib.parse import urlsplit
from rollout.model_proxy import MAX_BODY_BYTES, ModelProxy, UPSTREAM_HOST, UPSTREAM_PATH
class ModelProxyTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name).resolve()
self.key = "test-provider-credential-do-not-forward"
self.credential = self.root / "credential.json"
self.credential.write_text(json.dumps({"model_api_key": self.key}))
self.credential.chmod(0o600)
self.log = self.root / "proxy.jsonl"
def proxy(self):
return ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log)
def request(self, proxy, body=None, method="POST", path="/v1/chat/completions", headers=None):
endpoint = urlsplit(proxy.base_url)
client = http.client.HTTPConnection(endpoint.hostname, endpoint.port, timeout=5)
try:
client.request(method, path, body=body, headers=headers or {})
response = client.getresponse()
return response.status, dict(response.getheaders()), response.read()
finally:
client.close()
def upstream(self, status=200, chunks=None, content_type="application/json"):
response = Mock(status=status)
response.getheader.return_value = content_type
response.read1.side_effect = (chunks or [b'{"choices":[]}']) + [b""]
connection = Mock()
connection.getresponse.return_value = response
return connection, response
def test_credential_and_bind_trust_boundaries(self):
self.credential.chmod(0o640)
with self.assertRaises(ValueError):
self.proxy()
self.credential.chmod(0o600)
linked = self.root / "linked.json"
linked.symlink_to(self.credential)
with self.assertRaises(ValueError):
ModelProxy(linked, "127.0.0.1", "127.0.0.0/8", self.log)
self.credential.write_text('{"model_api_key":"bad\\nheader"}')
with self.assertRaises(ValueError):
self.proxy()
for bind, network in (("0.0.0.0", "0.0.0.0/8"), ("8.8.8.8", "8.0.0.0/8"),
("172.19.0.1", "172.18.0.0/16")):
with self.assertRaises(ValueError):
ModelProxy(self.credential, bind, network, self.log)
def test_source_model_path_method_and_body_are_rejected_without_upstream(self):
with patch("rollout.model_proxy.http.client.HTTPSConnection") as upstream:
with self.proxy() as proxy:
cases = [(b'{"model":"other"}', "POST", "/v1/chat/completions", 400),
(b'{"model":"glm-5.3","model":"other"}', "POST", "/v1/chat/completions", 400),
(b'[]', "POST", "/v1/chat/completions", 400),
(b'bad json', "POST", "/v1/chat/completions", 400),
(b'{"model":"glm-5.3"}', "POST", "/other", 404),
(None, "GET", "/v1/chat/completions", 405)]
for body, method, path, status in cases:
self.assertEqual(self.request(proxy, body, method, path)[0], status)
self.assertEqual(self.request(proxy, b"", headers={"Content-Length": str(MAX_BODY_BYTES + 1)})[0], 413)
self.assertEqual(self.request(proxy, b"", headers={"Transfer-Encoding": "chunked"})[0], 400)
proxy.network = ipaddress.ip_network("127.0.0.2/32")
self.assertEqual(self.request(proxy, b'{"model":"glm-5.3"}')[0], 403)
upstream.assert_not_called()
def test_fixed_upstream_authentication_and_json_transfer(self):
connection, response = self.upstream()
body = b'{"model":"glm-5.3","messages":[]}'
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream:
with self.proxy() as proxy:
status, headers, result = self.request(proxy, body, headers={"Authorization": "Bearer dummy"})
self.assertIsNone(proxy._key)
self.assertEqual((status, result), (200, b'{"choices":[]}'))
self.assertEqual(headers["Connection"], "close")
upstream.assert_called_once_with(UPSTREAM_HOST, timeout=30)
connection.request.assert_called_once_with("POST", UPSTREAM_PATH, body=body,
headers={"Authorization": "Bearer " + self.key,
"Content-Type": "application/json", "Connection": "close"})
connection.close.assert_called_once()
response.read1.assert_called_with(64 * 1024)
facts = json.loads(self.log.read_text())
self.assertEqual(facts["request_body_sha256"], hashlib.sha256(body).hexdigest())
self.assertEqual(facts["status"], 200)
self.assertEqual(facts["request_bytes"], len(body))
self.assertEqual(facts["response_bytes"], len(result))
self.assertNotIn(self.key, self.log.read_text())
self.assertNotIn("Authorization", self.log.read_text())
self.assertEqual(self.log.stat().st_mode & 0o777, 0o600)
def test_sse_transfer_preserves_chunks(self):
chunks = [b'data: {"choices":[]}\n\n', b'data: [DONE]\n\n']
connection, response = self.upstream(chunks=chunks, content_type="text/event-stream; charset=utf-8")
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection):
with self.proxy() as proxy:
status, headers, body = self.request(proxy, b'{"model":"glm-5.3","stream":true}')
self.assertEqual(status, 200)
self.assertEqual(headers["Content-Type"], "text/event-stream; charset=utf-8")
self.assertEqual(body, b"".join(chunks))
self.assertEqual(response.read1.call_count, 3)
def test_provider_errors_are_sanitized_and_redirect_is_not_followed(self):
for status, expected in ((401, 401), (429, 429), (302, 502)):
self.log.unlink(missing_ok=True)
connection, response = self.upstream(status=status)
response.reason = self.key
response.read.return_value = self.key.encode()
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream:
with self.proxy() as proxy:
actual, headers, body = self.request(proxy, b'{"model":"glm-5.3"}')
self.assertEqual(actual, expected)
self.assertNotIn(self.key.encode(), body)
self.assertNotIn(self.key, self.log.read_text())
self.assertNotIn("Location", headers)
upstream.assert_called_once()
response.read.assert_not_called()
response.read1.assert_not_called()
def test_context_exit_interrupts_active_transfer_and_closes_listener(self):
connection, response = self.upstream()
connected = threading.Event()
stopped = threading.Event()
connection.sock.shutdown.side_effect = lambda how: stopped.set()
def waiting_response():
connected.set()
if not stopped.wait(5):
raise RuntimeError("proxy did not interrupt the upstream socket")
raise OSError("closed")
connection.getresponse.side_effect = waiting_response
errors = []
def call(proxy):
try:
self.request(proxy, b'{"model":"glm-5.3"}')
except (OSError, http.client.HTTPException) as error:
errors.append(type(error).__name__)
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection):
with self.proxy() as proxy:
worker = threading.Thread(target=call, args=(proxy,))
worker.start()
self.assertTrue(connected.wait(5))
worker.join(5)
self.assertFalse(worker.is_alive())
self.assertTrue(stopped.is_set())
self.assertEqual(len(self.log.read_text().splitlines()), 1)
with self.assertRaises(OSError):
self.request(proxy)
def test_server_start_failure_cleans_up_without_waiting_for_an_unstarted_thread(self):
thread = Mock()
thread.start.side_effect = RuntimeError("thread unavailable")
thread.is_alive.return_value = False
proxy = self.proxy()
with patch("rollout.model_proxy.threading.Thread", return_value=thread):
with self.assertRaises(RuntimeError):
proxy.__enter__()
self.assertIsNone(proxy._key)
self.assertIsNone(proxy._log)
self.assertEqual(proxy._server.socket.fileno(), -1)
thread.join.assert_not_called()
if __name__ == "__main__":
unittest.main()