import hashlib import http.client import ipaddress import json from pathlib import Path import tempfile import threading import unittest from unittest.mock import Mock, patch from urllib.parse import urlsplit from rollout.model_proxy import MAX_BODY_BYTES, ModelProxy, MODEL, QWEN_MODEL, UPSTREAM_IDLE_SECONDS class ModelProxyTests(unittest.TestCase): def setUp(self): self.client_type = http.client.HTTPConnection self.temporary = tempfile.TemporaryDirectory() self.addCleanup(self.temporary.cleanup) self.root = Path(self.temporary.name).resolve() self.key = "test-provider-credential-do-not-forward" self.credential = self.root / "credential.json" self.credential.write_text(json.dumps({"model_api_key": self.key})) self.credential.chmod(0o600) self.log = self.root / "proxy.jsonl" def proxy(self, model=MODEL): return ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log, model=model) def request(self, proxy, body=None, method="POST", path="/v1/chat/completions", headers=None): endpoint = urlsplit(proxy.base_url) client = http.client.HTTPConnection(endpoint.hostname, endpoint.port, timeout=5) try: client.request(method, path, body=body, headers=headers or {}) response = client.getresponse() return response.status, dict(response.getheaders()), response.read() finally: client.close() def upstream(self, status=200, chunks=None, content_type="application/json"): response = Mock(status=status) response.getheader.return_value = content_type response.read1.side_effect = (chunks or [b'{"choices":[]}']) + [b""] connection = Mock() connection.getresponse.return_value = response return connection, response def test_credential_and_bind_trust_boundaries(self): self.credential.chmod(0o640) with self.assertRaises(ValueError): self.proxy() self.credential.chmod(0o600) linked = self.root / "linked.json" linked.symlink_to(self.credential) with self.assertRaises(ValueError): ModelProxy(linked, "127.0.0.1", "127.0.0.0/8", self.log) self.credential.write_text('{"model_api_key":"bad\\nheader"}') with self.assertRaises(ValueError): self.proxy() for bind, network in (("0.0.0.0", "0.0.0.0/8"), ("8.8.8.8", "8.0.0.0/8"), ("172.19.0.1", "172.18.0.0/16")): with self.assertRaises(ValueError): ModelProxy(self.credential, bind, network, self.log) def test_source_model_path_method_and_body_are_rejected_without_upstream(self): with patch("rollout.model_proxy.http.client.HTTPSConnection") as upstream: with self.proxy() as proxy: cases = [(b'{"model":"other"}', "POST", "/v1/chat/completions", 400), (b'{"model":"glm-5.3","model":"other"}', "POST", "/v1/chat/completions", 400), (b'[]', "POST", "/v1/chat/completions", 400), (b'bad json', "POST", "/v1/chat/completions", 400), (b'{"model":"glm-5.3"}', "POST", "/other", 404), (None, "GET", "/v1/chat/completions", 405)] for body, method, path, status in cases: self.assertEqual(self.request(proxy, body, method, path)[0], status) self.assertEqual(self.request(proxy, b"", headers={"Content-Length": str(MAX_BODY_BYTES + 1)})[0], 413) self.assertEqual(self.request(proxy, b"", headers={"Transfer-Encoding": "chunked"})[0], 400) proxy.network = ipaddress.ip_network("127.0.0.2/32") self.assertEqual(self.request(proxy, b'{"model":"glm-5.3"}')[0], 403) upstream.assert_not_called() def test_fixed_upstream_authentication_and_json_transfer(self): connection, response = self.upstream() body = b'{"model":"glm-5.3","messages":[]}' with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream: with self.proxy() as proxy: status, headers, result = self.request(proxy, body, headers={"Authorization": "Bearer dummy"}) self.assertIsNone(proxy._key) self.assertEqual((status, result), (200, b'{"choices":[]}')) self.assertEqual(headers["Connection"], "close") upstream.assert_called_once_with("open.bigmodel.cn", timeout=30) connection.request.assert_called_once_with("POST", "/api/coding/paas/v4/chat/completions", body=body, headers={"Authorization": "Bearer " + self.key, "Content-Type": "application/json", "Connection": "close"}) connection.close.assert_called_once() response.read1.assert_called_with(64 * 1024) facts = json.loads(self.log.read_text()) self.assertEqual(facts["request_body_sha256"], hashlib.sha256(body).hexdigest()) self.assertEqual(facts["status"], 200) self.assertEqual(facts["request_bytes"], len(body)) self.assertEqual(facts["response_bytes"], len(result)) self.assertNotIn(self.key, self.log.read_text()) self.assertNotIn("Authorization", self.log.read_text()) self.assertEqual(self.log.stat().st_mode & 0o777, 0o600) def test_sse_transfer_preserves_chunks(self): chunks = [b'data: {"choices":[]}\n\n', b'data: [DONE]\n\n'] connection, response = self.upstream(chunks=chunks, content_type="text/event-stream; charset=utf-8") with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection): with self.proxy() as proxy: status, headers, body = self.request(proxy, b'{"model":"glm-5.3","stream":true}') self.assertEqual(status, 200) self.assertEqual(headers["Content-Type"], "text/event-stream; charset=utf-8") self.assertEqual(body, b"".join(chunks)) self.assertEqual(response.read1.call_count, 3) self.assertEqual(UPSTREAM_IDLE_SECONDS, 900) connection.sock.settimeout.assert_called_once_with(UPSTREAM_IDLE_SECONDS) def test_qwen_responses_preserves_split_stream_and_confirms_terminal(self): terminal = json.dumps(dict(type="response.completed", response=dict(model=QWEN_MODEL, status="completed"))) chunks = [b'data: {"type":"response.created"}\n\nda', ("ta: " + terminal[:30]).encode(), (terminal[30:] + "\n\n").encode()] connection, response = self.upstream(chunks=chunks, content_type="text/event-stream") body = json.dumps(dict(model=QWEN_MODEL, input="fix", stream=True)).encode() with patch("rollout.model_proxy.http.client.HTTPConnection", return_value=connection) as upstream: # Keep the caller's HTTPConnection real while only mocking the fixed upstream. upstream.side_effect = lambda host, *args, **kwargs: connection if host == "114.111.18.115:18080" else self.client_type(host, *args, **kwargs) with self.proxy(QWEN_MODEL) as proxy: status, headers, result = self.request(proxy, body, path="/v1/responses") self.assertEqual((status, result), (200, b"".join(chunks))) connection.sock.settimeout.assert_called_once_with(UPSTREAM_IDLE_SECONDS) connection.request.assert_called_once_with("POST", "/v1/responses", body=body, headers={"Authorization": "Bearer " + self.key, "Content-Type": "application/json", "Connection": "close"}) facts = json.loads(self.log.read_text()) self.assertEqual(facts["responses_terminal"], "response.completed") self.assertEqual(facts["responses_terminal_count"], 1) self.assertEqual(facts["response_model"], QWEN_MODEL) self.assertEqual(facts["response_status"], "completed") self.assertTrue(facts["responses_stream_valid"] and facts["responses_stream_finished"]) self.assertNotIn(self.key, self.log.read_text()) def test_qwen_rejects_other_model_chat_path_or_nonstream_before_upstream(self): with patch("rollout.model_proxy.http.client.HTTPConnection") as upstream: upstream.side_effect = lambda host, *args, **kwargs: self.client_type(host, *args, **kwargs) with self.proxy(QWEN_MODEL) as proxy: for payload, path, expected in ((dict(model=MODEL, stream=True), "/v1/responses", 400), (dict(model=QWEN_MODEL, stream=True), "/v1/chat/completions", 404), (dict(model=QWEN_MODEL, stream=False), "/v1/responses", 400)): self.assertEqual(self.request(proxy, json.dumps(payload).encode(), path=path)[0], expected) self.assertFalse(any(call.args[0] == "114.111.18.115:18080" for call in upstream.call_args_list)) with self.assertRaises(ValueError): self.proxy("unknown") def test_provider_errors_are_sanitized_and_redirect_is_not_followed(self): for status, expected in ((401, 401), (429, 429), (302, 502)): self.log.unlink(missing_ok=True) connection, response = self.upstream(status=status) response.reason = self.key response.read.return_value = self.key.encode() with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream, \ patch("rollout.model_proxy.time.sleep") as sleep: with self.proxy() as proxy: actual, headers, body = self.request(proxy, b'{"model":"glm-5.3"}') self.assertEqual(actual, expected) self.assertNotIn(self.key.encode(), body) self.assertNotIn(self.key, self.log.read_text()) self.assertNotIn("Location", headers) self.assertEqual(upstream.call_count, 3 if status == 429 else 1) self.assertEqual(sleep.call_count, 2 if status == 429 else 0) response.read.assert_not_called() response.read1.assert_not_called() def test_transient_failures_before_the_response_are_retried_and_recorded(self): body = b'{"model":"glm-5.3","messages":[]}' broken = Mock() broken.connect.side_effect = ConnectionResetError("reset") for attempts, expected, statuses in (([self.upstream(500)[0], self.upstream()[0]], 200, [500, 200]), ([broken, self.upstream()[0]], 200, ["connection_error", 200]), ([self.upstream(500)[0] for _ in range(3)], 500, [500, 500, 500])): self.log.unlink(missing_ok=True) with patch("rollout.model_proxy.http.client.HTTPSConnection", side_effect=attempts), \ patch("rollout.model_proxy.time.sleep") as sleep: with self.proxy() as proxy: status, _, result = self.request(proxy, body) facts = json.loads(self.log.read_text()) self.assertEqual((status, facts["status"], facts["upstream_statuses"]), (expected, expected, statuses)) self.assertEqual([call.args[0] for call in sleep.call_args_list], [2, 5][:len(statuses) - 1]) if expected == 200: self.assertEqual(result, b'{"choices":[]}') def test_context_exit_interrupts_active_transfer_and_closes_listener(self): connection, response = self.upstream() connected = threading.Event() stopped = threading.Event() connection.sock.shutdown.side_effect = lambda how: stopped.set() def waiting_response(): connected.set() if not stopped.wait(5): raise RuntimeError("proxy did not interrupt the upstream socket") raise OSError("closed") connection.getresponse.side_effect = waiting_response errors = [] def call(proxy): try: self.request(proxy, b'{"model":"glm-5.3"}') except (OSError, http.client.HTTPException) as error: errors.append(type(error).__name__) with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection): with self.proxy() as proxy: worker = threading.Thread(target=call, args=(proxy,)) worker.start() self.assertTrue(connected.wait(5)) worker.join(5) self.assertFalse(worker.is_alive()) self.assertTrue(stopped.is_set()) self.assertEqual(len(self.log.read_text().splitlines()), 1) with self.assertRaises(OSError): self.request(proxy) def test_server_start_failure_cleans_up_without_waiting_for_an_unstarted_thread(self): thread = Mock() thread.start.side_effect = RuntimeError("thread unavailable") thread.is_alive.return_value = False proxy = self.proxy() with patch("rollout.model_proxy.threading.Thread", return_value=thread): with self.assertRaises(RuntimeError): proxy.__enter__() self.assertIsNone(proxy._key) self.assertIsNone(proxy._log) self.assertEqual(proxy._server.socket.fileno(), -1) thread.join.assert_not_called() if __name__ == "__main__": unittest.main()