import hashlib import http.client from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import ipaddress import json import os from pathlib import Path import shutil import subprocess import tempfile import threading import time import unittest from unittest.mock import Mock, patch from urllib.parse import urlsplit from rollout import model_proxy from rollout.model_proxy import MAX_BODY_BYTES, ModelProxy, MODEL, MODELS, QWEN_MODEL, UPSTREAM_IDLE_SECONDS from rollout.model_proxy import _credential ROOT = Path(__file__).resolve().parents[1] FAST = dict(STALL_SECONDS=0.4, KEEPALIVE_SECONDS=0.1, RETRY_DELAYS=(0, 0)) def sse(*events): return b"".join(b"data: " + json.dumps(event).encode() + b"\n\n" for event in events) PREAMBLE = sse(dict(type="response.created"), dict(type="response.in_progress")) REPLY = sse(dict(type="response.output_text.delta", delta="ok"), dict(type="response.completed", response=dict(model=QWEN_MODEL, status="completed"))) class ModelProxyTests(unittest.TestCase): def setUp(self): self.client_type = http.client.HTTPConnection self.temporary = tempfile.TemporaryDirectory() self.addCleanup(self.temporary.cleanup) self.root = Path(self.temporary.name).resolve() self.key = "test-provider-credential-do-not-forward" self.credential = self.root / "credential.json" self.credential.write_text(json.dumps({"model_api_key": self.key})) self.credential.chmod(0o600) self.log = self.root / "proxy.jsonl" def proxy(self, model=MODEL): return ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log, model=model) def request(self, proxy, body=None, method="POST", path="/v1/chat/completions", headers=None): endpoint = urlsplit(proxy.base_url) client = http.client.HTTPConnection(endpoint.hostname, endpoint.port, timeout=5) try: client.request(method, path, body=body, headers=headers or {}) response = client.getresponse() return response.status, dict(response.getheaders()), response.read() finally: client.close() def upstream(self, status=200, chunks=None, content_type="application/json"): response = Mock(status=status) response.getheader.return_value = content_type response.read1.side_effect = (chunks or [b'{"choices":[]}']) + [b""] connection = Mock() connection.getresponse.return_value = response return connection, response def test_credential_and_bind_trust_boundaries(self): self.credential.chmod(0o640) with self.assertRaises(ValueError): self.proxy() self.credential.chmod(0o600) linked = self.root / "linked.json" linked.symlink_to(self.credential) with self.assertRaises(ValueError): ModelProxy(linked, "127.0.0.1", "127.0.0.0/8", self.log) self.credential.write_text('{"model_api_key":"bad\\nheader"}') with self.assertRaises(ValueError): self.proxy() for bind, network in (("0.0.0.0", "0.0.0.0/8"), ("8.8.8.8", "8.0.0.0/8"), ("172.19.0.1", "172.18.0.0/16")): with self.assertRaises(ValueError): ModelProxy(self.credential, bind, network, self.log) def test_source_model_path_method_and_body_are_rejected_without_upstream(self): with patch("rollout.model_proxy.http.client.HTTPSConnection") as upstream: with self.proxy() as proxy: cases = [(b'{"model":"other"}', "POST", "/v1/chat/completions", 400), (b'{"model":"glm-5.3","model":"other"}', "POST", "/v1/chat/completions", 400), (b'[]', "POST", "/v1/chat/completions", 400), (b'bad json', "POST", "/v1/chat/completions", 400), (b'{"model":"glm-5.3"}', "POST", "/other", 404), (None, "GET", "/v1/chat/completions", 405)] for body, method, path, status in cases: self.assertEqual(self.request(proxy, body, method, path)[0], status) self.assertEqual(self.request(proxy, b"", headers={"Content-Length": str(MAX_BODY_BYTES + 1)})[0], 413) self.assertEqual(self.request(proxy, b"", headers={"Transfer-Encoding": "chunked"})[0], 400) proxy.network = ipaddress.ip_network("127.0.0.2/32") self.assertEqual(self.request(proxy, b'{"model":"glm-5.3"}')[0], 403) upstream.assert_not_called() def test_token_proxy_rejects_missing_or_wrong_bearer_before_upstream(self): token = "a" * 64 with self.assertRaises(ValueError): ModelProxy(self.credential, "127.0.0.1", "10.0.0.0/8", self.log) with self.assertRaises(ValueError): ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log, token="short") connection, _ = self.upstream() body = b'{"model":"glm-5.3","messages":[]}' with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream: with ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log, token=token) as proxy: self.assertEqual(self.request(proxy, body)[0], 401) self.assertEqual(self.request(proxy, body, headers={"Authorization": "Bearer " + "b" * 64})[0], 401) upstream.assert_not_called() self.assertEqual(self.request(proxy, body, headers={"Authorization": "Bearer " + token})[0], 200) self.assertEqual(self.request(proxy, None, "GET", "/health", headers={"Authorization": "Bearer " + token})[0], 204) self.assertEqual(self.request(proxy, None, "GET", "/health")[0], 401) self.assertEqual([json.loads(line)["status"] for line in self.log.read_text().splitlines()], [401, 401, 200, 401]) self.assertNotIn(token, self.log.read_text()) def test_fixed_upstream_authentication_and_json_transfer(self): connection, response = self.upstream() body = b'{"model":"glm-5.3","messages":[]}' with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream: with self.proxy() as proxy: status, headers, result = self.request(proxy, body, headers={"Authorization": "Bearer dummy"}) self.assertIsNone(proxy._key) self.assertEqual((status, result), (200, b'{"choices":[]}')) self.assertEqual(headers["Connection"], "close") upstream.assert_called_once_with("open.bigmodel.cn", timeout=30) connection.request.assert_called_once_with("POST", "/api/coding/paas/v4/chat/completions", body=body, headers={"Authorization": "Bearer " + self.key, "Content-Type": "application/json", "Connection": "close"}) connection.close.assert_called_once() response.read1.assert_called_with(64 * 1024) facts = json.loads(self.log.read_text()) self.assertEqual(facts["request_body_sha256"], hashlib.sha256(body).hexdigest()) self.assertEqual(facts["status"], 200) self.assertEqual(facts["request_bytes"], len(body)) self.assertEqual(facts["response_bytes"], len(result)) self.assertNotIn(self.key, self.log.read_text()) self.assertNotIn("Authorization", self.log.read_text()) self.assertEqual(self.log.stat().st_mode & 0o777, 0o600) def test_sse_transfer_preserves_chunks(self): chunks = [b'data: {"choices":[]}\n\n', b'data: [DONE]\n\n'] connection, response = self.upstream(chunks=chunks, content_type="text/event-stream; charset=utf-8") with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection): with self.proxy() as proxy: status, headers, body = self.request(proxy, b'{"model":"glm-5.3","stream":true}') self.assertEqual(status, 200) self.assertEqual(headers["Content-Type"], "text/event-stream; charset=utf-8") self.assertEqual(body, b"".join(chunks)) self.assertEqual(response.read1.call_count, 3) self.assertEqual(UPSTREAM_IDLE_SECONDS, 900) connection.sock.settimeout.assert_called_once_with(UPSTREAM_IDLE_SECONDS) def test_qwen_responses_preserves_split_stream_and_confirms_terminal(self): terminal = json.dumps(dict(type="response.completed", response=dict(model=QWEN_MODEL, status="completed"))) chunks = [b'data: {"type":"response.created"}\n\nda', ("ta: " + terminal[:30]).encode(), (terminal[30:] + "\n\n").encode()] connection, response = self.upstream(chunks=chunks, content_type="text/event-stream") body = json.dumps(dict(model=QWEN_MODEL, input="fix", stream=True)).encode() with patch("rollout.model_proxy.http.client.HTTPConnection", return_value=connection) as upstream: # Keep the caller's HTTPConnection real while only mocking the fixed upstream. upstream.side_effect = lambda host, *args, **kwargs: connection if host == "114.111.18.115:18080" else self.client_type(host, *args, **kwargs) with self.proxy(QWEN_MODEL) as proxy: status, headers, result = self.request(proxy, body, path="/v1/responses") self.assertEqual((status, result), (200, b"".join(chunks))) connection.sock.settimeout.assert_called_once_with(UPSTREAM_IDLE_SECONDS) connection.request.assert_called_once_with("POST", "/v1/responses", body=body, headers={"Authorization": "Bearer " + self.key, "Content-Type": "application/json", "Connection": "close"}) facts = json.loads(self.log.read_text()) self.assertEqual(facts["responses_terminal"], "response.completed") self.assertEqual(facts["responses_terminal_count"], 1) self.assertEqual(facts["response_model"], QWEN_MODEL) self.assertEqual(facts["response_status"], "completed") self.assertTrue(facts["responses_stream_valid"] and facts["responses_stream_finished"]) self.assertNotIn(self.key, self.log.read_text()) def qwen(self, *streams): """Run one Qwen request against scripted upstream attempts; a stream step is bytes, a pause or 'hang'.""" released = threading.Event() connections = [] for steps in streams: steps = iter(list(steps) + [b""]) def read1(size, steps=steps): step = next(steps) while isinstance(step, float): time.sleep(step) step = next(steps) if step == "hang": released.wait(5) raise OSError("closed") return step connection, response = self.upstream(content_type="text/event-stream") response.read1.side_effect = read1 connection.sock.shutdown.side_effect = lambda how: released.set() connections.append(connection) attempts = iter(connections) body = json.dumps(dict(model=QWEN_MODEL, input="fix", stream=True)).encode() with patch("rollout.model_proxy.http.client.HTTPConnection") as upstream, \ patch.multiple(model_proxy, **FAST): upstream.side_effect = lambda host, *args, **kwargs: ( next(attempts) if host == "114.111.18.115:18080" else self.client_type(host, *args, **kwargs)) with self.proxy(QWEN_MODEL) as proxy: try: result = self.request(proxy, body, path="/v1/responses")[2] except (OSError, http.client.HTTPException): result = None return result, json.loads(self.log.read_text()) def test_stall_before_generation_is_retried_and_pi_sees_one_response(self): started = time.monotonic() result, facts = self.qwen([PREAMBLE, "hang"], [PREAMBLE, REPLY]) self.assertLess(time.monotonic() - started, 3) # The stall timer fired, not the 5 s test fallback. self.assertEqual(result.replace(b": keepalive\n\n", b""), PREAMBLE + REPLY) self.assertIn(b": keepalive\n\n", result) self.assertEqual(facts["upstream_statuses"], ["stalled", 200]) self.assertEqual((facts["status"], facts["responses_terminal"], facts["responses_terminal_count"]), (200, "response.completed", 1)) self.assertTrue(facts["responses_stream_finished"]) def test_slow_first_event_gets_keepalives_and_no_retry(self): result, facts = self.qwen([PREAMBLE, 0.3, REPLY]) self.assertEqual(result.replace(b": keepalive\n\n", b""), PREAMBLE + REPLY) self.assertIn(b": keepalive\n\n", result) self.assertEqual(facts["upstream_statuses"], [200]) self.assertGreaterEqual(facts["first_event_seconds"], 0.3) def test_generation_already_forwarded_is_never_retried(self): started = sse(dict(type="response.output_text.delta", delta="o")) result, facts = self.qwen([PREAMBLE, started, "hang"], [PREAMBLE, REPLY]) self.assertEqual(facts["upstream_statuses"], [200]) self.assertIsNone(facts["responses_terminal"]) self.assertFalse(facts["responses_stream_finished"]) def test_every_attempt_stalling_ends_without_forwarding_a_preamble(self): result, facts = self.qwen([PREAMBLE, "hang"], [PREAMBLE, "hang"], [PREAMBLE, "hang"]) self.assertEqual(facts["upstream_statuses"], ["stalled"] * 3) self.assertNotIn(b"response.created", result or b"") self.assertIsNone(facts["responses_terminal"]) @unittest.skipUnless(shutil.which("node") and (ROOT / "agent/node_modules/undici").is_dir(), "needs Node and agent/node_modules with the pinned Pi SDK") def test_real_pi_sdk_survives_a_stall_only_because_of_keepalives(self): """Pi's fetch gives up after 0.6 s here; the proxy waits 1 s, retries, and the retry takes 0.9 s.""" posts = [] class Upstream(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" def log_message(self, *args): pass def do_POST(self): self.rfile.read(int(self.headers["Content-Length"])) posts.append(time.monotonic()) self.send_response(200) self.send_header("Content-Type", "text/event-stream") self.send_header("Connection", "close") self.end_headers() self.wfile.write(sse(dict(type="response.created", response=dict(id="r")), dict(type="response.in_progress", response=dict(id="r")))) self.wfile.flush() time.sleep(3 if len(posts) == 1 else 0.9) item = dict(type="message", id="m", role="assistant", status="completed", content=[dict(type="output_text", text="Done.", annotations=[])]) self.wfile.write(sse( dict(type="response.output_item.added", output_index=0, item=dict(item, content=[])), dict(type="response.output_text.delta", item_id="m", output_index=0, content_index=0, delta="Done."), dict(type="response.output_item.done", output_index=0, item=item), dict(type="response.completed", response=dict(id="r", model=QWEN_MODEL, status="completed", usage=dict(input_tokens=3, output_tokens=1, total_tokens=4))))) script = """ import {Agent, setGlobalDispatcher} from 'undici'; import fs from 'node:fs'; import {run, loadSdk} from './pi.mjs'; setGlobalDispatcher(new Agent({headersTimeout: 600, bodyTimeout: 600})); const [baseUrl, directory] = process.argv.slice(1); const summary = await run('Say done.', {cwd: directory, traceFile: directory + '/native.jsonl', baseUrl, modelId: 'qwen3.8-27b-nvfp4', maxTokens: 64, mirror: () => {}}, await loadSdk()); const last = fs.readFileSync(directory + '/native.jsonl', 'utf8').trim().split('\\n').map(JSON.parse) .filter(event => event.type === 'message_end').at(-1); console.log(JSON.stringify({complete: summary.complete, error: last?.message?.errorMessage ?? null})); """ upstream = ThreadingHTTPServer(("127.0.0.1", 0), Upstream) threading.Thread(target=upstream.serve_forever, daemon=True).start() self.addCleanup(upstream.server_close) self.addCleanup(upstream.shutdown) outcomes = [] for keepalive in (0.2, 30): posts.clear() self.log.unlink(missing_ok=True) work = Path(tempfile.mkdtemp(dir=self.root)) route = dict(MODELS[QWEN_MODEL], upstream="127.0.0.1:%d" % upstream.server_port) with patch.dict(MODELS, {QWEN_MODEL: route}), patch.multiple( model_proxy, STALL_SECONDS=1, KEEPALIVE_SECONDS=keepalive, RETRY_DELAYS=(0, 0)): with self.proxy(QWEN_MODEL) as proxy: done = subprocess.run(["node", "--input-type=module", "-e", script, proxy.base_url, str(work)], cwd=ROOT / "agent", capture_output=True, text=True, timeout=60, env=dict(os.environ, NODE_NO_WARNINGS="1")) self.assertEqual(done.returncode, 0, done.stderr) outcomes.append((json.loads(done.stdout.splitlines()[-1]), len(posts), json.loads(self.log.read_text().splitlines()[0])["upstream_statuses"])) (kept, kept_posts, kept_statuses), (bare, _, _) = outcomes self.assertEqual(kept, dict(complete=True, error=None)) self.assertEqual((kept_posts, kept_statuses), (2, ["stalled", 200])) # Control: without keepalives the same stall kills the session the way batch 6 lost 26 attempts. self.assertEqual(bare, dict(complete=False, error="terminated")) def test_qwen_rejects_other_model_chat_path_or_nonstream_before_upstream(self): with patch("rollout.model_proxy.http.client.HTTPConnection") as upstream: upstream.side_effect = lambda host, *args, **kwargs: self.client_type(host, *args, **kwargs) with self.proxy(QWEN_MODEL) as proxy: for payload, path, expected in ((dict(model=MODEL, stream=True), "/v1/responses", 400), (dict(model=QWEN_MODEL, stream=True), "/v1/chat/completions", 404), (dict(model=QWEN_MODEL, stream=False), "/v1/responses", 400)): self.assertEqual(self.request(proxy, json.dumps(payload).encode(), path=path)[0], expected) self.assertFalse(any(call.args[0] == "114.111.18.115:18080" for call in upstream.call_args_list)) with self.assertRaises(ValueError): self.proxy("unknown") def test_provider_errors_are_sanitized_and_redirect_is_not_followed(self): for status, expected in ((401, 401), (429, 429), (302, 502)): self.log.unlink(missing_ok=True) connection, response = self.upstream(status=status) response.reason = self.key response.read.return_value = self.key.encode() with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream, \ patch("rollout.model_proxy.time.sleep") as sleep: with self.proxy() as proxy: actual, headers, body = self.request(proxy, b'{"model":"glm-5.3"}') self.assertEqual(actual, expected) self.assertNotIn(self.key.encode(), body) self.assertNotIn(self.key, self.log.read_text()) self.assertNotIn("Location", headers) self.assertEqual(upstream.call_count, 3 if status == 429 else 1) self.assertEqual(sleep.call_count, 2 if status == 429 else 0) response.read.assert_not_called() response.read1.assert_not_called() def test_transient_failures_before_the_response_are_retried_and_recorded(self): body = b'{"model":"glm-5.3","messages":[]}' broken = Mock() broken.connect.side_effect = ConnectionResetError("reset") for attempts, expected, statuses in (([self.upstream(500)[0], self.upstream()[0]], 200, [500, 200]), ([broken, self.upstream()[0]], 200, ["connection_error", 200]), ([self.upstream(500)[0] for _ in range(3)], 500, [500, 500, 500])): self.log.unlink(missing_ok=True) with patch("rollout.model_proxy.http.client.HTTPSConnection", side_effect=attempts), \ patch("rollout.model_proxy.time.sleep") as sleep: with self.proxy() as proxy: status, _, result = self.request(proxy, body) facts = json.loads(self.log.read_text()) self.assertEqual((status, facts["status"], facts["upstream_statuses"]), (expected, expected, statuses)) self.assertEqual([call.args[0] for call in sleep.call_args_list], [2, 5][:len(statuses) - 1]) if expected == 200: self.assertEqual(result, b'{"choices":[]}') def test_context_exit_interrupts_active_transfer_and_closes_listener(self): connection, response = self.upstream() connected = threading.Event() stopped = threading.Event() connection.sock.shutdown.side_effect = lambda how: stopped.set() def waiting_response(): connected.set() if not stopped.wait(5): raise RuntimeError("proxy did not interrupt the upstream socket") raise OSError("closed") connection.getresponse.side_effect = waiting_response errors = [] def call(proxy): try: self.request(proxy, b'{"model":"glm-5.3"}') except (OSError, http.client.HTTPException) as error: errors.append(type(error).__name__) with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection): with self.proxy() as proxy: worker = threading.Thread(target=call, args=(proxy,)) worker.start() self.assertTrue(connected.wait(5)) worker.join(5) self.assertFalse(worker.is_alive()) self.assertTrue(stopped.is_set()) self.assertEqual(len(self.log.read_text().splitlines()), 1) with self.assertRaises(OSError): self.request(proxy) def test_server_start_failure_cleans_up_without_waiting_for_an_unstarted_thread(self): thread = Mock() thread.start.side_effect = RuntimeError("thread unavailable") thread.is_alive.return_value = False proxy = self.proxy() with patch("rollout.model_proxy.threading.Thread", return_value=thread): with self.assertRaises(RuntimeError): proxy.__enter__() self.assertIsNone(proxy._key) self.assertIsNone(proxy._log) self.assertEqual(proxy._server.socket.fileno(), -1) thread.join.assert_not_called() if __name__ == "__main__": unittest.main() class CredentialInstanceTests(unittest.TestCase): def write(self, data): folder = os.path.realpath(tempfile.mkdtemp()) # macOS /var is a symlink, which credentials refuse self.addCleanup(shutil.rmtree, folder) path = Path(folder, "key.json") path.write_text(json.dumps(data)) path.chmod(0o600) return path def test_a_credential_may_pick_only_a_listed_instance_of_its_model(self): qwen, glm = MODELS[QWEN_MODEL], MODELS["glm-5.3"] self.assertEqual(_credential(self.write(dict(model_api_key="k")), qwen), ("k", "114.111.18.115:18080")) self.assertEqual(_credential(self.write(dict(model_api_key="k", upstream_port=18083)), qwen), ("k", "114.111.18.115:18083")) for port, route in ((18084, qwen), ("18081", qwen), (18081, glm)): with self.assertRaises(ValueError): _credential(self.write(dict(model_api_key="k", upstream_port=port)), route)