data-gen-ebd/tests/test_model_proxy.py
amos c93aeb8fd8 Let a credential pick one of the four Qwen instances
The Qwen service runs four identical instances on 114.111.18.115:18080-18083. A
credential may now carry upstream_port, limited to the ports the route lists, so
rollout.batch's least-busy key choice spreads attempts across instances. The broker
log records the upstream each request used.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-30 14:58:12 +08:00

435 lines
24 KiB
Python

import hashlib
import http.client
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import ipaddress
import json
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import threading
import time
import unittest
from unittest.mock import Mock, patch
from urllib.parse import urlsplit
from rollout import model_proxy
from rollout.model_proxy import MAX_BODY_BYTES, ModelProxy, MODEL, MODELS, QWEN_MODEL, UPSTREAM_IDLE_SECONDS
from rollout.model_proxy import _credential
ROOT = Path(__file__).resolve().parents[1]
FAST = dict(STALL_SECONDS=0.4, KEEPALIVE_SECONDS=0.1, RETRY_DELAYS=(0, 0))
def sse(*events):
return b"".join(b"data: " + json.dumps(event).encode() + b"\n\n" for event in events)
PREAMBLE = sse(dict(type="response.created"), dict(type="response.in_progress"))
REPLY = sse(dict(type="response.output_text.delta", delta="ok"),
dict(type="response.completed", response=dict(model=QWEN_MODEL, status="completed")))
class ModelProxyTests(unittest.TestCase):
def setUp(self):
self.client_type = http.client.HTTPConnection
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name).resolve()
self.key = "test-provider-credential-do-not-forward"
self.credential = self.root / "credential.json"
self.credential.write_text(json.dumps({"model_api_key": self.key}))
self.credential.chmod(0o600)
self.log = self.root / "proxy.jsonl"
def proxy(self, model=MODEL):
return ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log, model=model)
def request(self, proxy, body=None, method="POST", path="/v1/chat/completions", headers=None):
endpoint = urlsplit(proxy.base_url)
client = http.client.HTTPConnection(endpoint.hostname, endpoint.port, timeout=5)
try:
client.request(method, path, body=body, headers=headers or {})
response = client.getresponse()
return response.status, dict(response.getheaders()), response.read()
finally:
client.close()
def upstream(self, status=200, chunks=None, content_type="application/json"):
response = Mock(status=status)
response.getheader.return_value = content_type
response.read1.side_effect = (chunks or [b'{"choices":[]}']) + [b""]
connection = Mock()
connection.getresponse.return_value = response
return connection, response
def test_credential_and_bind_trust_boundaries(self):
self.credential.chmod(0o640)
with self.assertRaises(ValueError):
self.proxy()
self.credential.chmod(0o600)
linked = self.root / "linked.json"
linked.symlink_to(self.credential)
with self.assertRaises(ValueError):
ModelProxy(linked, "127.0.0.1", "127.0.0.0/8", self.log)
self.credential.write_text('{"model_api_key":"bad\\nheader"}')
with self.assertRaises(ValueError):
self.proxy()
for bind, network in (("0.0.0.0", "0.0.0.0/8"), ("8.8.8.8", "8.0.0.0/8"),
("172.19.0.1", "172.18.0.0/16")):
with self.assertRaises(ValueError):
ModelProxy(self.credential, bind, network, self.log)
def test_source_model_path_method_and_body_are_rejected_without_upstream(self):
with patch("rollout.model_proxy.http.client.HTTPSConnection") as upstream:
with self.proxy() as proxy:
cases = [(b'{"model":"other"}', "POST", "/v1/chat/completions", 400),
(b'{"model":"glm-5.3","model":"other"}', "POST", "/v1/chat/completions", 400),
(b'[]', "POST", "/v1/chat/completions", 400),
(b'bad json', "POST", "/v1/chat/completions", 400),
(b'{"model":"glm-5.3"}', "POST", "/other", 404),
(None, "GET", "/v1/chat/completions", 405)]
for body, method, path, status in cases:
self.assertEqual(self.request(proxy, body, method, path)[0], status)
self.assertEqual(self.request(proxy, b"", headers={"Content-Length": str(MAX_BODY_BYTES + 1)})[0], 413)
self.assertEqual(self.request(proxy, b"", headers={"Transfer-Encoding": "chunked"})[0], 400)
proxy.network = ipaddress.ip_network("127.0.0.2/32")
self.assertEqual(self.request(proxy, b'{"model":"glm-5.3"}')[0], 403)
upstream.assert_not_called()
def test_token_proxy_rejects_missing_or_wrong_bearer_before_upstream(self):
token = "a" * 64
with self.assertRaises(ValueError):
ModelProxy(self.credential, "127.0.0.1", "10.0.0.0/8", self.log)
with self.assertRaises(ValueError):
ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log, token="short")
connection, _ = self.upstream()
body = b'{"model":"glm-5.3","messages":[]}'
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream:
with ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log, token=token) as proxy:
self.assertEqual(self.request(proxy, body)[0], 401)
self.assertEqual(self.request(proxy, body, headers={"Authorization": "Bearer " + "b" * 64})[0], 401)
upstream.assert_not_called()
self.assertEqual(self.request(proxy, body, headers={"Authorization": "Bearer " + token})[0], 200)
self.assertEqual(self.request(proxy, None, "GET", "/health",
headers={"Authorization": "Bearer " + token})[0], 204)
self.assertEqual(self.request(proxy, None, "GET", "/health")[0], 401)
self.assertEqual([json.loads(line)["status"] for line in self.log.read_text().splitlines()],
[401, 401, 200, 401])
self.assertNotIn(token, self.log.read_text())
def test_fixed_upstream_authentication_and_json_transfer(self):
connection, response = self.upstream()
body = b'{"model":"glm-5.3","messages":[]}'
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream:
with self.proxy() as proxy:
status, headers, result = self.request(proxy, body, headers={"Authorization": "Bearer dummy"})
self.assertIsNone(proxy._key)
self.assertEqual((status, result), (200, b'{"choices":[]}'))
self.assertEqual(headers["Connection"], "close")
upstream.assert_called_once_with("open.bigmodel.cn", timeout=30)
connection.request.assert_called_once_with("POST", "/api/coding/paas/v4/chat/completions", body=body,
headers={"Authorization": "Bearer " + self.key,
"Content-Type": "application/json", "Connection": "close"})
connection.close.assert_called_once()
response.read1.assert_called_with(64 * 1024)
facts = json.loads(self.log.read_text())
self.assertEqual(facts["request_body_sha256"], hashlib.sha256(body).hexdigest())
self.assertEqual(facts["status"], 200)
self.assertEqual(facts["request_bytes"], len(body))
self.assertEqual(facts["response_bytes"], len(result))
self.assertNotIn(self.key, self.log.read_text())
self.assertNotIn("Authorization", self.log.read_text())
self.assertEqual(self.log.stat().st_mode & 0o777, 0o600)
def test_sse_transfer_preserves_chunks(self):
chunks = [b'data: {"choices":[]}\n\n', b'data: [DONE]\n\n']
connection, response = self.upstream(chunks=chunks, content_type="text/event-stream; charset=utf-8")
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection):
with self.proxy() as proxy:
status, headers, body = self.request(proxy, b'{"model":"glm-5.3","stream":true}')
self.assertEqual(status, 200)
self.assertEqual(headers["Content-Type"], "text/event-stream; charset=utf-8")
self.assertEqual(body, b"".join(chunks))
self.assertEqual(response.read1.call_count, 3)
self.assertEqual(UPSTREAM_IDLE_SECONDS, 900)
connection.sock.settimeout.assert_called_once_with(UPSTREAM_IDLE_SECONDS)
def test_qwen_responses_preserves_split_stream_and_confirms_terminal(self):
terminal = json.dumps(dict(type="response.completed", response=dict(model=QWEN_MODEL, status="completed")))
chunks = [b'data: {"type":"response.created"}\n\nda',
("ta: " + terminal[:30]).encode(), (terminal[30:] + "\n\n").encode()]
connection, response = self.upstream(chunks=chunks, content_type="text/event-stream")
body = json.dumps(dict(model=QWEN_MODEL, input="fix", stream=True)).encode()
with patch("rollout.model_proxy.http.client.HTTPConnection", return_value=connection) as upstream:
# Keep the caller's HTTPConnection real while only mocking the fixed upstream.
upstream.side_effect = lambda host, *args, **kwargs: connection if host == "114.111.18.115:18080" else self.client_type(host, *args, **kwargs)
with self.proxy(QWEN_MODEL) as proxy:
status, headers, result = self.request(proxy, body, path="/v1/responses")
self.assertEqual((status, result), (200, b"".join(chunks)))
connection.sock.settimeout.assert_called_once_with(UPSTREAM_IDLE_SECONDS)
connection.request.assert_called_once_with("POST", "/v1/responses", body=body,
headers={"Authorization": "Bearer " + self.key, "Content-Type": "application/json", "Connection": "close"})
facts = json.loads(self.log.read_text())
self.assertEqual(facts["responses_terminal"], "response.completed")
self.assertEqual(facts["responses_terminal_count"], 1)
self.assertEqual(facts["response_model"], QWEN_MODEL)
self.assertEqual(facts["response_status"], "completed")
self.assertTrue(facts["responses_stream_valid"] and facts["responses_stream_finished"])
self.assertNotIn(self.key, self.log.read_text())
def qwen(self, *streams):
"""Run one Qwen request against scripted upstream attempts; a stream step is bytes, a pause or 'hang'."""
released = threading.Event()
connections = []
for steps in streams:
steps = iter(list(steps) + [b""])
def read1(size, steps=steps):
step = next(steps)
while isinstance(step, float):
time.sleep(step)
step = next(steps)
if step == "hang":
released.wait(5)
raise OSError("closed")
return step
connection, response = self.upstream(content_type="text/event-stream")
response.read1.side_effect = read1
connection.sock.shutdown.side_effect = lambda how: released.set()
connections.append(connection)
attempts = iter(connections)
body = json.dumps(dict(model=QWEN_MODEL, input="fix", stream=True)).encode()
with patch("rollout.model_proxy.http.client.HTTPConnection") as upstream, \
patch.multiple(model_proxy, **FAST):
upstream.side_effect = lambda host, *args, **kwargs: (
next(attempts) if host == "114.111.18.115:18080" else self.client_type(host, *args, **kwargs))
with self.proxy(QWEN_MODEL) as proxy:
try:
result = self.request(proxy, body, path="/v1/responses")[2]
except (OSError, http.client.HTTPException):
result = None
return result, json.loads(self.log.read_text())
def test_stall_before_generation_is_retried_and_pi_sees_one_response(self):
started = time.monotonic()
result, facts = self.qwen([PREAMBLE, "hang"], [PREAMBLE, REPLY])
self.assertLess(time.monotonic() - started, 3) # The stall timer fired, not the 5 s test fallback.
self.assertEqual(result.replace(b": keepalive\n\n", b""), PREAMBLE + REPLY)
self.assertIn(b": keepalive\n\n", result)
self.assertEqual(facts["upstream_statuses"], ["stalled", 200])
self.assertEqual((facts["status"], facts["responses_terminal"], facts["responses_terminal_count"]),
(200, "response.completed", 1))
self.assertTrue(facts["responses_stream_finished"])
def test_slow_first_event_gets_keepalives_and_no_retry(self):
result, facts = self.qwen([PREAMBLE, 0.3, REPLY])
self.assertEqual(result.replace(b": keepalive\n\n", b""), PREAMBLE + REPLY)
self.assertIn(b": keepalive\n\n", result)
self.assertEqual(facts["upstream_statuses"], [200])
self.assertGreaterEqual(facts["first_event_seconds"], 0.3)
def test_generation_already_forwarded_is_never_retried(self):
started = sse(dict(type="response.output_text.delta", delta="o"))
result, facts = self.qwen([PREAMBLE, started, "hang"], [PREAMBLE, REPLY])
self.assertEqual(facts["upstream_statuses"], [200])
self.assertIsNone(facts["responses_terminal"])
self.assertFalse(facts["responses_stream_finished"])
def test_every_attempt_stalling_ends_without_forwarding_a_preamble(self):
result, facts = self.qwen([PREAMBLE, "hang"], [PREAMBLE, "hang"], [PREAMBLE, "hang"])
self.assertEqual(facts["upstream_statuses"], ["stalled"] * 3)
self.assertNotIn(b"response.created", result or b"")
self.assertIsNone(facts["responses_terminal"])
@unittest.skipUnless(shutil.which("node") and (ROOT / "agent/node_modules/undici").is_dir(),
"needs Node and agent/node_modules with the pinned Pi SDK")
def test_real_pi_sdk_survives_a_stall_only_because_of_keepalives(self):
"""Pi's fetch gives up after 0.6 s here; the proxy waits 1 s, retries, and the retry takes 0.9 s."""
posts = []
class Upstream(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def log_message(self, *args):
pass
def do_POST(self):
self.rfile.read(int(self.headers["Content-Length"]))
posts.append(time.monotonic())
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Connection", "close")
self.end_headers()
self.wfile.write(sse(dict(type="response.created", response=dict(id="r")),
dict(type="response.in_progress", response=dict(id="r"))))
self.wfile.flush()
time.sleep(3 if len(posts) == 1 else 0.9)
item = dict(type="message", id="m", role="assistant", status="completed",
content=[dict(type="output_text", text="Done.", annotations=[])])
self.wfile.write(sse(
dict(type="response.output_item.added", output_index=0, item=dict(item, content=[])),
dict(type="response.output_text.delta", item_id="m", output_index=0, content_index=0,
delta="Done."),
dict(type="response.output_item.done", output_index=0, item=item),
dict(type="response.completed", response=dict(id="r", model=QWEN_MODEL, status="completed",
usage=dict(input_tokens=3, output_tokens=1,
total_tokens=4)))))
script = """
import {Agent, setGlobalDispatcher} from 'undici';
import fs from 'node:fs';
import {run, loadSdk} from './pi.mjs';
setGlobalDispatcher(new Agent({headersTimeout: 600, bodyTimeout: 600}));
const [baseUrl, directory] = process.argv.slice(1);
const summary = await run('Say done.', {cwd: directory, traceFile: directory + '/native.jsonl', baseUrl,
modelId: 'qwen3.8-27b-nvfp4', maxTokens: 64, mirror: () => {}}, await loadSdk());
const last = fs.readFileSync(directory + '/native.jsonl', 'utf8').trim().split('\\n').map(JSON.parse)
.filter(event => event.type === 'message_end').at(-1);
console.log(JSON.stringify({complete: summary.complete, error: last?.message?.errorMessage ?? null}));
"""
upstream = ThreadingHTTPServer(("127.0.0.1", 0), Upstream)
threading.Thread(target=upstream.serve_forever, daemon=True).start()
self.addCleanup(upstream.server_close)
self.addCleanup(upstream.shutdown)
outcomes = []
for keepalive in (0.2, 30):
posts.clear()
self.log.unlink(missing_ok=True)
work = Path(tempfile.mkdtemp(dir=self.root))
route = dict(MODELS[QWEN_MODEL], upstream="127.0.0.1:%d" % upstream.server_port)
with patch.dict(MODELS, {QWEN_MODEL: route}), patch.multiple(
model_proxy, STALL_SECONDS=1, KEEPALIVE_SECONDS=keepalive, RETRY_DELAYS=(0, 0)):
with self.proxy(QWEN_MODEL) as proxy:
done = subprocess.run(["node", "--input-type=module", "-e", script, proxy.base_url, str(work)],
cwd=ROOT / "agent", capture_output=True, text=True, timeout=60,
env=dict(os.environ, NODE_NO_WARNINGS="1"))
self.assertEqual(done.returncode, 0, done.stderr)
outcomes.append((json.loads(done.stdout.splitlines()[-1]), len(posts),
json.loads(self.log.read_text().splitlines()[0])["upstream_statuses"]))
(kept, kept_posts, kept_statuses), (bare, _, _) = outcomes
self.assertEqual(kept, dict(complete=True, error=None))
self.assertEqual((kept_posts, kept_statuses), (2, ["stalled", 200]))
# Control: without keepalives the same stall kills the session the way batch 6 lost 26 attempts.
self.assertEqual(bare, dict(complete=False, error="terminated"))
def test_qwen_rejects_other_model_chat_path_or_nonstream_before_upstream(self):
with patch("rollout.model_proxy.http.client.HTTPConnection") as upstream:
upstream.side_effect = lambda host, *args, **kwargs: self.client_type(host, *args, **kwargs)
with self.proxy(QWEN_MODEL) as proxy:
for payload, path, expected in ((dict(model=MODEL, stream=True), "/v1/responses", 400),
(dict(model=QWEN_MODEL, stream=True), "/v1/chat/completions", 404),
(dict(model=QWEN_MODEL, stream=False), "/v1/responses", 400)):
self.assertEqual(self.request(proxy, json.dumps(payload).encode(), path=path)[0], expected)
self.assertFalse(any(call.args[0] == "114.111.18.115:18080" for call in upstream.call_args_list))
with self.assertRaises(ValueError):
self.proxy("unknown")
def test_provider_errors_are_sanitized_and_redirect_is_not_followed(self):
for status, expected in ((401, 401), (429, 429), (302, 502)):
self.log.unlink(missing_ok=True)
connection, response = self.upstream(status=status)
response.reason = self.key
response.read.return_value = self.key.encode()
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream, \
patch("rollout.model_proxy.time.sleep") as sleep:
with self.proxy() as proxy:
actual, headers, body = self.request(proxy, b'{"model":"glm-5.3"}')
self.assertEqual(actual, expected)
self.assertNotIn(self.key.encode(), body)
self.assertNotIn(self.key, self.log.read_text())
self.assertNotIn("Location", headers)
self.assertEqual(upstream.call_count, 3 if status == 429 else 1)
self.assertEqual(sleep.call_count, 2 if status == 429 else 0)
response.read.assert_not_called()
response.read1.assert_not_called()
def test_transient_failures_before_the_response_are_retried_and_recorded(self):
body = b'{"model":"glm-5.3","messages":[]}'
broken = Mock()
broken.connect.side_effect = ConnectionResetError("reset")
for attempts, expected, statuses in (([self.upstream(500)[0], self.upstream()[0]], 200, [500, 200]),
([broken, self.upstream()[0]], 200, ["connection_error", 200]),
([self.upstream(500)[0] for _ in range(3)], 500, [500, 500, 500])):
self.log.unlink(missing_ok=True)
with patch("rollout.model_proxy.http.client.HTTPSConnection", side_effect=attempts), \
patch("rollout.model_proxy.time.sleep") as sleep:
with self.proxy() as proxy:
status, _, result = self.request(proxy, body)
facts = json.loads(self.log.read_text())
self.assertEqual((status, facts["status"], facts["upstream_statuses"]), (expected, expected, statuses))
self.assertEqual([call.args[0] for call in sleep.call_args_list], [2, 5][:len(statuses) - 1])
if expected == 200:
self.assertEqual(result, b'{"choices":[]}')
def test_context_exit_interrupts_active_transfer_and_closes_listener(self):
connection, response = self.upstream()
connected = threading.Event()
stopped = threading.Event()
connection.sock.shutdown.side_effect = lambda how: stopped.set()
def waiting_response():
connected.set()
if not stopped.wait(5):
raise RuntimeError("proxy did not interrupt the upstream socket")
raise OSError("closed")
connection.getresponse.side_effect = waiting_response
errors = []
def call(proxy):
try:
self.request(proxy, b'{"model":"glm-5.3"}')
except (OSError, http.client.HTTPException) as error:
errors.append(type(error).__name__)
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection):
with self.proxy() as proxy:
worker = threading.Thread(target=call, args=(proxy,))
worker.start()
self.assertTrue(connected.wait(5))
worker.join(5)
self.assertFalse(worker.is_alive())
self.assertTrue(stopped.is_set())
self.assertEqual(len(self.log.read_text().splitlines()), 1)
with self.assertRaises(OSError):
self.request(proxy)
def test_server_start_failure_cleans_up_without_waiting_for_an_unstarted_thread(self):
thread = Mock()
thread.start.side_effect = RuntimeError("thread unavailable")
thread.is_alive.return_value = False
proxy = self.proxy()
with patch("rollout.model_proxy.threading.Thread", return_value=thread):
with self.assertRaises(RuntimeError):
proxy.__enter__()
self.assertIsNone(proxy._key)
self.assertIsNone(proxy._log)
self.assertEqual(proxy._server.socket.fileno(), -1)
thread.join.assert_not_called()
if __name__ == "__main__":
unittest.main()
class CredentialInstanceTests(unittest.TestCase):
def write(self, data):
folder = os.path.realpath(tempfile.mkdtemp()) # macOS /var is a symlink, which credentials refuse
self.addCleanup(shutil.rmtree, folder)
path = Path(folder, "key.json")
path.write_text(json.dumps(data))
path.chmod(0o600)
return path
def test_a_credential_may_pick_only_a_listed_instance_of_its_model(self):
qwen, glm = MODELS[QWEN_MODEL], MODELS["glm-5.3"]
self.assertEqual(_credential(self.write(dict(model_api_key="k")), qwen), ("k", "114.111.18.115:18080"))
self.assertEqual(_credential(self.write(dict(model_api_key="k", upstream_port=18083)), qwen),
("k", "114.111.18.115:18083"))
for port, route in ((18084, qwen), ("18081", qwen), (18081, glm)):
with self.assertRaises(ValueError):
_credential(self.write(dict(model_api_key="k", upstream_port=port)), route)