data-gen-ebd/tests/test_model_proxy.py
amos 31c862cd3c fix: retry transient upstream failures and classify broken sessions
Two trace4 runs lost their whole session to one upstream HTTP 500 on the
first request (2 of 562 requests), and their controllers still reported
the template's FAIL grade.
- The proxy retries upstream 429/5xx and connection errors up to twice
  (2 s, 5 s) before any byte reaches Pi, records every status in
  upstream_statuses, and never retries while it is closing.
- A controller reports PASS or FAIL only with VERIFIED evidence and a
  clean Agent exit; otherwise the grade stays in task_verdict and the
  status is INFRA_ERROR when the model channel returned non-200, else
  UNVERIFIED.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 02:10:05 +08:00

237 lines
13 KiB
Python

import hashlib
import http.client
import ipaddress
import json
from pathlib import Path
import tempfile
import threading
import unittest
from unittest.mock import Mock, patch
from urllib.parse import urlsplit
from rollout.model_proxy import MAX_BODY_BYTES, ModelProxy, MODEL, QWEN_MODEL
class ModelProxyTests(unittest.TestCase):
def setUp(self):
self.client_type = http.client.HTTPConnection
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name).resolve()
self.key = "test-provider-credential-do-not-forward"
self.credential = self.root / "credential.json"
self.credential.write_text(json.dumps({"model_api_key": self.key}))
self.credential.chmod(0o600)
self.log = self.root / "proxy.jsonl"
def proxy(self, model=MODEL):
return ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log, model=model)
def request(self, proxy, body=None, method="POST", path="/v1/chat/completions", headers=None):
endpoint = urlsplit(proxy.base_url)
client = http.client.HTTPConnection(endpoint.hostname, endpoint.port, timeout=5)
try:
client.request(method, path, body=body, headers=headers or {})
response = client.getresponse()
return response.status, dict(response.getheaders()), response.read()
finally:
client.close()
def upstream(self, status=200, chunks=None, content_type="application/json"):
response = Mock(status=status)
response.getheader.return_value = content_type
response.read1.side_effect = (chunks or [b'{"choices":[]}']) + [b""]
connection = Mock()
connection.getresponse.return_value = response
return connection, response
def test_credential_and_bind_trust_boundaries(self):
self.credential.chmod(0o640)
with self.assertRaises(ValueError):
self.proxy()
self.credential.chmod(0o600)
linked = self.root / "linked.json"
linked.symlink_to(self.credential)
with self.assertRaises(ValueError):
ModelProxy(linked, "127.0.0.1", "127.0.0.0/8", self.log)
self.credential.write_text('{"model_api_key":"bad\\nheader"}')
with self.assertRaises(ValueError):
self.proxy()
for bind, network in (("0.0.0.0", "0.0.0.0/8"), ("8.8.8.8", "8.0.0.0/8"),
("172.19.0.1", "172.18.0.0/16")):
with self.assertRaises(ValueError):
ModelProxy(self.credential, bind, network, self.log)
def test_source_model_path_method_and_body_are_rejected_without_upstream(self):
with patch("rollout.model_proxy.http.client.HTTPSConnection") as upstream:
with self.proxy() as proxy:
cases = [(b'{"model":"other"}', "POST", "/v1/chat/completions", 400),
(b'{"model":"glm-5.3","model":"other"}', "POST", "/v1/chat/completions", 400),
(b'[]', "POST", "/v1/chat/completions", 400),
(b'bad json', "POST", "/v1/chat/completions", 400),
(b'{"model":"glm-5.3"}', "POST", "/other", 404),
(None, "GET", "/v1/chat/completions", 405)]
for body, method, path, status in cases:
self.assertEqual(self.request(proxy, body, method, path)[0], status)
self.assertEqual(self.request(proxy, b"", headers={"Content-Length": str(MAX_BODY_BYTES + 1)})[0], 413)
self.assertEqual(self.request(proxy, b"", headers={"Transfer-Encoding": "chunked"})[0], 400)
proxy.network = ipaddress.ip_network("127.0.0.2/32")
self.assertEqual(self.request(proxy, b'{"model":"glm-5.3"}')[0], 403)
upstream.assert_not_called()
def test_fixed_upstream_authentication_and_json_transfer(self):
connection, response = self.upstream()
body = b'{"model":"glm-5.3","messages":[]}'
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream:
with self.proxy() as proxy:
status, headers, result = self.request(proxy, body, headers={"Authorization": "Bearer dummy"})
self.assertIsNone(proxy._key)
self.assertEqual((status, result), (200, b'{"choices":[]}'))
self.assertEqual(headers["Connection"], "close")
upstream.assert_called_once_with("open.bigmodel.cn", timeout=30)
connection.request.assert_called_once_with("POST", "/api/coding/paas/v4/chat/completions", body=body,
headers={"Authorization": "Bearer " + self.key,
"Content-Type": "application/json", "Connection": "close"})
connection.close.assert_called_once()
response.read1.assert_called_with(64 * 1024)
facts = json.loads(self.log.read_text())
self.assertEqual(facts["request_body_sha256"], hashlib.sha256(body).hexdigest())
self.assertEqual(facts["status"], 200)
self.assertEqual(facts["request_bytes"], len(body))
self.assertEqual(facts["response_bytes"], len(result))
self.assertNotIn(self.key, self.log.read_text())
self.assertNotIn("Authorization", self.log.read_text())
self.assertEqual(self.log.stat().st_mode & 0o777, 0o600)
def test_sse_transfer_preserves_chunks(self):
chunks = [b'data: {"choices":[]}\n\n', b'data: [DONE]\n\n']
connection, response = self.upstream(chunks=chunks, content_type="text/event-stream; charset=utf-8")
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection):
with self.proxy() as proxy:
status, headers, body = self.request(proxy, b'{"model":"glm-5.3","stream":true}')
self.assertEqual(status, 200)
self.assertEqual(headers["Content-Type"], "text/event-stream; charset=utf-8")
self.assertEqual(body, b"".join(chunks))
self.assertEqual(response.read1.call_count, 3)
def test_qwen_responses_preserves_split_stream_and_confirms_terminal(self):
terminal = json.dumps(dict(type="response.completed", response=dict(model=QWEN_MODEL, status="completed")))
chunks = [b'data: {"type":"response.created"}\n\nda',
("ta: " + terminal[:30]).encode(), (terminal[30:] + "\n\n").encode()]
connection, response = self.upstream(chunks=chunks, content_type="text/event-stream")
body = json.dumps(dict(model=QWEN_MODEL, input="fix", stream=True)).encode()
with patch("rollout.model_proxy.http.client.HTTPConnection", return_value=connection) as upstream:
# Keep the caller's HTTPConnection real while only mocking the fixed upstream.
upstream.side_effect = lambda host, *args, **kwargs: connection if host == "114.111.18.115:18080" else self.client_type(host, *args, **kwargs)
with self.proxy(QWEN_MODEL) as proxy:
status, headers, result = self.request(proxy, body, path="/v1/responses")
self.assertEqual((status, result), (200, b"".join(chunks)))
connection.request.assert_called_once_with("POST", "/v1/responses", body=body,
headers={"Authorization": "Bearer " + self.key, "Content-Type": "application/json", "Connection": "close"})
facts = json.loads(self.log.read_text())
self.assertEqual(facts["responses_terminal"], "response.completed")
self.assertEqual(facts["responses_terminal_count"], 1)
self.assertEqual(facts["response_model"], QWEN_MODEL)
self.assertEqual(facts["response_status"], "completed")
self.assertTrue(facts["responses_stream_valid"] and facts["responses_stream_finished"])
self.assertNotIn(self.key, self.log.read_text())
def test_qwen_rejects_other_model_chat_path_or_nonstream_before_upstream(self):
with patch("rollout.model_proxy.http.client.HTTPConnection") as upstream:
upstream.side_effect = lambda host, *args, **kwargs: self.client_type(host, *args, **kwargs)
with self.proxy(QWEN_MODEL) as proxy:
for payload, path, expected in ((dict(model=MODEL, stream=True), "/v1/responses", 400),
(dict(model=QWEN_MODEL, stream=True), "/v1/chat/completions", 404),
(dict(model=QWEN_MODEL, stream=False), "/v1/responses", 400)):
self.assertEqual(self.request(proxy, json.dumps(payload).encode(), path=path)[0], expected)
self.assertFalse(any(call.args[0] == "114.111.18.115:18080" for call in upstream.call_args_list))
with self.assertRaises(ValueError):
self.proxy("unknown")
def test_provider_errors_are_sanitized_and_redirect_is_not_followed(self):
for status, expected in ((401, 401), (429, 429), (302, 502)):
self.log.unlink(missing_ok=True)
connection, response = self.upstream(status=status)
response.reason = self.key
response.read.return_value = self.key.encode()
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream, \
patch("rollout.model_proxy.time.sleep") as sleep:
with self.proxy() as proxy:
actual, headers, body = self.request(proxy, b'{"model":"glm-5.3"}')
self.assertEqual(actual, expected)
self.assertNotIn(self.key.encode(), body)
self.assertNotIn(self.key, self.log.read_text())
self.assertNotIn("Location", headers)
self.assertEqual(upstream.call_count, 3 if status == 429 else 1)
self.assertEqual(sleep.call_count, 2 if status == 429 else 0)
response.read.assert_not_called()
response.read1.assert_not_called()
def test_transient_failures_before_the_response_are_retried_and_recorded(self):
body = b'{"model":"glm-5.3","messages":[]}'
broken = Mock()
broken.connect.side_effect = ConnectionResetError("reset")
for attempts, expected, statuses in (([self.upstream(500)[0], self.upstream()[0]], 200, [500, 200]),
([broken, self.upstream()[0]], 200, ["connection_error", 200]),
([self.upstream(500)[0] for _ in range(3)], 500, [500, 500, 500])):
self.log.unlink(missing_ok=True)
with patch("rollout.model_proxy.http.client.HTTPSConnection", side_effect=attempts), \
patch("rollout.model_proxy.time.sleep") as sleep:
with self.proxy() as proxy:
status, _, result = self.request(proxy, body)
facts = json.loads(self.log.read_text())
self.assertEqual((status, facts["status"], facts["upstream_statuses"]), (expected, expected, statuses))
self.assertEqual([call.args[0] for call in sleep.call_args_list], [2, 5][:len(statuses) - 1])
if expected == 200:
self.assertEqual(result, b'{"choices":[]}')
def test_context_exit_interrupts_active_transfer_and_closes_listener(self):
connection, response = self.upstream()
connected = threading.Event()
stopped = threading.Event()
connection.sock.shutdown.side_effect = lambda how: stopped.set()
def waiting_response():
connected.set()
if not stopped.wait(5):
raise RuntimeError("proxy did not interrupt the upstream socket")
raise OSError("closed")
connection.getresponse.side_effect = waiting_response
errors = []
def call(proxy):
try:
self.request(proxy, b'{"model":"glm-5.3"}')
except (OSError, http.client.HTTPException) as error:
errors.append(type(error).__name__)
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection):
with self.proxy() as proxy:
worker = threading.Thread(target=call, args=(proxy,))
worker.start()
self.assertTrue(connected.wait(5))
worker.join(5)
self.assertFalse(worker.is_alive())
self.assertTrue(stopped.is_set())
self.assertEqual(len(self.log.read_text().splitlines()), 1)
with self.assertRaises(OSError):
self.request(proxy)
def test_server_start_failure_cleans_up_without_waiting_for_an_unstarted_thread(self):
thread = Mock()
thread.start.side_effect = RuntimeError("thread unavailable")
thread.is_alive.return_value = False
proxy = self.proxy()
with patch("rollout.model_proxy.threading.Thread", return_value=thread):
with self.assertRaises(RuntimeError):
proxy.__enter__()
self.assertIsNone(proxy._key)
self.assertIsNone(proxy._log)
self.assertEqual(proxy._server.socket.fileno(), -1)
thread.join.assert_not_called()
if __name__ == "__main__":
unittest.main()