data-gen-ebd/tests/test_model_proxy.py
amos 3adab31da7 Self-heal rollouts: retry stalled model streams and run batches from receipts
Batch 6 lost 26 of 103 attempts to Qwen requests that sent their preamble and
then stalled; Pi's own Node fetch gave up after 300 s and reported "terminated".

- model_proxy: hold the SSE preamble until the first generation event, retry a
  stall (180 s) within the existing retry budget, and send SSE keepalive
  comments so Pi never times out first; record first_event_seconds.
- report: name one exclusion reason per incomplete attempt and compute the
  JOBS lines still owed, with a per-line resample budget.
- batch: run a JOBS file to completion, resume from receipts, resample
  infrastructure failures, pause after three environment failures in a row,
  forward SIGTERM to running controllers, and refuse a second runner.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 08:28:53 +08:00

394 lines
21 KiB
Python

import hashlib
import http.client
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import ipaddress
import json
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import threading
import time
import unittest
from unittest.mock import Mock, patch
from urllib.parse import urlsplit
from rollout import model_proxy
from rollout.model_proxy import MAX_BODY_BYTES, ModelProxy, MODEL, MODELS, QWEN_MODEL, UPSTREAM_IDLE_SECONDS
ROOT = Path(__file__).resolve().parents[1]
FAST = dict(STALL_SECONDS=0.4, KEEPALIVE_SECONDS=0.1, RETRY_DELAYS=(0, 0))
def sse(*events):
return b"".join(b"data: " + json.dumps(event).encode() + b"\n\n" for event in events)
PREAMBLE = sse(dict(type="response.created"), dict(type="response.in_progress"))
REPLY = sse(dict(type="response.output_text.delta", delta="ok"),
dict(type="response.completed", response=dict(model=QWEN_MODEL, status="completed")))
class ModelProxyTests(unittest.TestCase):
def setUp(self):
self.client_type = http.client.HTTPConnection
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name).resolve()
self.key = "test-provider-credential-do-not-forward"
self.credential = self.root / "credential.json"
self.credential.write_text(json.dumps({"model_api_key": self.key}))
self.credential.chmod(0o600)
self.log = self.root / "proxy.jsonl"
def proxy(self, model=MODEL):
return ModelProxy(self.credential, "127.0.0.1", "127.0.0.0/8", self.log, model=model)
def request(self, proxy, body=None, method="POST", path="/v1/chat/completions", headers=None):
endpoint = urlsplit(proxy.base_url)
client = http.client.HTTPConnection(endpoint.hostname, endpoint.port, timeout=5)
try:
client.request(method, path, body=body, headers=headers or {})
response = client.getresponse()
return response.status, dict(response.getheaders()), response.read()
finally:
client.close()
def upstream(self, status=200, chunks=None, content_type="application/json"):
response = Mock(status=status)
response.getheader.return_value = content_type
response.read1.side_effect = (chunks or [b'{"choices":[]}']) + [b""]
connection = Mock()
connection.getresponse.return_value = response
return connection, response
def test_credential_and_bind_trust_boundaries(self):
self.credential.chmod(0o640)
with self.assertRaises(ValueError):
self.proxy()
self.credential.chmod(0o600)
linked = self.root / "linked.json"
linked.symlink_to(self.credential)
with self.assertRaises(ValueError):
ModelProxy(linked, "127.0.0.1", "127.0.0.0/8", self.log)
self.credential.write_text('{"model_api_key":"bad\\nheader"}')
with self.assertRaises(ValueError):
self.proxy()
for bind, network in (("0.0.0.0", "0.0.0.0/8"), ("8.8.8.8", "8.0.0.0/8"),
("172.19.0.1", "172.18.0.0/16")):
with self.assertRaises(ValueError):
ModelProxy(self.credential, bind, network, self.log)
def test_source_model_path_method_and_body_are_rejected_without_upstream(self):
with patch("rollout.model_proxy.http.client.HTTPSConnection") as upstream:
with self.proxy() as proxy:
cases = [(b'{"model":"other"}', "POST", "/v1/chat/completions", 400),
(b'{"model":"glm-5.3","model":"other"}', "POST", "/v1/chat/completions", 400),
(b'[]', "POST", "/v1/chat/completions", 400),
(b'bad json', "POST", "/v1/chat/completions", 400),
(b'{"model":"glm-5.3"}', "POST", "/other", 404),
(None, "GET", "/v1/chat/completions", 405)]
for body, method, path, status in cases:
self.assertEqual(self.request(proxy, body, method, path)[0], status)
self.assertEqual(self.request(proxy, b"", headers={"Content-Length": str(MAX_BODY_BYTES + 1)})[0], 413)
self.assertEqual(self.request(proxy, b"", headers={"Transfer-Encoding": "chunked"})[0], 400)
proxy.network = ipaddress.ip_network("127.0.0.2/32")
self.assertEqual(self.request(proxy, b'{"model":"glm-5.3"}')[0], 403)
upstream.assert_not_called()
def test_fixed_upstream_authentication_and_json_transfer(self):
connection, response = self.upstream()
body = b'{"model":"glm-5.3","messages":[]}'
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream:
with self.proxy() as proxy:
status, headers, result = self.request(proxy, body, headers={"Authorization": "Bearer dummy"})
self.assertIsNone(proxy._key)
self.assertEqual((status, result), (200, b'{"choices":[]}'))
self.assertEqual(headers["Connection"], "close")
upstream.assert_called_once_with("open.bigmodel.cn", timeout=30)
connection.request.assert_called_once_with("POST", "/api/coding/paas/v4/chat/completions", body=body,
headers={"Authorization": "Bearer " + self.key,
"Content-Type": "application/json", "Connection": "close"})
connection.close.assert_called_once()
response.read1.assert_called_with(64 * 1024)
facts = json.loads(self.log.read_text())
self.assertEqual(facts["request_body_sha256"], hashlib.sha256(body).hexdigest())
self.assertEqual(facts["status"], 200)
self.assertEqual(facts["request_bytes"], len(body))
self.assertEqual(facts["response_bytes"], len(result))
self.assertNotIn(self.key, self.log.read_text())
self.assertNotIn("Authorization", self.log.read_text())
self.assertEqual(self.log.stat().st_mode & 0o777, 0o600)
def test_sse_transfer_preserves_chunks(self):
chunks = [b'data: {"choices":[]}\n\n', b'data: [DONE]\n\n']
connection, response = self.upstream(chunks=chunks, content_type="text/event-stream; charset=utf-8")
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection):
with self.proxy() as proxy:
status, headers, body = self.request(proxy, b'{"model":"glm-5.3","stream":true}')
self.assertEqual(status, 200)
self.assertEqual(headers["Content-Type"], "text/event-stream; charset=utf-8")
self.assertEqual(body, b"".join(chunks))
self.assertEqual(response.read1.call_count, 3)
self.assertEqual(UPSTREAM_IDLE_SECONDS, 900)
connection.sock.settimeout.assert_called_once_with(UPSTREAM_IDLE_SECONDS)
def test_qwen_responses_preserves_split_stream_and_confirms_terminal(self):
terminal = json.dumps(dict(type="response.completed", response=dict(model=QWEN_MODEL, status="completed")))
chunks = [b'data: {"type":"response.created"}\n\nda',
("ta: " + terminal[:30]).encode(), (terminal[30:] + "\n\n").encode()]
connection, response = self.upstream(chunks=chunks, content_type="text/event-stream")
body = json.dumps(dict(model=QWEN_MODEL, input="fix", stream=True)).encode()
with patch("rollout.model_proxy.http.client.HTTPConnection", return_value=connection) as upstream:
# Keep the caller's HTTPConnection real while only mocking the fixed upstream.
upstream.side_effect = lambda host, *args, **kwargs: connection if host == "114.111.18.115:18080" else self.client_type(host, *args, **kwargs)
with self.proxy(QWEN_MODEL) as proxy:
status, headers, result = self.request(proxy, body, path="/v1/responses")
self.assertEqual((status, result), (200, b"".join(chunks)))
connection.sock.settimeout.assert_called_once_with(UPSTREAM_IDLE_SECONDS)
connection.request.assert_called_once_with("POST", "/v1/responses", body=body,
headers={"Authorization": "Bearer " + self.key, "Content-Type": "application/json", "Connection": "close"})
facts = json.loads(self.log.read_text())
self.assertEqual(facts["responses_terminal"], "response.completed")
self.assertEqual(facts["responses_terminal_count"], 1)
self.assertEqual(facts["response_model"], QWEN_MODEL)
self.assertEqual(facts["response_status"], "completed")
self.assertTrue(facts["responses_stream_valid"] and facts["responses_stream_finished"])
self.assertNotIn(self.key, self.log.read_text())
def qwen(self, *streams):
"""Run one Qwen request against scripted upstream attempts; a stream step is bytes, a pause or 'hang'."""
released = threading.Event()
connections = []
for steps in streams:
steps = iter(list(steps) + [b""])
def read1(size, steps=steps):
step = next(steps)
while isinstance(step, float):
time.sleep(step)
step = next(steps)
if step == "hang":
released.wait(5)
raise OSError("closed")
return step
connection, response = self.upstream(content_type="text/event-stream")
response.read1.side_effect = read1
connection.sock.shutdown.side_effect = lambda how: released.set()
connections.append(connection)
attempts = iter(connections)
body = json.dumps(dict(model=QWEN_MODEL, input="fix", stream=True)).encode()
with patch("rollout.model_proxy.http.client.HTTPConnection") as upstream, \
patch.multiple(model_proxy, **FAST):
upstream.side_effect = lambda host, *args, **kwargs: (
next(attempts) if host == "114.111.18.115:18080" else self.client_type(host, *args, **kwargs))
with self.proxy(QWEN_MODEL) as proxy:
try:
result = self.request(proxy, body, path="/v1/responses")[2]
except (OSError, http.client.HTTPException):
result = None
return result, json.loads(self.log.read_text())
def test_stall_before_generation_is_retried_and_pi_sees_one_response(self):
started = time.monotonic()
result, facts = self.qwen([PREAMBLE, "hang"], [PREAMBLE, REPLY])
self.assertLess(time.monotonic() - started, 3) # The stall timer fired, not the 5 s test fallback.
self.assertEqual(result.replace(b": keepalive\n\n", b""), PREAMBLE + REPLY)
self.assertIn(b": keepalive\n\n", result)
self.assertEqual(facts["upstream_statuses"], ["stalled", 200])
self.assertEqual((facts["status"], facts["responses_terminal"], facts["responses_terminal_count"]),
(200, "response.completed", 1))
self.assertTrue(facts["responses_stream_finished"])
def test_slow_first_event_gets_keepalives_and_no_retry(self):
result, facts = self.qwen([PREAMBLE, 0.3, REPLY])
self.assertEqual(result.replace(b": keepalive\n\n", b""), PREAMBLE + REPLY)
self.assertIn(b": keepalive\n\n", result)
self.assertEqual(facts["upstream_statuses"], [200])
self.assertGreaterEqual(facts["first_event_seconds"], 0.3)
def test_generation_already_forwarded_is_never_retried(self):
started = sse(dict(type="response.output_text.delta", delta="o"))
result, facts = self.qwen([PREAMBLE, started, "hang"], [PREAMBLE, REPLY])
self.assertEqual(facts["upstream_statuses"], [200])
self.assertIsNone(facts["responses_terminal"])
self.assertFalse(facts["responses_stream_finished"])
def test_every_attempt_stalling_ends_without_forwarding_a_preamble(self):
result, facts = self.qwen([PREAMBLE, "hang"], [PREAMBLE, "hang"], [PREAMBLE, "hang"])
self.assertEqual(facts["upstream_statuses"], ["stalled"] * 3)
self.assertNotIn(b"response.created", result or b"")
self.assertIsNone(facts["responses_terminal"])
@unittest.skipUnless(shutil.which("node") and (ROOT / "agent/node_modules/undici").is_dir(),
"needs Node and agent/node_modules with the pinned Pi SDK")
def test_real_pi_sdk_survives_a_stall_only_because_of_keepalives(self):
"""Pi's fetch gives up after 0.6 s here; the proxy waits 1 s, retries, and the retry takes 0.9 s."""
posts = []
class Upstream(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def log_message(self, *args):
pass
def do_POST(self):
self.rfile.read(int(self.headers["Content-Length"]))
posts.append(time.monotonic())
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Connection", "close")
self.end_headers()
self.wfile.write(sse(dict(type="response.created", response=dict(id="r")),
dict(type="response.in_progress", response=dict(id="r"))))
self.wfile.flush()
time.sleep(3 if len(posts) == 1 else 0.9)
item = dict(type="message", id="m", role="assistant", status="completed",
content=[dict(type="output_text", text="Done.", annotations=[])])
self.wfile.write(sse(
dict(type="response.output_item.added", output_index=0, item=dict(item, content=[])),
dict(type="response.output_text.delta", item_id="m", output_index=0, content_index=0,
delta="Done."),
dict(type="response.output_item.done", output_index=0, item=item),
dict(type="response.completed", response=dict(id="r", model=QWEN_MODEL, status="completed",
usage=dict(input_tokens=3, output_tokens=1,
total_tokens=4)))))
script = """
import {Agent, setGlobalDispatcher} from 'undici';
import fs from 'node:fs';
import {run, loadSdk} from './pi.mjs';
setGlobalDispatcher(new Agent({headersTimeout: 600, bodyTimeout: 600}));
const [baseUrl, directory] = process.argv.slice(1);
const summary = await run('Say done.', {cwd: directory, traceFile: directory + '/native.jsonl', baseUrl,
modelId: 'qwen3.8-27b-nvfp4', maxTokens: 64, mirror: () => {}}, await loadSdk());
const last = fs.readFileSync(directory + '/native.jsonl', 'utf8').trim().split('\\n').map(JSON.parse)
.filter(event => event.type === 'message_end').at(-1);
console.log(JSON.stringify({complete: summary.complete, error: last?.message?.errorMessage ?? null}));
"""
upstream = ThreadingHTTPServer(("127.0.0.1", 0), Upstream)
threading.Thread(target=upstream.serve_forever, daemon=True).start()
self.addCleanup(upstream.server_close)
self.addCleanup(upstream.shutdown)
outcomes = []
for keepalive in (0.2, 30):
posts.clear()
self.log.unlink(missing_ok=True)
work = Path(tempfile.mkdtemp(dir=self.root))
route = dict(MODELS[QWEN_MODEL], upstream="127.0.0.1:%d" % upstream.server_port)
with patch.dict(MODELS, {QWEN_MODEL: route}), patch.multiple(
model_proxy, STALL_SECONDS=1, KEEPALIVE_SECONDS=keepalive, RETRY_DELAYS=(0, 0)):
with self.proxy(QWEN_MODEL) as proxy:
done = subprocess.run(["node", "--input-type=module", "-e", script, proxy.base_url, str(work)],
cwd=ROOT / "agent", capture_output=True, text=True, timeout=60,
env=dict(os.environ, NODE_NO_WARNINGS="1"))
self.assertEqual(done.returncode, 0, done.stderr)
outcomes.append((json.loads(done.stdout.splitlines()[-1]), len(posts),
json.loads(self.log.read_text().splitlines()[0])["upstream_statuses"]))
(kept, kept_posts, kept_statuses), (bare, _, _) = outcomes
self.assertEqual(kept, dict(complete=True, error=None))
self.assertEqual((kept_posts, kept_statuses), (2, ["stalled", 200]))
# Control: without keepalives the same stall kills the session the way batch 6 lost 26 attempts.
self.assertEqual(bare, dict(complete=False, error="terminated"))
def test_qwen_rejects_other_model_chat_path_or_nonstream_before_upstream(self):
with patch("rollout.model_proxy.http.client.HTTPConnection") as upstream:
upstream.side_effect = lambda host, *args, **kwargs: self.client_type(host, *args, **kwargs)
with self.proxy(QWEN_MODEL) as proxy:
for payload, path, expected in ((dict(model=MODEL, stream=True), "/v1/responses", 400),
(dict(model=QWEN_MODEL, stream=True), "/v1/chat/completions", 404),
(dict(model=QWEN_MODEL, stream=False), "/v1/responses", 400)):
self.assertEqual(self.request(proxy, json.dumps(payload).encode(), path=path)[0], expected)
self.assertFalse(any(call.args[0] == "114.111.18.115:18080" for call in upstream.call_args_list))
with self.assertRaises(ValueError):
self.proxy("unknown")
def test_provider_errors_are_sanitized_and_redirect_is_not_followed(self):
for status, expected in ((401, 401), (429, 429), (302, 502)):
self.log.unlink(missing_ok=True)
connection, response = self.upstream(status=status)
response.reason = self.key
response.read.return_value = self.key.encode()
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection) as upstream, \
patch("rollout.model_proxy.time.sleep") as sleep:
with self.proxy() as proxy:
actual, headers, body = self.request(proxy, b'{"model":"glm-5.3"}')
self.assertEqual(actual, expected)
self.assertNotIn(self.key.encode(), body)
self.assertNotIn(self.key, self.log.read_text())
self.assertNotIn("Location", headers)
self.assertEqual(upstream.call_count, 3 if status == 429 else 1)
self.assertEqual(sleep.call_count, 2 if status == 429 else 0)
response.read.assert_not_called()
response.read1.assert_not_called()
def test_transient_failures_before_the_response_are_retried_and_recorded(self):
body = b'{"model":"glm-5.3","messages":[]}'
broken = Mock()
broken.connect.side_effect = ConnectionResetError("reset")
for attempts, expected, statuses in (([self.upstream(500)[0], self.upstream()[0]], 200, [500, 200]),
([broken, self.upstream()[0]], 200, ["connection_error", 200]),
([self.upstream(500)[0] for _ in range(3)], 500, [500, 500, 500])):
self.log.unlink(missing_ok=True)
with patch("rollout.model_proxy.http.client.HTTPSConnection", side_effect=attempts), \
patch("rollout.model_proxy.time.sleep") as sleep:
with self.proxy() as proxy:
status, _, result = self.request(proxy, body)
facts = json.loads(self.log.read_text())
self.assertEqual((status, facts["status"], facts["upstream_statuses"]), (expected, expected, statuses))
self.assertEqual([call.args[0] for call in sleep.call_args_list], [2, 5][:len(statuses) - 1])
if expected == 200:
self.assertEqual(result, b'{"choices":[]}')
def test_context_exit_interrupts_active_transfer_and_closes_listener(self):
connection, response = self.upstream()
connected = threading.Event()
stopped = threading.Event()
connection.sock.shutdown.side_effect = lambda how: stopped.set()
def waiting_response():
connected.set()
if not stopped.wait(5):
raise RuntimeError("proxy did not interrupt the upstream socket")
raise OSError("closed")
connection.getresponse.side_effect = waiting_response
errors = []
def call(proxy):
try:
self.request(proxy, b'{"model":"glm-5.3"}')
except (OSError, http.client.HTTPException) as error:
errors.append(type(error).__name__)
with patch("rollout.model_proxy.http.client.HTTPSConnection", return_value=connection):
with self.proxy() as proxy:
worker = threading.Thread(target=call, args=(proxy,))
worker.start()
self.assertTrue(connected.wait(5))
worker.join(5)
self.assertFalse(worker.is_alive())
self.assertTrue(stopped.is_set())
self.assertEqual(len(self.log.read_text().splitlines()), 1)
with self.assertRaises(OSError):
self.request(proxy)
def test_server_start_failure_cleans_up_without_waiting_for_an_unstarted_thread(self):
thread = Mock()
thread.start.side_effect = RuntimeError("thread unavailable")
thread.is_alive.return_value = False
proxy = self.proxy()
with patch("rollout.model_proxy.threading.Thread", return_value=thread):
with self.assertRaises(RuntimeError):
proxy.__enter__()
self.assertIsNone(proxy._key)
self.assertIsNone(proxy._log)
self.assertEqual(proxy._server.socket.fileno(), -1)
thread.join.assert_not_called()
if __name__ == "__main__":
unittest.main()