data-gen-ebd/rollout/model_proxy.py
amos c93aeb8fd8 Let a credential pick one of the four Qwen instances
The Qwen service runs four identical instances on 114.111.18.115:18080-18083. A
credential may now carry upstream_port, limited to the ports the route lists, so
rollout.batch's least-busy key choice spreads attempts across instances. The broker
log records the upstream each request used.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-30 14:58:12 +08:00

425 lines
19 KiB
Python

"""One-attempt host proxy: the provider credential never enters the Agent."""
import hashlib
import hmac
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import http.client
import ipaddress
import json
import os
import queue
import socket
import stat
import threading
import time
from .core import _path
# Fixed upstream routes and per-response output limits; the Agent never chooses either.
# Sampling is left at each service's defaults. Both limits were accepted upstream on 2026-10-12/13.
MODELS = {
"glm-5.3": dict(api="openai-completions", path="/v1/chat/completions", https=True,
upstream="open.bigmodel.cn", upstream_path="/api/coding/paas/v4/chat/completions",
max_tokens=131072, max_tokens_field="max_tokens"),
# User-specified plain-HTTP service; the Bearer key crosses the network unencrypted.
"qwen3.8-27b-nvfp4": dict(api="openai-responses", path="/v1/responses", https=False,
upstream="114.111.18.115:18080", upstream_path="/v1/responses",
# Four identical instances of the same model; a credential may pick one of them.
upstream_ports=(18080, 18081, 18082, 18083),
max_tokens=65536, max_tokens_field="max_output_tokens"),
}
MODEL = "glm-5.3"
QWEN_MODEL = "qwen3.8-27b-nvfp4"
MODEL_IDS = tuple(MODELS)
MAX_BODY_BYTES = 4 * 1024 * 1024
# Qwen streams may pause for over 300 seconds; allow 900 seconds before treating upstream silence as failure.
UPSTREAM_IDLE_SECONDS = 900
# Transient upstream failures and stalls are retried before any generated byte reaches Pi,
# so its trace stays one real response.
RETRYABLE = (429, 500, 502, 503, 504)
RETRY_DELAYS = (2, 5)
# Pi's Node fetch gives up after 300 s without a byte; SSE comments keep it waiting while the proxy owns the limits.
KEEPALIVE_SECONDS = 30
# A request with no generation event by now is stalled and retried; tune from broker first_event_seconds.
STALL_SECONDS = 180
PREAMBLE = ("response.created", "response.in_progress")
TERMINALS = ("response.completed", "response.incomplete", "response.failed", "error")
def _credential(path, route):
"""Return (key, upstream host:port) from a private JSON credential; `upstream_port` picks one of the route's
listed instances."""
path = _path(path)
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
with os.fdopen(descriptor, "rb") as stream:
info = os.fstat(stream.fileno())
if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077:
raise ValueError("credential must be a private regular file (0600)")
raw = stream.read(64 * 1024 + 1)
try:
if len(raw) > 64 * 1024:
raise ValueError
data = json.loads(raw)
value = data["model_api_key"]
if not isinstance(value, str) or not value.strip() or "\r" in value or "\n" in value:
raise ValueError
except (ValueError, KeyError, TypeError, UnicodeDecodeError):
raise ValueError("credential must contain a nonempty model_api_key") from None
port = data.get("upstream_port")
if port is None:
return value, route["upstream"]
if type(port) is not int or port not in route.get("upstream_ports", ()):
raise ValueError("credential upstream_port is not one of this model's instances")
return value, route["upstream"].rsplit(":", 1)[0] + ":%d" % port
def _json_object(pairs):
value = {}
for key, item in pairs:
if key in value:
raise ValueError("duplicate JSON key")
value[key] = item
return value
class _Server(ThreadingHTTPServer):
def handle_error(self, request, client_address):
pass # Provider or client exception text must not reach the host log.
class _Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def setup(self):
super().setup()
self.connection.settimeout(30)
self.server.proxy._register(self.connection)
def finish(self):
try:
super().finish()
finally:
self.server.proxy._unregister(self.connection)
def log_message(self, format, *args):
pass
def _error(self, status):
body = b'{"error":{"message":"Model proxy request rejected","type":"proxy_error"}}'
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Connection", "close")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.close_connection = True
self.wfile.write(body)
self.wfile.flush()
return len(body)
def _release(self, connection, upstream_socket):
"""Close one upstream attempt; shutdown first wakes a reader thread still blocked on it."""
try:
upstream_socket.shutdown(socket.SHUT_RDWR)
except (AttributeError, OSError):
pass
connection.close()
if upstream_socket is not None:
self.server.proxy._unregister(upstream_socket)
def _send(self, data, facts):
self.wfile.write(data)
self.wfile.flush()
facts["response_bytes"] += len(data)
def _forward(self, response, facts, sse):
"""Relay one upstream 200; False when it stalled or broke before any generation event reached Pi."""
proxy = self.server.proxy
chunks = queue.Queue()
def read():
try:
while True:
chunk = response.read1(64 * 1024)
chunks.put(chunk)
if not chunk:
return
except Exception as error:
chunks.put(error)
threading.Thread(target=read, daemon=True).start()
started = time.monotonic()
held = pending = b""
live = not sse # Plain JSON bodies have no events to wait for.
if proxy.responses:
facts.update(responses_terminal=None, responses_terminal_count=0,
responses_stream_valid=True, responses_stream_finished=False)
while True:
try:
chunk = chunks.get(timeout=KEEPALIVE_SECONDS)
except queue.Empty:
if not live and time.monotonic() - started >= STALL_SECONDS:
return False
if sse:
self._send(b": keepalive\n\n", facts)
continue
if isinstance(chunk, Exception):
if live:
raise chunk
return False
if not chunk:
break
if sse or proxy.responses:
pending += chunk
lines = pending.split(b"\n")
pending = lines.pop()
if len(pending) > MAX_BODY_BYTES:
raise OSError("oversized stream event")
for line in lines:
if not line.startswith(b"data:"):
continue
kind = None
if line[5:].strip() != b"[DONE]":
try:
event = json.loads(line[5:])
kind = event.get("type")
if proxy.responses and kind in TERMINALS:
facts["responses_terminal_count"] += 1
facts["responses_terminal"] = kind
facts["response_model"] = event.get("response", {}).get("model")
facts["response_status"] = event.get("response", {}).get("status")
except (ValueError, AttributeError, TypeError):
if proxy.responses:
facts["responses_stream_valid"] = False
if not live and kind not in PREAMBLE:
live = True
facts["first_event_seconds"] = round(time.monotonic() - started, 3)
held += chunk
if live or len(held) > MAX_BODY_BYTES:
live = True
self._send(held, facts)
held = b""
if not live:
return False
if proxy.responses:
facts["responses_stream_valid"] &= not bool(pending.strip())
facts["responses_stream_finished"] = True
return True
def do_POST(self):
self._request()
def do_GET(self):
proxy = self.server.proxy
# Tunnel check for the Salty starter: token-checked and never forwarded, so it is not a broker record.
if (self.path == "/health" and proxy._token is not None and
ipaddress.ip_address(self.client_address[0]) in proxy.network and hmac.compare_digest(
self.headers.get("Authorization", "").encode(), ("Bearer " + proxy._token).encode())):
self.send_response(204)
self.send_header("Connection", "close")
self.send_header("Content-Length", "0")
self.end_headers()
self.close_connection = True
return
self._request()
do_HEAD = do_GET
do_PUT = do_GET
do_PATCH = do_GET
do_DELETE = do_GET
do_OPTIONS = do_GET
def _request(self):
proxy = self.server.proxy
started = time.monotonic()
facts = dict(sequence=proxy._next_sequence(), request_body_sha256=None,
status=502, request_bytes=0, response_bytes=0)
connection = upstream_socket = None
response_started = False
try:
if ipaddress.ip_address(self.client_address[0]) not in proxy.network:
facts["status"] = 403
elif proxy._token is not None and not hmac.compare_digest(
self.headers.get("Authorization", "").encode(), ("Bearer " + proxy._token).encode()):
facts["status"] = 401
elif self.command != "POST":
facts["status"] = 405
elif self.path != proxy.request_path:
facts["status"] = 404
elif self.headers.get("Transfer-Encoding") is not None:
facts["status"] = 400
elif self.headers.get("Content-Length") is None:
facts["status"] = 411
else:
try:
length = int(self.headers["Content-Length"])
if not 0 < length <= MAX_BODY_BYTES:
facts["status"] = 413
else:
body = self.rfile.read(length)
facts.update(request_bytes=len(body), request_body_sha256=hashlib.sha256(body).hexdigest())
payload = json.loads(body, object_pairs_hook=_json_object)
facts["request_payload_sha256"] = hashlib.sha256(json.dumps(
payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
).encode("utf-8")).hexdigest()
if (len(body) != length or not isinstance(payload, dict) or
payload.get("model") != proxy.model or
(proxy.responses and payload.get("stream") is not True)):
facts["status"] = 400
else:
route = MODELS[proxy.model]
statuses = facts["upstream_statuses"] = []
facts["upstream"] = proxy.upstream
for delay in RETRY_DELAYS + (None,):
response = None
try:
connection = (http.client.HTTPSConnection if route["https"] else
http.client.HTTPConnection)(proxy.upstream, timeout=30)
connection.connect()
upstream_socket = connection.sock
proxy._register(upstream_socket)
upstream_socket.settimeout(UPSTREAM_IDLE_SECONDS)
connection.request("POST", route["upstream_path"], body=body,
headers={"Authorization": "Bearer " + proxy._key,
"Content-Type": "application/json",
"Connection": "close"})
response = connection.getresponse()
statuses.append(response.status)
except OSError:
statuses.append("connection_error")
if delay is None or proxy._closing:
raise
if response is not None and response.status == 200:
content_type = response.getheader("Content-Type", "application/json")
if ("\r" in content_type or "\n" in content_type or
content_type.split(";", 1)[0].strip() not in ("application/json", "text/event-stream")):
content_type = "application/json"
if not response_started:
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Connection", "close")
self.end_headers()
self.close_connection = True
response_started = True
facts["status"] = 200
if self._forward(response, facts, content_type.startswith("text/event-stream")):
return
statuses[-1] = "stalled"
if delay is None or proxy._closing:
raise OSError("upstream stalled before generating")
elif response is not None and (delay is None or proxy._closing or
response.status not in RETRYABLE):
break
self._release(connection, upstream_socket)
connection = upstream_socket = None
time.sleep(delay)
facts["status"] = response.status if 400 <= response.status <= 599 else 502
if response_started:
raise OSError("upstream failed after the response began")
except (ValueError, TypeError, UnicodeDecodeError):
facts["status"] = 400
response_started = True
facts["response_bytes"] = self._error(facts["status"])
except Exception:
if not response_started:
try:
facts["status"] = 502
facts["response_bytes"] = self._error(502)
except OSError:
pass
finally:
if connection is not None:
self._release(connection, upstream_socket)
facts["duration_seconds"] = round(time.monotonic() - started, 6)
proxy._record(facts)
class ModelProxy:
"""Use one private Docker bridge and a fresh log for each rollout attempt."""
def __init__(self, credential_file, bind, allowed_network, log_path, model=MODEL, port=0, token=None):
if model not in MODELS:
raise ValueError("unsupported rollout model")
self.model = model
self.request_path = MODELS[model]["path"]
self.responses = MODELS[model]["api"] == "openai-responses"
self.network = ipaddress.ip_network(allowed_network)
address = ipaddress.ip_address(bind)
# A token-checked proxy (Salty) binds the host address while clients come from another private network.
if (address.version != 4 or self.network.version != 4 or not self.network.is_private or
not address.is_private or address.is_unspecified or (token is None and address not in self.network)):
raise ValueError("bind must be an IPv4 address inside the allowed private network")
if token is not None and (not isinstance(token, str) or len(token) < 32 or not token.isalnum()):
raise ValueError("token must be at least 32 alphanumeric characters")
if type(port) is not int or not 0 <= port <= 65535:
raise ValueError("port must be 0..65535")
self._token = token
self._port = port
self._key, self.upstream = _credential(credential_file, MODELS[model])
self._bind = str(address)
self._log_path = _path(log_path)
self._lock = threading.Lock()
self._sequence = 0
self._sockets = set()
self._closing = False
self._server = self._thread = self._log = None
self.base_url = None
def _register(self, connection):
with self._lock:
if self._closing:
connection.close()
raise RuntimeError("proxy has closed")
self._sockets.add(connection)
def _unregister(self, connection):
with self._lock:
self._sockets.discard(connection)
def _next_sequence(self):
with self._lock:
self._sequence += 1
return self._sequence
def _record(self, facts):
with self._lock:
if self._log is not None:
self._log.write(json.dumps(facts, sort_keys=True) + "\n")
self._log.flush()
def __enter__(self):
if self._server is not None or self._closing:
raise RuntimeError("ModelProxy is single-use")
descriptor = os.open(self._log_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
self._log = os.fdopen(descriptor, "w", encoding="utf-8")
try:
self._server = _Server((self._bind, self._port), _Handler)
self._server.proxy = self
self.base_url = "http://{}:{}/v1".format(self._bind, self._server.server_port)
self._thread = threading.Thread(target=self._server.serve_forever, kwargs={"poll_interval": 0.1}, daemon=True)
self._thread.start()
return self
except Exception:
self.__exit__(None, None, None)
raise
def __exit__(self, exc_type, exc_value, traceback):
with self._lock:
self._closing = True
connections = list(self._sockets)
for connection in connections:
try:
connection.shutdown(socket.SHUT_RDWR)
except OSError:
pass
connection.close()
if self._thread is not None and self._thread.is_alive():
self._server.shutdown()
self._thread.join()
if self._server is not None:
self._server.server_close()
with self._lock:
if self._log is not None:
self._log.close()
self._log = None
self._key = None