data-gen-ebd/rollout/model_proxy.py

257 lines
10 KiB
Python

"""One-attempt host proxy: the provider credential never enters the Agent."""
import hashlib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import http.client
import ipaddress
import json
import os
import socket
import stat
import threading
import time
from .core import _path
UPSTREAM_HOST = "open.bigmodel.cn"
UPSTREAM_PATH = "/api/coding/paas/v4/chat/completions"
MODEL = "glm-5.3"
MAX_BODY_BYTES = 4 * 1024 * 1024
def _credential(path):
path = _path(path)
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
with os.fdopen(descriptor, "rb") as stream:
info = os.fstat(stream.fileno())
if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077:
raise ValueError("credential must be a private regular file (0600)")
raw = stream.read(64 * 1024 + 1)
try:
if len(raw) > 64 * 1024:
raise ValueError
value = json.loads(raw)["model_api_key"]
if not isinstance(value, str) or not value.strip() or "\r" in value or "\n" in value:
raise ValueError
return value
except (ValueError, KeyError, TypeError, UnicodeDecodeError):
raise ValueError("credential must contain a nonempty model_api_key") from None
def _json_object(pairs):
value = {}
for key, item in pairs:
if key in value:
raise ValueError("duplicate JSON key")
value[key] = item
return value
class _Server(ThreadingHTTPServer):
def handle_error(self, request, client_address):
pass # Provider or client exception text must not reach the host log.
class _Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def setup(self):
super().setup()
self.connection.settimeout(30)
self.server.proxy._register(self.connection)
def finish(self):
try:
super().finish()
finally:
self.server.proxy._unregister(self.connection)
def log_message(self, format, *args):
pass
def _error(self, status):
body = b'{"error":{"message":"Model proxy request rejected","type":"proxy_error"}}'
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Connection", "close")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.close_connection = True
self.wfile.write(body)
self.wfile.flush()
return len(body)
def do_POST(self):
self._request()
def do_GET(self):
self._request()
do_HEAD = do_GET
do_PUT = do_GET
do_PATCH = do_GET
do_DELETE = do_GET
do_OPTIONS = do_GET
def _request(self):
proxy = self.server.proxy
started = time.monotonic()
facts = dict(sequence=proxy._next_sequence(), request_body_sha256=None,
status=502, request_bytes=0, response_bytes=0)
connection = upstream_socket = None
response_started = False
try:
if ipaddress.ip_address(self.client_address[0]) not in proxy.network:
facts["status"] = 403
elif self.command != "POST":
facts["status"] = 405
elif self.path != "/v1/chat/completions":
facts["status"] = 404
elif self.headers.get("Transfer-Encoding") is not None:
facts["status"] = 400
elif self.headers.get("Content-Length") is None:
facts["status"] = 411
else:
try:
length = int(self.headers["Content-Length"])
if not 0 < length <= MAX_BODY_BYTES:
facts["status"] = 413
else:
body = self.rfile.read(length)
facts.update(request_bytes=len(body), request_body_sha256=hashlib.sha256(body).hexdigest())
payload = json.loads(body, object_pairs_hook=_json_object)
facts["request_payload_sha256"] = hashlib.sha256(json.dumps(
payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
).encode("utf-8")).hexdigest()
if len(body) != length or not isinstance(payload, dict) or payload.get("model") != MODEL:
facts["status"] = 400
else:
connection = http.client.HTTPSConnection(UPSTREAM_HOST, timeout=30)
connection.connect()
upstream_socket = connection.sock
proxy._register(upstream_socket)
upstream_socket.settimeout(300)
connection.request("POST", UPSTREAM_PATH, body=body,
headers={"Authorization": "Bearer " + proxy._key,
"Content-Type": "application/json",
"Connection": "close"})
response = connection.getresponse()
facts["status"] = response.status if 400 <= response.status <= 599 else 502
if response.status == 200:
facts["status"] = 200
content_type = response.getheader("Content-Type", "application/json")
if ("\r" in content_type or "\n" in content_type or
content_type.split(";", 1)[0].strip() not in ("application/json", "text/event-stream")):
content_type = "application/json"
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Connection", "close")
self.end_headers()
self.close_connection = True
response_started = True
while True:
chunk = response.read1(64 * 1024)
if not chunk:
break
self.wfile.write(chunk)
self.wfile.flush()
facts["response_bytes"] += len(chunk)
return
except (ValueError, TypeError, UnicodeDecodeError):
facts["status"] = 400
response_started = True
facts["response_bytes"] = self._error(facts["status"])
except Exception:
if not response_started:
try:
facts["status"] = 502
facts["response_bytes"] = self._error(502)
except OSError:
pass
finally:
if connection is not None:
connection.close()
if upstream_socket is not None:
proxy._unregister(upstream_socket)
facts["duration_seconds"] = round(time.monotonic() - started, 6)
proxy._record(facts)
class ModelProxy:
"""Use one private Docker bridge and a fresh log for each rollout attempt."""
def __init__(self, credential_file, bind, allowed_network, log_path):
self.network = ipaddress.ip_network(allowed_network)
address = ipaddress.ip_address(bind)
if (address.version != 4 or self.network.version != 4 or not self.network.is_private or
not address.is_private or address.is_unspecified or address not in self.network):
raise ValueError("bind must be an IPv4 address inside the allowed private network")
self._key = _credential(credential_file)
self._bind = str(address)
self._log_path = _path(log_path)
self._lock = threading.Lock()
self._sequence = 0
self._sockets = set()
self._closing = False
self._server = self._thread = self._log = None
self.base_url = None
def _register(self, connection):
with self._lock:
if self._closing:
connection.close()
raise RuntimeError("proxy has closed")
self._sockets.add(connection)
def _unregister(self, connection):
with self._lock:
self._sockets.discard(connection)
def _next_sequence(self):
with self._lock:
self._sequence += 1
return self._sequence
def _record(self, facts):
with self._lock:
if self._log is not None:
self._log.write(json.dumps(facts, sort_keys=True) + "\n")
self._log.flush()
def __enter__(self):
if self._server is not None or self._closing:
raise RuntimeError("ModelProxy is single-use")
descriptor = os.open(self._log_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
self._log = os.fdopen(descriptor, "w", encoding="utf-8")
try:
self._server = _Server((self._bind, 0), _Handler)
self._server.proxy = self
self.base_url = "http://{}:{}/v1".format(self._bind, self._server.server_port)
self._thread = threading.Thread(target=self._server.serve_forever, kwargs={"poll_interval": 0.1}, daemon=True)
self._thread.start()
return self
except Exception:
self.__exit__(None, None, None)
raise
def __exit__(self, exc_type, exc_value, traceback):
with self._lock:
self._closing = True
connections = list(self._sockets)
for connection in connections:
try:
connection.shutdown(socket.SHUT_RDWR)
except OSError:
pass
connection.close()
if self._thread is not None and self._thread.is_alive():
self._server.shutdown()
self._thread.join()
if self._server is not None:
self._server.server_close()
with self._lock:
if self._log is not None:
self._log.close()
self._log = None
self._key = None