257 lines
10 KiB
Python
257 lines
10 KiB
Python
"""One-attempt host proxy: the provider credential never enters the Agent."""
|
|
|
|
import hashlib
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
import http.client
|
|
import ipaddress
|
|
import json
|
|
import os
|
|
import socket
|
|
import stat
|
|
import threading
|
|
import time
|
|
|
|
from .core import _path
|
|
|
|
UPSTREAM_HOST = "open.bigmodel.cn"
|
|
UPSTREAM_PATH = "/api/coding/paas/v4/chat/completions"
|
|
MODEL = "glm-5.3"
|
|
MAX_BODY_BYTES = 4 * 1024 * 1024
|
|
|
|
|
|
def _credential(path):
|
|
path = _path(path)
|
|
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
with os.fdopen(descriptor, "rb") as stream:
|
|
info = os.fstat(stream.fileno())
|
|
if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077:
|
|
raise ValueError("credential must be a private regular file (0600)")
|
|
raw = stream.read(64 * 1024 + 1)
|
|
try:
|
|
if len(raw) > 64 * 1024:
|
|
raise ValueError
|
|
value = json.loads(raw)["model_api_key"]
|
|
if not isinstance(value, str) or not value.strip() or "\r" in value or "\n" in value:
|
|
raise ValueError
|
|
return value
|
|
except (ValueError, KeyError, TypeError, UnicodeDecodeError):
|
|
raise ValueError("credential must contain a nonempty model_api_key") from None
|
|
|
|
|
|
def _json_object(pairs):
|
|
value = {}
|
|
for key, item in pairs:
|
|
if key in value:
|
|
raise ValueError("duplicate JSON key")
|
|
value[key] = item
|
|
return value
|
|
|
|
|
|
class _Server(ThreadingHTTPServer):
|
|
def handle_error(self, request, client_address):
|
|
pass # Provider or client exception text must not reach the host log.
|
|
|
|
|
|
class _Handler(BaseHTTPRequestHandler):
|
|
protocol_version = "HTTP/1.1"
|
|
|
|
def setup(self):
|
|
super().setup()
|
|
self.connection.settimeout(30)
|
|
self.server.proxy._register(self.connection)
|
|
|
|
def finish(self):
|
|
try:
|
|
super().finish()
|
|
finally:
|
|
self.server.proxy._unregister(self.connection)
|
|
|
|
def log_message(self, format, *args):
|
|
pass
|
|
|
|
def _error(self, status):
|
|
body = b'{"error":{"message":"Model proxy request rejected","type":"proxy_error"}}'
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Connection", "close")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.close_connection = True
|
|
self.wfile.write(body)
|
|
self.wfile.flush()
|
|
return len(body)
|
|
|
|
def do_POST(self):
|
|
self._request()
|
|
|
|
def do_GET(self):
|
|
self._request()
|
|
|
|
do_HEAD = do_GET
|
|
do_PUT = do_GET
|
|
do_PATCH = do_GET
|
|
do_DELETE = do_GET
|
|
do_OPTIONS = do_GET
|
|
|
|
def _request(self):
|
|
proxy = self.server.proxy
|
|
started = time.monotonic()
|
|
facts = dict(sequence=proxy._next_sequence(), request_body_sha256=None,
|
|
status=502, request_bytes=0, response_bytes=0)
|
|
connection = upstream_socket = None
|
|
response_started = False
|
|
try:
|
|
if ipaddress.ip_address(self.client_address[0]) not in proxy.network:
|
|
facts["status"] = 403
|
|
elif self.command != "POST":
|
|
facts["status"] = 405
|
|
elif self.path != "/v1/chat/completions":
|
|
facts["status"] = 404
|
|
elif self.headers.get("Transfer-Encoding") is not None:
|
|
facts["status"] = 400
|
|
elif self.headers.get("Content-Length") is None:
|
|
facts["status"] = 411
|
|
else:
|
|
try:
|
|
length = int(self.headers["Content-Length"])
|
|
if not 0 < length <= MAX_BODY_BYTES:
|
|
facts["status"] = 413
|
|
else:
|
|
body = self.rfile.read(length)
|
|
facts.update(request_bytes=len(body), request_body_sha256=hashlib.sha256(body).hexdigest())
|
|
payload = json.loads(body, object_pairs_hook=_json_object)
|
|
facts["request_payload_sha256"] = hashlib.sha256(json.dumps(
|
|
payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
|
|
).encode("utf-8")).hexdigest()
|
|
if len(body) != length or not isinstance(payload, dict) or payload.get("model") != MODEL:
|
|
facts["status"] = 400
|
|
else:
|
|
connection = http.client.HTTPSConnection(UPSTREAM_HOST, timeout=30)
|
|
connection.connect()
|
|
upstream_socket = connection.sock
|
|
proxy._register(upstream_socket)
|
|
upstream_socket.settimeout(300)
|
|
connection.request("POST", UPSTREAM_PATH, body=body,
|
|
headers={"Authorization": "Bearer " + proxy._key,
|
|
"Content-Type": "application/json",
|
|
"Connection": "close"})
|
|
response = connection.getresponse()
|
|
facts["status"] = response.status if 400 <= response.status <= 599 else 502
|
|
if response.status == 200:
|
|
facts["status"] = 200
|
|
content_type = response.getheader("Content-Type", "application/json")
|
|
if ("\r" in content_type or "\n" in content_type or
|
|
content_type.split(";", 1)[0].strip() not in ("application/json", "text/event-stream")):
|
|
content_type = "application/json"
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", content_type)
|
|
self.send_header("Connection", "close")
|
|
self.end_headers()
|
|
self.close_connection = True
|
|
response_started = True
|
|
while True:
|
|
chunk = response.read1(64 * 1024)
|
|
if not chunk:
|
|
break
|
|
self.wfile.write(chunk)
|
|
self.wfile.flush()
|
|
facts["response_bytes"] += len(chunk)
|
|
return
|
|
except (ValueError, TypeError, UnicodeDecodeError):
|
|
facts["status"] = 400
|
|
response_started = True
|
|
facts["response_bytes"] = self._error(facts["status"])
|
|
except Exception:
|
|
if not response_started:
|
|
try:
|
|
facts["status"] = 502
|
|
facts["response_bytes"] = self._error(502)
|
|
except OSError:
|
|
pass
|
|
finally:
|
|
if connection is not None:
|
|
connection.close()
|
|
if upstream_socket is not None:
|
|
proxy._unregister(upstream_socket)
|
|
facts["duration_seconds"] = round(time.monotonic() - started, 6)
|
|
proxy._record(facts)
|
|
|
|
|
|
class ModelProxy:
|
|
"""Use one private Docker bridge and a fresh log for each rollout attempt."""
|
|
|
|
def __init__(self, credential_file, bind, allowed_network, log_path):
|
|
self.network = ipaddress.ip_network(allowed_network)
|
|
address = ipaddress.ip_address(bind)
|
|
if (address.version != 4 or self.network.version != 4 or not self.network.is_private or
|
|
not address.is_private or address.is_unspecified or address not in self.network):
|
|
raise ValueError("bind must be an IPv4 address inside the allowed private network")
|
|
self._key = _credential(credential_file)
|
|
self._bind = str(address)
|
|
self._log_path = _path(log_path)
|
|
self._lock = threading.Lock()
|
|
self._sequence = 0
|
|
self._sockets = set()
|
|
self._closing = False
|
|
self._server = self._thread = self._log = None
|
|
self.base_url = None
|
|
|
|
def _register(self, connection):
|
|
with self._lock:
|
|
if self._closing:
|
|
connection.close()
|
|
raise RuntimeError("proxy has closed")
|
|
self._sockets.add(connection)
|
|
|
|
def _unregister(self, connection):
|
|
with self._lock:
|
|
self._sockets.discard(connection)
|
|
|
|
def _next_sequence(self):
|
|
with self._lock:
|
|
self._sequence += 1
|
|
return self._sequence
|
|
|
|
def _record(self, facts):
|
|
with self._lock:
|
|
if self._log is not None:
|
|
self._log.write(json.dumps(facts, sort_keys=True) + "\n")
|
|
self._log.flush()
|
|
|
|
def __enter__(self):
|
|
if self._server is not None or self._closing:
|
|
raise RuntimeError("ModelProxy is single-use")
|
|
descriptor = os.open(self._log_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
|
self._log = os.fdopen(descriptor, "w", encoding="utf-8")
|
|
try:
|
|
self._server = _Server((self._bind, 0), _Handler)
|
|
self._server.proxy = self
|
|
self.base_url = "http://{}:{}/v1".format(self._bind, self._server.server_port)
|
|
self._thread = threading.Thread(target=self._server.serve_forever, kwargs={"poll_interval": 0.1}, daemon=True)
|
|
self._thread.start()
|
|
return self
|
|
except Exception:
|
|
self.__exit__(None, None, None)
|
|
raise
|
|
|
|
def __exit__(self, exc_type, exc_value, traceback):
|
|
with self._lock:
|
|
self._closing = True
|
|
connections = list(self._sockets)
|
|
for connection in connections:
|
|
try:
|
|
connection.shutdown(socket.SHUT_RDWR)
|
|
except OSError:
|
|
pass
|
|
connection.close()
|
|
if self._thread is not None and self._thread.is_alive():
|
|
self._server.shutdown()
|
|
self._thread.join()
|
|
if self._server is not None:
|
|
self._server.server_close()
|
|
with self._lock:
|
|
if self._log is not None:
|
|
self._log.close()
|
|
self._log = None
|
|
self._key = None
|