data-gen-ebd/rollout/model_proxy.py

325 lines
15 KiB
Python

"""One-attempt host proxy: the provider credential never enters the Agent."""
import hashlib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import http.client
import ipaddress
import json
import os
import socket
import stat
import threading
import time
from .core import _path
# Fixed upstream routes and per-response output limits; the Agent never chooses either.
# Sampling is left at each service's defaults. Both limits were accepted upstream on 2026-10-12/13.
MODELS = {
"glm-5.3": dict(api="openai-completions", path="/v1/chat/completions", https=True,
upstream="open.bigmodel.cn", upstream_path="/api/coding/paas/v4/chat/completions",
max_tokens=131072, max_tokens_field="max_tokens"),
# User-specified plain-HTTP service; the Bearer key crosses the network unencrypted.
"qwen3.8-27b-nvfp4": dict(api="openai-responses", path="/v1/responses", https=False,
upstream="114.111.18.115:18080", upstream_path="/v1/responses",
max_tokens=65536, max_tokens_field="max_output_tokens"),
}
MODEL = "glm-5.3"
QWEN_MODEL = "qwen3.8-27b-nvfp4"
MODEL_IDS = tuple(MODELS)
MAX_BODY_BYTES = 4 * 1024 * 1024
# Qwen streams may pause for over 300 seconds; allow 900 seconds before treating upstream silence as failure.
UPSTREAM_IDLE_SECONDS = 900
# Transient upstream failures are retried before any byte reaches Pi, so its trace stays one real response.
RETRYABLE = (429, 500, 502, 503, 504)
RETRY_DELAYS = (2, 5)
def _credential(path):
path = _path(path)
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
with os.fdopen(descriptor, "rb") as stream:
info = os.fstat(stream.fileno())
if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077:
raise ValueError("credential must be a private regular file (0600)")
raw = stream.read(64 * 1024 + 1)
try:
if len(raw) > 64 * 1024:
raise ValueError
value = json.loads(raw)["model_api_key"]
if not isinstance(value, str) or not value.strip() or "\r" in value or "\n" in value:
raise ValueError
return value
except (ValueError, KeyError, TypeError, UnicodeDecodeError):
raise ValueError("credential must contain a nonempty model_api_key") from None
def _json_object(pairs):
value = {}
for key, item in pairs:
if key in value:
raise ValueError("duplicate JSON key")
value[key] = item
return value
class _Server(ThreadingHTTPServer):
def handle_error(self, request, client_address):
pass # Provider or client exception text must not reach the host log.
class _Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def setup(self):
super().setup()
self.connection.settimeout(30)
self.server.proxy._register(self.connection)
def finish(self):
try:
super().finish()
finally:
self.server.proxy._unregister(self.connection)
def log_message(self, format, *args):
pass
def _error(self, status):
body = b'{"error":{"message":"Model proxy request rejected","type":"proxy_error"}}'
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Connection", "close")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.close_connection = True
self.wfile.write(body)
self.wfile.flush()
return len(body)
def do_POST(self):
self._request()
def do_GET(self):
self._request()
do_HEAD = do_GET
do_PUT = do_GET
do_PATCH = do_GET
do_DELETE = do_GET
do_OPTIONS = do_GET
def _request(self):
proxy = self.server.proxy
started = time.monotonic()
facts = dict(sequence=proxy._next_sequence(), request_body_sha256=None,
status=502, request_bytes=0, response_bytes=0)
connection = upstream_socket = None
response_started = False
try:
if ipaddress.ip_address(self.client_address[0]) not in proxy.network:
facts["status"] = 403
elif self.command != "POST":
facts["status"] = 405
elif self.path != proxy.request_path:
facts["status"] = 404
elif self.headers.get("Transfer-Encoding") is not None:
facts["status"] = 400
elif self.headers.get("Content-Length") is None:
facts["status"] = 411
else:
try:
length = int(self.headers["Content-Length"])
if not 0 < length <= MAX_BODY_BYTES:
facts["status"] = 413
else:
body = self.rfile.read(length)
facts.update(request_bytes=len(body), request_body_sha256=hashlib.sha256(body).hexdigest())
payload = json.loads(body, object_pairs_hook=_json_object)
facts["request_payload_sha256"] = hashlib.sha256(json.dumps(
payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
).encode("utf-8")).hexdigest()
if (len(body) != length or not isinstance(payload, dict) or
payload.get("model") != proxy.model or
(proxy.responses and payload.get("stream") is not True)):
facts["status"] = 400
else:
route = MODELS[proxy.model]
facts["upstream_statuses"] = []
for delay in RETRY_DELAYS + (None,):
response = None
try:
connection = (http.client.HTTPSConnection if route["https"] else
http.client.HTTPConnection)(route["upstream"], timeout=30)
connection.connect()
upstream_socket = connection.sock
proxy._register(upstream_socket)
upstream_socket.settimeout(UPSTREAM_IDLE_SECONDS)
connection.request("POST", route["upstream_path"], body=body,
headers={"Authorization": "Bearer " + proxy._key,
"Content-Type": "application/json",
"Connection": "close"})
response = connection.getresponse()
facts["upstream_statuses"].append(response.status)
except OSError:
facts["upstream_statuses"].append("connection_error")
if delay is None or proxy._closing:
raise
if response is not None and (delay is None or proxy._closing or
response.status not in RETRYABLE):
break
connection.close()
if upstream_socket is not None:
proxy._unregister(upstream_socket)
connection = upstream_socket = None
time.sleep(delay)
facts["status"] = response.status if 400 <= response.status <= 599 else 502
if response.status == 200:
facts["status"] = 200
content_type = response.getheader("Content-Type", "application/json")
if ("\r" in content_type or "\n" in content_type or
content_type.split(";", 1)[0].strip() not in ("application/json", "text/event-stream")):
content_type = "application/json"
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Connection", "close")
self.end_headers()
self.close_connection = True
response_started = True
pending = b""
if proxy.responses:
facts.update(responses_terminal=None, responses_terminal_count=0,
responses_stream_valid=True, responses_stream_finished=False)
while True:
chunk = response.read1(64 * 1024)
if not chunk:
break
if proxy.responses:
pending += chunk
lines = pending.split(b"\n")
pending = lines.pop()
if len(pending) > MAX_BODY_BYTES:
raise OSError("oversized Responses event")
for line in lines:
if not line.startswith(b"data:") or line[5:].strip() == b"[DONE]":
continue
try:
event = json.loads(line[5:])
kind = event.get("type")
if kind in ("response.completed", "response.incomplete", "response.failed", "error"):
facts["responses_terminal_count"] += 1
facts["responses_terminal"] = kind
facts["response_model"] = event.get("response", {}).get("model")
facts["response_status"] = event.get("response", {}).get("status")
except (ValueError, AttributeError, TypeError):
facts["responses_stream_valid"] = False
self.wfile.write(chunk)
self.wfile.flush()
facts["response_bytes"] += len(chunk)
if proxy.responses:
facts["responses_stream_valid"] &= not bool(pending.strip())
facts["responses_stream_finished"] = True
return
except (ValueError, TypeError, UnicodeDecodeError):
facts["status"] = 400
response_started = True
facts["response_bytes"] = self._error(facts["status"])
except Exception:
if not response_started:
try:
facts["status"] = 502
facts["response_bytes"] = self._error(502)
except OSError:
pass
finally:
if connection is not None:
connection.close()
if upstream_socket is not None:
proxy._unregister(upstream_socket)
facts["duration_seconds"] = round(time.monotonic() - started, 6)
proxy._record(facts)
class ModelProxy:
"""Use one private Docker bridge and a fresh log for each rollout attempt."""
def __init__(self, credential_file, bind, allowed_network, log_path, model=MODEL):
if model not in MODELS:
raise ValueError("unsupported rollout model")
self.model = model
self.request_path = MODELS[model]["path"]
self.responses = MODELS[model]["api"] == "openai-responses"
self.network = ipaddress.ip_network(allowed_network)
address = ipaddress.ip_address(bind)
if (address.version != 4 or self.network.version != 4 or not self.network.is_private or
not address.is_private or address.is_unspecified or address not in self.network):
raise ValueError("bind must be an IPv4 address inside the allowed private network")
self._key = _credential(credential_file)
self._bind = str(address)
self._log_path = _path(log_path)
self._lock = threading.Lock()
self._sequence = 0
self._sockets = set()
self._closing = False
self._server = self._thread = self._log = None
self.base_url = None
def _register(self, connection):
with self._lock:
if self._closing:
connection.close()
raise RuntimeError("proxy has closed")
self._sockets.add(connection)
def _unregister(self, connection):
with self._lock:
self._sockets.discard(connection)
def _next_sequence(self):
with self._lock:
self._sequence += 1
return self._sequence
def _record(self, facts):
with self._lock:
if self._log is not None:
self._log.write(json.dumps(facts, sort_keys=True) + "\n")
self._log.flush()
def __enter__(self):
if self._server is not None or self._closing:
raise RuntimeError("ModelProxy is single-use")
descriptor = os.open(self._log_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
self._log = os.fdopen(descriptor, "w", encoding="utf-8")
try:
self._server = _Server((self._bind, 0), _Handler)
self._server.proxy = self
self.base_url = "http://{}:{}/v1".format(self._bind, self._server.server_port)
self._thread = threading.Thread(target=self._server.serve_forever, kwargs={"poll_interval": 0.1}, daemon=True)
self._thread.start()
return self
except Exception:
self.__exit__(None, None, None)
raise
def __exit__(self, exc_type, exc_value, traceback):
with self._lock:
self._closing = True
connections = list(self._sockets)
for connection in connections:
try:
connection.shutdown(socket.SHUT_RDWR)
except OSError:
pass
connection.close()
if self._thread is not None and self._thread.is_alive():
self._server.shutdown()
self._thread.join()
if self._server is not None:
self._server.server_close()
with self._lock:
if self._log is not None:
self._log.close()
self._log = None
self._key = None