Batch 6 lost 26 of 103 attempts to Qwen requests that sent their preamble and then stalled; Pi's own Node fetch gave up after 300 s and reported "terminated". - model_proxy: hold the SSE preamble until the first generation event, retry a stall (180 s) within the existing retry budget, and send SSE keepalive comments so Pi never times out first; record first_event_seconds. - report: name one exclusion reason per incomplete attempt and compute the JOBS lines still owed, with a per-line resample budget. - batch: run a JOBS file to completion, resume from receipts, resample infrastructure failures, pause after three environment failures in a row, forward SIGTERM to running controllers, and refuse a second runner. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
392 lines
17 KiB
Python
392 lines
17 KiB
Python
"""One-attempt host proxy: the provider credential never enters the Agent."""
|
|
|
|
import hashlib
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
import http.client
|
|
import ipaddress
|
|
import json
|
|
import os
|
|
import queue
|
|
import socket
|
|
import stat
|
|
import threading
|
|
import time
|
|
|
|
from .core import _path
|
|
|
|
# Fixed upstream routes and per-response output limits; the Agent never chooses either.
|
|
# Sampling is left at each service's defaults. Both limits were accepted upstream on 2026-10-12/13.
|
|
MODELS = {
|
|
"glm-5.3": dict(api="openai-completions", path="/v1/chat/completions", https=True,
|
|
upstream="open.bigmodel.cn", upstream_path="/api/coding/paas/v4/chat/completions",
|
|
max_tokens=131072, max_tokens_field="max_tokens"),
|
|
# User-specified plain-HTTP service; the Bearer key crosses the network unencrypted.
|
|
"qwen3.8-27b-nvfp4": dict(api="openai-responses", path="/v1/responses", https=False,
|
|
upstream="114.111.18.115:18080", upstream_path="/v1/responses",
|
|
max_tokens=65536, max_tokens_field="max_output_tokens"),
|
|
}
|
|
MODEL = "glm-5.3"
|
|
QWEN_MODEL = "qwen3.8-27b-nvfp4"
|
|
MODEL_IDS = tuple(MODELS)
|
|
MAX_BODY_BYTES = 4 * 1024 * 1024
|
|
# Qwen streams may pause for over 300 seconds; allow 900 seconds before treating upstream silence as failure.
|
|
UPSTREAM_IDLE_SECONDS = 900
|
|
# Transient upstream failures and stalls are retried before any generated byte reaches Pi,
|
|
# so its trace stays one real response.
|
|
RETRYABLE = (429, 500, 502, 503, 504)
|
|
RETRY_DELAYS = (2, 5)
|
|
# Pi's Node fetch gives up after 300 s without a byte; SSE comments keep it waiting while the proxy owns the limits.
|
|
KEEPALIVE_SECONDS = 30
|
|
# A request with no generation event by now is stalled and retried; tune from broker first_event_seconds.
|
|
STALL_SECONDS = 180
|
|
PREAMBLE = ("response.created", "response.in_progress")
|
|
TERMINALS = ("response.completed", "response.incomplete", "response.failed", "error")
|
|
|
|
|
|
def _credential(path):
|
|
path = _path(path)
|
|
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
|
with os.fdopen(descriptor, "rb") as stream:
|
|
info = os.fstat(stream.fileno())
|
|
if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077:
|
|
raise ValueError("credential must be a private regular file (0600)")
|
|
raw = stream.read(64 * 1024 + 1)
|
|
try:
|
|
if len(raw) > 64 * 1024:
|
|
raise ValueError
|
|
value = json.loads(raw)["model_api_key"]
|
|
if not isinstance(value, str) or not value.strip() or "\r" in value or "\n" in value:
|
|
raise ValueError
|
|
return value
|
|
except (ValueError, KeyError, TypeError, UnicodeDecodeError):
|
|
raise ValueError("credential must contain a nonempty model_api_key") from None
|
|
|
|
|
|
def _json_object(pairs):
|
|
value = {}
|
|
for key, item in pairs:
|
|
if key in value:
|
|
raise ValueError("duplicate JSON key")
|
|
value[key] = item
|
|
return value
|
|
|
|
|
|
class _Server(ThreadingHTTPServer):
|
|
def handle_error(self, request, client_address):
|
|
pass # Provider or client exception text must not reach the host log.
|
|
|
|
|
|
class _Handler(BaseHTTPRequestHandler):
|
|
protocol_version = "HTTP/1.1"
|
|
|
|
def setup(self):
|
|
super().setup()
|
|
self.connection.settimeout(30)
|
|
self.server.proxy._register(self.connection)
|
|
|
|
def finish(self):
|
|
try:
|
|
super().finish()
|
|
finally:
|
|
self.server.proxy._unregister(self.connection)
|
|
|
|
def log_message(self, format, *args):
|
|
pass
|
|
|
|
def _error(self, status):
|
|
body = b'{"error":{"message":"Model proxy request rejected","type":"proxy_error"}}'
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Connection", "close")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.close_connection = True
|
|
self.wfile.write(body)
|
|
self.wfile.flush()
|
|
return len(body)
|
|
|
|
def _release(self, connection, upstream_socket):
|
|
"""Close one upstream attempt; shutdown first wakes a reader thread still blocked on it."""
|
|
try:
|
|
upstream_socket.shutdown(socket.SHUT_RDWR)
|
|
except (AttributeError, OSError):
|
|
pass
|
|
connection.close()
|
|
if upstream_socket is not None:
|
|
self.server.proxy._unregister(upstream_socket)
|
|
|
|
def _send(self, data, facts):
|
|
self.wfile.write(data)
|
|
self.wfile.flush()
|
|
facts["response_bytes"] += len(data)
|
|
|
|
def _forward(self, response, facts, sse):
|
|
"""Relay one upstream 200; False when it stalled or broke before any generation event reached Pi."""
|
|
proxy = self.server.proxy
|
|
chunks = queue.Queue()
|
|
|
|
def read():
|
|
try:
|
|
while True:
|
|
chunk = response.read1(64 * 1024)
|
|
chunks.put(chunk)
|
|
if not chunk:
|
|
return
|
|
except Exception as error:
|
|
chunks.put(error)
|
|
|
|
threading.Thread(target=read, daemon=True).start()
|
|
started = time.monotonic()
|
|
held = pending = b""
|
|
live = not sse # Plain JSON bodies have no events to wait for.
|
|
if proxy.responses:
|
|
facts.update(responses_terminal=None, responses_terminal_count=0,
|
|
responses_stream_valid=True, responses_stream_finished=False)
|
|
while True:
|
|
try:
|
|
chunk = chunks.get(timeout=KEEPALIVE_SECONDS)
|
|
except queue.Empty:
|
|
if not live and time.monotonic() - started >= STALL_SECONDS:
|
|
return False
|
|
if sse:
|
|
self._send(b": keepalive\n\n", facts)
|
|
continue
|
|
if isinstance(chunk, Exception):
|
|
if live:
|
|
raise chunk
|
|
return False
|
|
if not chunk:
|
|
break
|
|
if sse or proxy.responses:
|
|
pending += chunk
|
|
lines = pending.split(b"\n")
|
|
pending = lines.pop()
|
|
if len(pending) > MAX_BODY_BYTES:
|
|
raise OSError("oversized stream event")
|
|
for line in lines:
|
|
if not line.startswith(b"data:"):
|
|
continue
|
|
kind = None
|
|
if line[5:].strip() != b"[DONE]":
|
|
try:
|
|
event = json.loads(line[5:])
|
|
kind = event.get("type")
|
|
if proxy.responses and kind in TERMINALS:
|
|
facts["responses_terminal_count"] += 1
|
|
facts["responses_terminal"] = kind
|
|
facts["response_model"] = event.get("response", {}).get("model")
|
|
facts["response_status"] = event.get("response", {}).get("status")
|
|
except (ValueError, AttributeError, TypeError):
|
|
if proxy.responses:
|
|
facts["responses_stream_valid"] = False
|
|
if not live and kind not in PREAMBLE:
|
|
live = True
|
|
facts["first_event_seconds"] = round(time.monotonic() - started, 3)
|
|
held += chunk
|
|
if live or len(held) > MAX_BODY_BYTES:
|
|
live = True
|
|
self._send(held, facts)
|
|
held = b""
|
|
if not live:
|
|
return False
|
|
if proxy.responses:
|
|
facts["responses_stream_valid"] &= not bool(pending.strip())
|
|
facts["responses_stream_finished"] = True
|
|
return True
|
|
|
|
def do_POST(self):
|
|
self._request()
|
|
|
|
def do_GET(self):
|
|
self._request()
|
|
|
|
do_HEAD = do_GET
|
|
do_PUT = do_GET
|
|
do_PATCH = do_GET
|
|
do_DELETE = do_GET
|
|
do_OPTIONS = do_GET
|
|
|
|
def _request(self):
|
|
proxy = self.server.proxy
|
|
started = time.monotonic()
|
|
facts = dict(sequence=proxy._next_sequence(), request_body_sha256=None,
|
|
status=502, request_bytes=0, response_bytes=0)
|
|
connection = upstream_socket = None
|
|
response_started = False
|
|
try:
|
|
if ipaddress.ip_address(self.client_address[0]) not in proxy.network:
|
|
facts["status"] = 403
|
|
elif self.command != "POST":
|
|
facts["status"] = 405
|
|
elif self.path != proxy.request_path:
|
|
facts["status"] = 404
|
|
elif self.headers.get("Transfer-Encoding") is not None:
|
|
facts["status"] = 400
|
|
elif self.headers.get("Content-Length") is None:
|
|
facts["status"] = 411
|
|
else:
|
|
try:
|
|
length = int(self.headers["Content-Length"])
|
|
if not 0 < length <= MAX_BODY_BYTES:
|
|
facts["status"] = 413
|
|
else:
|
|
body = self.rfile.read(length)
|
|
facts.update(request_bytes=len(body), request_body_sha256=hashlib.sha256(body).hexdigest())
|
|
payload = json.loads(body, object_pairs_hook=_json_object)
|
|
facts["request_payload_sha256"] = hashlib.sha256(json.dumps(
|
|
payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
|
|
).encode("utf-8")).hexdigest()
|
|
if (len(body) != length or not isinstance(payload, dict) or
|
|
payload.get("model") != proxy.model or
|
|
(proxy.responses and payload.get("stream") is not True)):
|
|
facts["status"] = 400
|
|
else:
|
|
route = MODELS[proxy.model]
|
|
statuses = facts["upstream_statuses"] = []
|
|
for delay in RETRY_DELAYS + (None,):
|
|
response = None
|
|
try:
|
|
connection = (http.client.HTTPSConnection if route["https"] else
|
|
http.client.HTTPConnection)(route["upstream"], timeout=30)
|
|
connection.connect()
|
|
upstream_socket = connection.sock
|
|
proxy._register(upstream_socket)
|
|
upstream_socket.settimeout(UPSTREAM_IDLE_SECONDS)
|
|
connection.request("POST", route["upstream_path"], body=body,
|
|
headers={"Authorization": "Bearer " + proxy._key,
|
|
"Content-Type": "application/json",
|
|
"Connection": "close"})
|
|
response = connection.getresponse()
|
|
statuses.append(response.status)
|
|
except OSError:
|
|
statuses.append("connection_error")
|
|
if delay is None or proxy._closing:
|
|
raise
|
|
if response is not None and response.status == 200:
|
|
content_type = response.getheader("Content-Type", "application/json")
|
|
if ("\r" in content_type or "\n" in content_type or
|
|
content_type.split(";", 1)[0].strip() not in ("application/json", "text/event-stream")):
|
|
content_type = "application/json"
|
|
if not response_started:
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", content_type)
|
|
self.send_header("Connection", "close")
|
|
self.end_headers()
|
|
self.close_connection = True
|
|
response_started = True
|
|
facts["status"] = 200
|
|
if self._forward(response, facts, content_type.startswith("text/event-stream")):
|
|
return
|
|
statuses[-1] = "stalled"
|
|
if delay is None or proxy._closing:
|
|
raise OSError("upstream stalled before generating")
|
|
elif response is not None and (delay is None or proxy._closing or
|
|
response.status not in RETRYABLE):
|
|
break
|
|
self._release(connection, upstream_socket)
|
|
connection = upstream_socket = None
|
|
time.sleep(delay)
|
|
facts["status"] = response.status if 400 <= response.status <= 599 else 502
|
|
if response_started:
|
|
raise OSError("upstream failed after the response began")
|
|
except (ValueError, TypeError, UnicodeDecodeError):
|
|
facts["status"] = 400
|
|
response_started = True
|
|
facts["response_bytes"] = self._error(facts["status"])
|
|
except Exception:
|
|
if not response_started:
|
|
try:
|
|
facts["status"] = 502
|
|
facts["response_bytes"] = self._error(502)
|
|
except OSError:
|
|
pass
|
|
finally:
|
|
if connection is not None:
|
|
self._release(connection, upstream_socket)
|
|
facts["duration_seconds"] = round(time.monotonic() - started, 6)
|
|
proxy._record(facts)
|
|
|
|
|
|
class ModelProxy:
|
|
"""Use one private Docker bridge and a fresh log for each rollout attempt."""
|
|
|
|
def __init__(self, credential_file, bind, allowed_network, log_path, model=MODEL):
|
|
if model not in MODELS:
|
|
raise ValueError("unsupported rollout model")
|
|
self.model = model
|
|
self.request_path = MODELS[model]["path"]
|
|
self.responses = MODELS[model]["api"] == "openai-responses"
|
|
self.network = ipaddress.ip_network(allowed_network)
|
|
address = ipaddress.ip_address(bind)
|
|
if (address.version != 4 or self.network.version != 4 or not self.network.is_private or
|
|
not address.is_private or address.is_unspecified or address not in self.network):
|
|
raise ValueError("bind must be an IPv4 address inside the allowed private network")
|
|
self._key = _credential(credential_file)
|
|
self._bind = str(address)
|
|
self._log_path = _path(log_path)
|
|
self._lock = threading.Lock()
|
|
self._sequence = 0
|
|
self._sockets = set()
|
|
self._closing = False
|
|
self._server = self._thread = self._log = None
|
|
self.base_url = None
|
|
|
|
def _register(self, connection):
|
|
with self._lock:
|
|
if self._closing:
|
|
connection.close()
|
|
raise RuntimeError("proxy has closed")
|
|
self._sockets.add(connection)
|
|
|
|
def _unregister(self, connection):
|
|
with self._lock:
|
|
self._sockets.discard(connection)
|
|
|
|
def _next_sequence(self):
|
|
with self._lock:
|
|
self._sequence += 1
|
|
return self._sequence
|
|
|
|
def _record(self, facts):
|
|
with self._lock:
|
|
if self._log is not None:
|
|
self._log.write(json.dumps(facts, sort_keys=True) + "\n")
|
|
self._log.flush()
|
|
|
|
def __enter__(self):
|
|
if self._server is not None or self._closing:
|
|
raise RuntimeError("ModelProxy is single-use")
|
|
descriptor = os.open(self._log_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
|
self._log = os.fdopen(descriptor, "w", encoding="utf-8")
|
|
try:
|
|
self._server = _Server((self._bind, 0), _Handler)
|
|
self._server.proxy = self
|
|
self.base_url = "http://{}:{}/v1".format(self._bind, self._server.server_port)
|
|
self._thread = threading.Thread(target=self._server.serve_forever, kwargs={"poll_interval": 0.1}, daemon=True)
|
|
self._thread.start()
|
|
return self
|
|
except Exception:
|
|
self.__exit__(None, None, None)
|
|
raise
|
|
|
|
def __exit__(self, exc_type, exc_value, traceback):
|
|
with self._lock:
|
|
self._closing = True
|
|
connections = list(self._sockets)
|
|
for connection in connections:
|
|
try:
|
|
connection.shutdown(socket.SHUT_RDWR)
|
|
except OSError:
|
|
pass
|
|
connection.close()
|
|
if self._thread is not None and self._thread.is_alive():
|
|
self._server.shutdown()
|
|
self._thread.join()
|
|
if self._server is not None:
|
|
self._server.server_close()
|
|
with self._lock:
|
|
if self._log is not None:
|
|
self._log.close()
|
|
self._log = None
|
|
self._key = None
|