data-gen-ebd/rollout/model_proxy.py
amos 3adab31da7 Self-heal rollouts: retry stalled model streams and run batches from receipts
Batch 6 lost 26 of 103 attempts to Qwen requests that sent their preamble and
then stalled; Pi's own Node fetch gave up after 300 s and reported "terminated".

- model_proxy: hold the SSE preamble until the first generation event, retry a
  stall (180 s) within the existing retry budget, and send SSE keepalive
  comments so Pi never times out first; record first_event_seconds.
- report: name one exclusion reason per incomplete attempt and compute the
  JOBS lines still owed, with a per-line resample budget.
- batch: run a JOBS file to completion, resume from receipts, resample
  infrastructure failures, pause after three environment failures in a row,
  forward SIGTERM to running controllers, and refuse a second runner.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 08:28:53 +08:00

392 lines
17 KiB
Python

"""One-attempt host proxy: the provider credential never enters the Agent."""
import hashlib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import http.client
import ipaddress
import json
import os
import queue
import socket
import stat
import threading
import time
from .core import _path
# Fixed upstream routes and per-response output limits; the Agent never chooses either.
# Sampling is left at each service's defaults. Both limits were accepted upstream on 2026-10-12/13.
MODELS = {
"glm-5.3": dict(api="openai-completions", path="/v1/chat/completions", https=True,
upstream="open.bigmodel.cn", upstream_path="/api/coding/paas/v4/chat/completions",
max_tokens=131072, max_tokens_field="max_tokens"),
# User-specified plain-HTTP service; the Bearer key crosses the network unencrypted.
"qwen3.8-27b-nvfp4": dict(api="openai-responses", path="/v1/responses", https=False,
upstream="114.111.18.115:18080", upstream_path="/v1/responses",
max_tokens=65536, max_tokens_field="max_output_tokens"),
}
MODEL = "glm-5.3"
QWEN_MODEL = "qwen3.8-27b-nvfp4"
MODEL_IDS = tuple(MODELS)
MAX_BODY_BYTES = 4 * 1024 * 1024
# Qwen streams may pause for over 300 seconds; allow 900 seconds before treating upstream silence as failure.
UPSTREAM_IDLE_SECONDS = 900
# Transient upstream failures and stalls are retried before any generated byte reaches Pi,
# so its trace stays one real response.
RETRYABLE = (429, 500, 502, 503, 504)
RETRY_DELAYS = (2, 5)
# Pi's Node fetch gives up after 300 s without a byte; SSE comments keep it waiting while the proxy owns the limits.
KEEPALIVE_SECONDS = 30
# A request with no generation event by now is stalled and retried; tune from broker first_event_seconds.
STALL_SECONDS = 180
PREAMBLE = ("response.created", "response.in_progress")
TERMINALS = ("response.completed", "response.incomplete", "response.failed", "error")
def _credential(path):
path = _path(path)
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
with os.fdopen(descriptor, "rb") as stream:
info = os.fstat(stream.fileno())
if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077:
raise ValueError("credential must be a private regular file (0600)")
raw = stream.read(64 * 1024 + 1)
try:
if len(raw) > 64 * 1024:
raise ValueError
value = json.loads(raw)["model_api_key"]
if not isinstance(value, str) or not value.strip() or "\r" in value or "\n" in value:
raise ValueError
return value
except (ValueError, KeyError, TypeError, UnicodeDecodeError):
raise ValueError("credential must contain a nonempty model_api_key") from None
def _json_object(pairs):
value = {}
for key, item in pairs:
if key in value:
raise ValueError("duplicate JSON key")
value[key] = item
return value
class _Server(ThreadingHTTPServer):
def handle_error(self, request, client_address):
pass # Provider or client exception text must not reach the host log.
class _Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def setup(self):
super().setup()
self.connection.settimeout(30)
self.server.proxy._register(self.connection)
def finish(self):
try:
super().finish()
finally:
self.server.proxy._unregister(self.connection)
def log_message(self, format, *args):
pass
def _error(self, status):
body = b'{"error":{"message":"Model proxy request rejected","type":"proxy_error"}}'
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Connection", "close")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.close_connection = True
self.wfile.write(body)
self.wfile.flush()
return len(body)
def _release(self, connection, upstream_socket):
"""Close one upstream attempt; shutdown first wakes a reader thread still blocked on it."""
try:
upstream_socket.shutdown(socket.SHUT_RDWR)
except (AttributeError, OSError):
pass
connection.close()
if upstream_socket is not None:
self.server.proxy._unregister(upstream_socket)
def _send(self, data, facts):
self.wfile.write(data)
self.wfile.flush()
facts["response_bytes"] += len(data)
def _forward(self, response, facts, sse):
"""Relay one upstream 200; False when it stalled or broke before any generation event reached Pi."""
proxy = self.server.proxy
chunks = queue.Queue()
def read():
try:
while True:
chunk = response.read1(64 * 1024)
chunks.put(chunk)
if not chunk:
return
except Exception as error:
chunks.put(error)
threading.Thread(target=read, daemon=True).start()
started = time.monotonic()
held = pending = b""
live = not sse # Plain JSON bodies have no events to wait for.
if proxy.responses:
facts.update(responses_terminal=None, responses_terminal_count=0,
responses_stream_valid=True, responses_stream_finished=False)
while True:
try:
chunk = chunks.get(timeout=KEEPALIVE_SECONDS)
except queue.Empty:
if not live and time.monotonic() - started >= STALL_SECONDS:
return False
if sse:
self._send(b": keepalive\n\n", facts)
continue
if isinstance(chunk, Exception):
if live:
raise chunk
return False
if not chunk:
break
if sse or proxy.responses:
pending += chunk
lines = pending.split(b"\n")
pending = lines.pop()
if len(pending) > MAX_BODY_BYTES:
raise OSError("oversized stream event")
for line in lines:
if not line.startswith(b"data:"):
continue
kind = None
if line[5:].strip() != b"[DONE]":
try:
event = json.loads(line[5:])
kind = event.get("type")
if proxy.responses and kind in TERMINALS:
facts["responses_terminal_count"] += 1
facts["responses_terminal"] = kind
facts["response_model"] = event.get("response", {}).get("model")
facts["response_status"] = event.get("response", {}).get("status")
except (ValueError, AttributeError, TypeError):
if proxy.responses:
facts["responses_stream_valid"] = False
if not live and kind not in PREAMBLE:
live = True
facts["first_event_seconds"] = round(time.monotonic() - started, 3)
held += chunk
if live or len(held) > MAX_BODY_BYTES:
live = True
self._send(held, facts)
held = b""
if not live:
return False
if proxy.responses:
facts["responses_stream_valid"] &= not bool(pending.strip())
facts["responses_stream_finished"] = True
return True
def do_POST(self):
self._request()
def do_GET(self):
self._request()
do_HEAD = do_GET
do_PUT = do_GET
do_PATCH = do_GET
do_DELETE = do_GET
do_OPTIONS = do_GET
def _request(self):
proxy = self.server.proxy
started = time.monotonic()
facts = dict(sequence=proxy._next_sequence(), request_body_sha256=None,
status=502, request_bytes=0, response_bytes=0)
connection = upstream_socket = None
response_started = False
try:
if ipaddress.ip_address(self.client_address[0]) not in proxy.network:
facts["status"] = 403
elif self.command != "POST":
facts["status"] = 405
elif self.path != proxy.request_path:
facts["status"] = 404
elif self.headers.get("Transfer-Encoding") is not None:
facts["status"] = 400
elif self.headers.get("Content-Length") is None:
facts["status"] = 411
else:
try:
length = int(self.headers["Content-Length"])
if not 0 < length <= MAX_BODY_BYTES:
facts["status"] = 413
else:
body = self.rfile.read(length)
facts.update(request_bytes=len(body), request_body_sha256=hashlib.sha256(body).hexdigest())
payload = json.loads(body, object_pairs_hook=_json_object)
facts["request_payload_sha256"] = hashlib.sha256(json.dumps(
payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False,
).encode("utf-8")).hexdigest()
if (len(body) != length or not isinstance(payload, dict) or
payload.get("model") != proxy.model or
(proxy.responses and payload.get("stream") is not True)):
facts["status"] = 400
else:
route = MODELS[proxy.model]
statuses = facts["upstream_statuses"] = []
for delay in RETRY_DELAYS + (None,):
response = None
try:
connection = (http.client.HTTPSConnection if route["https"] else
http.client.HTTPConnection)(route["upstream"], timeout=30)
connection.connect()
upstream_socket = connection.sock
proxy._register(upstream_socket)
upstream_socket.settimeout(UPSTREAM_IDLE_SECONDS)
connection.request("POST", route["upstream_path"], body=body,
headers={"Authorization": "Bearer " + proxy._key,
"Content-Type": "application/json",
"Connection": "close"})
response = connection.getresponse()
statuses.append(response.status)
except OSError:
statuses.append("connection_error")
if delay is None or proxy._closing:
raise
if response is not None and response.status == 200:
content_type = response.getheader("Content-Type", "application/json")
if ("\r" in content_type or "\n" in content_type or
content_type.split(";", 1)[0].strip() not in ("application/json", "text/event-stream")):
content_type = "application/json"
if not response_started:
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Connection", "close")
self.end_headers()
self.close_connection = True
response_started = True
facts["status"] = 200
if self._forward(response, facts, content_type.startswith("text/event-stream")):
return
statuses[-1] = "stalled"
if delay is None or proxy._closing:
raise OSError("upstream stalled before generating")
elif response is not None and (delay is None or proxy._closing or
response.status not in RETRYABLE):
break
self._release(connection, upstream_socket)
connection = upstream_socket = None
time.sleep(delay)
facts["status"] = response.status if 400 <= response.status <= 599 else 502
if response_started:
raise OSError("upstream failed after the response began")
except (ValueError, TypeError, UnicodeDecodeError):
facts["status"] = 400
response_started = True
facts["response_bytes"] = self._error(facts["status"])
except Exception:
if not response_started:
try:
facts["status"] = 502
facts["response_bytes"] = self._error(502)
except OSError:
pass
finally:
if connection is not None:
self._release(connection, upstream_socket)
facts["duration_seconds"] = round(time.monotonic() - started, 6)
proxy._record(facts)
class ModelProxy:
"""Use one private Docker bridge and a fresh log for each rollout attempt."""
def __init__(self, credential_file, bind, allowed_network, log_path, model=MODEL):
if model not in MODELS:
raise ValueError("unsupported rollout model")
self.model = model
self.request_path = MODELS[model]["path"]
self.responses = MODELS[model]["api"] == "openai-responses"
self.network = ipaddress.ip_network(allowed_network)
address = ipaddress.ip_address(bind)
if (address.version != 4 or self.network.version != 4 or not self.network.is_private or
not address.is_private or address.is_unspecified or address not in self.network):
raise ValueError("bind must be an IPv4 address inside the allowed private network")
self._key = _credential(credential_file)
self._bind = str(address)
self._log_path = _path(log_path)
self._lock = threading.Lock()
self._sequence = 0
self._sockets = set()
self._closing = False
self._server = self._thread = self._log = None
self.base_url = None
def _register(self, connection):
with self._lock:
if self._closing:
connection.close()
raise RuntimeError("proxy has closed")
self._sockets.add(connection)
def _unregister(self, connection):
with self._lock:
self._sockets.discard(connection)
def _next_sequence(self):
with self._lock:
self._sequence += 1
return self._sequence
def _record(self, facts):
with self._lock:
if self._log is not None:
self._log.write(json.dumps(facts, sort_keys=True) + "\n")
self._log.flush()
def __enter__(self):
if self._server is not None or self._closing:
raise RuntimeError("ModelProxy is single-use")
descriptor = os.open(self._log_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
self._log = os.fdopen(descriptor, "w", encoding="utf-8")
try:
self._server = _Server((self._bind, 0), _Handler)
self._server.proxy = self
self.base_url = "http://{}:{}/v1".format(self._bind, self._server.server_port)
self._thread = threading.Thread(target=self._server.serve_forever, kwargs={"poll_interval": 0.1}, daemon=True)
self._thread.start()
return self
except Exception:
self.__exit__(None, None, None)
raise
def __exit__(self, exc_type, exc_value, traceback):
with self._lock:
self._closing = True
connections = list(self._sockets)
for connection in connections:
try:
connection.shutdown(socket.SHUT_RDWR)
except OSError:
pass
connection.close()
if self._thread is not None and self._thread.is_alive():
self._server.shutdown()
self._thread.join()
if self._server is not None:
self._server.server_close()
with self._lock:
if self._log is not None:
self._log.close()
self._log = None
self._key = None