Let a credential pick one of the four Qwen instances

The Qwen service runs four identical instances on 114.111.18.115:18080-18083. A
credential may now carry upstream_port, limited to the ports the route lists, so
rollout.batch's least-busy key choice spreads attempts across instances. The broker
log records the upstream each request used.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
amos 2026-10-30 14:58:12 +08:00
parent 078efb55e2
commit c93aeb8fd8
2 changed files with 36 additions and 5 deletions

View File

@ -24,6 +24,8 @@ MODELS = {
# User-specified plain-HTTP service; the Bearer key crosses the network unencrypted.
"qwen3.8-27b-nvfp4": dict(api="openai-responses", path="/v1/responses", https=False,
upstream="114.111.18.115:18080", upstream_path="/v1/responses",
# Four identical instances of the same model; a credential may pick one of them.
upstream_ports=(18080, 18081, 18082, 18083),
max_tokens=65536, max_tokens_field="max_output_tokens"),
}
MODEL = "glm-5.3"
@ -44,7 +46,9 @@ PREAMBLE = ("response.created", "response.in_progress")
TERMINALS = ("response.completed", "response.incomplete", "response.failed", "error")
def _credential(path):
def _credential(path, route):
"""Return (key, upstream host:port) from a private JSON credential; `upstream_port` picks one of the route's
listed instances."""
path = _path(path)
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
with os.fdopen(descriptor, "rb") as stream:
@ -55,12 +59,18 @@ def _credential(path):
try:
if len(raw) > 64 * 1024:
raise ValueError
value = json.loads(raw)["model_api_key"]
data = json.loads(raw)
value = data["model_api_key"]
if not isinstance(value, str) or not value.strip() or "\r" in value or "\n" in value:
raise ValueError
return value
except (ValueError, KeyError, TypeError, UnicodeDecodeError):
raise ValueError("credential must contain a nonempty model_api_key") from None
port = data.get("upstream_port")
if port is None:
return value, route["upstream"]
if type(port) is not int or port not in route.get("upstream_ports", ()):
raise ValueError("credential upstream_port is not one of this model's instances")
return value, route["upstream"].rsplit(":", 1)[0] + ":%d" % port
def _json_object(pairs):
@ -258,11 +268,12 @@ class _Handler(BaseHTTPRequestHandler):
else:
route = MODELS[proxy.model]
statuses = facts["upstream_statuses"] = []
facts["upstream"] = proxy.upstream
for delay in RETRY_DELAYS + (None,):
response = None
try:
connection = (http.client.HTTPSConnection if route["https"] else
http.client.HTTPConnection)(route["upstream"], timeout=30)
http.client.HTTPConnection)(proxy.upstream, timeout=30)
connection.connect()
upstream_socket = connection.sock
proxy._register(upstream_socket)
@ -343,7 +354,7 @@ class ModelProxy:
raise ValueError("port must be 0..65535")
self._token = token
self._port = port
self._key = _credential(credential_file)
self._key, self.upstream = _credential(credential_file, MODELS[model])
self._bind = str(address)
self._log_path = _path(log_path)
self._lock = threading.Lock()

View File

@ -16,6 +16,7 @@ from urllib.parse import urlsplit
from rollout import model_proxy
from rollout.model_proxy import MAX_BODY_BYTES, ModelProxy, MODEL, MODELS, QWEN_MODEL, UPSTREAM_IDLE_SECONDS
from rollout.model_proxy import _credential
ROOT = Path(__file__).resolve().parents[1]
FAST = dict(STALL_SECONDS=0.4, KEEPALIVE_SECONDS=0.1, RETRY_DELAYS=(0, 0))
@ -412,3 +413,22 @@ console.log(JSON.stringify({complete: summary.complete, error: last?.message?.er
if __name__ == "__main__":
unittest.main()
class CredentialInstanceTests(unittest.TestCase):
def write(self, data):
folder = os.path.realpath(tempfile.mkdtemp()) # macOS /var is a symlink, which credentials refuse
self.addCleanup(shutil.rmtree, folder)
path = Path(folder, "key.json")
path.write_text(json.dumps(data))
path.chmod(0o600)
return path
def test_a_credential_may_pick_only_a_listed_instance_of_its_model(self):
qwen, glm = MODELS[QWEN_MODEL], MODELS["glm-5.3"]
self.assertEqual(_credential(self.write(dict(model_api_key="k")), qwen), ("k", "114.111.18.115:18080"))
self.assertEqual(_credential(self.write(dict(model_api_key="k", upstream_port=18083)), qwen),
("k", "114.111.18.115:18083"))
for port, route in ((18084, qwen), ("18081", qwen), (18081, glm)):
with self.assertRaises(ValueError):
_credential(self.write(dict(model_api_key="k", upstream_port=port)), route)