Let a credential pick one of the four Qwen instances
The Qwen service runs four identical instances on 114.111.18.115:18080-18083. A credential may now carry upstream_port, limited to the ports the route lists, so rollout.batch's least-busy key choice spreads attempts across instances. The broker log records the upstream each request used. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
078efb55e2
commit
c93aeb8fd8
@ -24,6 +24,8 @@ MODELS = {
|
||||
# User-specified plain-HTTP service; the Bearer key crosses the network unencrypted.
|
||||
"qwen3.8-27b-nvfp4": dict(api="openai-responses", path="/v1/responses", https=False,
|
||||
upstream="114.111.18.115:18080", upstream_path="/v1/responses",
|
||||
# Four identical instances of the same model; a credential may pick one of them.
|
||||
upstream_ports=(18080, 18081, 18082, 18083),
|
||||
max_tokens=65536, max_tokens_field="max_output_tokens"),
|
||||
}
|
||||
MODEL = "glm-5.3"
|
||||
@ -44,7 +46,9 @@ PREAMBLE = ("response.created", "response.in_progress")
|
||||
TERMINALS = ("response.completed", "response.incomplete", "response.failed", "error")
|
||||
|
||||
|
||||
def _credential(path):
|
||||
def _credential(path, route):
|
||||
"""Return (key, upstream host:port) from a private JSON credential; `upstream_port` picks one of the route's
|
||||
listed instances."""
|
||||
path = _path(path)
|
||||
descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
||||
with os.fdopen(descriptor, "rb") as stream:
|
||||
@ -55,12 +59,18 @@ def _credential(path):
|
||||
try:
|
||||
if len(raw) > 64 * 1024:
|
||||
raise ValueError
|
||||
value = json.loads(raw)["model_api_key"]
|
||||
data = json.loads(raw)
|
||||
value = data["model_api_key"]
|
||||
if not isinstance(value, str) or not value.strip() or "\r" in value or "\n" in value:
|
||||
raise ValueError
|
||||
return value
|
||||
except (ValueError, KeyError, TypeError, UnicodeDecodeError):
|
||||
raise ValueError("credential must contain a nonempty model_api_key") from None
|
||||
port = data.get("upstream_port")
|
||||
if port is None:
|
||||
return value, route["upstream"]
|
||||
if type(port) is not int or port not in route.get("upstream_ports", ()):
|
||||
raise ValueError("credential upstream_port is not one of this model's instances")
|
||||
return value, route["upstream"].rsplit(":", 1)[0] + ":%d" % port
|
||||
|
||||
|
||||
def _json_object(pairs):
|
||||
@ -258,11 +268,12 @@ class _Handler(BaseHTTPRequestHandler):
|
||||
else:
|
||||
route = MODELS[proxy.model]
|
||||
statuses = facts["upstream_statuses"] = []
|
||||
facts["upstream"] = proxy.upstream
|
||||
for delay in RETRY_DELAYS + (None,):
|
||||
response = None
|
||||
try:
|
||||
connection = (http.client.HTTPSConnection if route["https"] else
|
||||
http.client.HTTPConnection)(route["upstream"], timeout=30)
|
||||
http.client.HTTPConnection)(proxy.upstream, timeout=30)
|
||||
connection.connect()
|
||||
upstream_socket = connection.sock
|
||||
proxy._register(upstream_socket)
|
||||
@ -343,7 +354,7 @@ class ModelProxy:
|
||||
raise ValueError("port must be 0..65535")
|
||||
self._token = token
|
||||
self._port = port
|
||||
self._key = _credential(credential_file)
|
||||
self._key, self.upstream = _credential(credential_file, MODELS[model])
|
||||
self._bind = str(address)
|
||||
self._log_path = _path(log_path)
|
||||
self._lock = threading.Lock()
|
||||
|
||||
@ -16,6 +16,7 @@ from urllib.parse import urlsplit
|
||||
|
||||
from rollout import model_proxy
|
||||
from rollout.model_proxy import MAX_BODY_BYTES, ModelProxy, MODEL, MODELS, QWEN_MODEL, UPSTREAM_IDLE_SECONDS
|
||||
from rollout.model_proxy import _credential
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FAST = dict(STALL_SECONDS=0.4, KEEPALIVE_SECONDS=0.1, RETRY_DELAYS=(0, 0))
|
||||
@ -412,3 +413,22 @@ console.log(JSON.stringify({complete: summary.complete, error: last?.message?.er
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class CredentialInstanceTests(unittest.TestCase):
|
||||
def write(self, data):
|
||||
folder = os.path.realpath(tempfile.mkdtemp()) # macOS /var is a symlink, which credentials refuse
|
||||
self.addCleanup(shutil.rmtree, folder)
|
||||
path = Path(folder, "key.json")
|
||||
path.write_text(json.dumps(data))
|
||||
path.chmod(0o600)
|
||||
return path
|
||||
|
||||
def test_a_credential_may_pick_only_a_listed_instance_of_its_model(self):
|
||||
qwen, glm = MODELS[QWEN_MODEL], MODELS["glm-5.3"]
|
||||
self.assertEqual(_credential(self.write(dict(model_api_key="k")), qwen), ("k", "114.111.18.115:18080"))
|
||||
self.assertEqual(_credential(self.write(dict(model_api_key="k", upstream_port=18083)), qwen),
|
||||
("k", "114.111.18.115:18083"))
|
||||
for port, route in ((18084, qwen), ("18081", qwen), (18081, glm)):
|
||||
with self.assertRaises(ValueError):
|
||||
_credential(self.write(dict(model_api_key="k", upstream_port=port)), route)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user