diff --git a/rollout/model_proxy.py b/rollout/model_proxy.py index 0d03c531..365c5ae7 100644 --- a/rollout/model_proxy.py +++ b/rollout/model_proxy.py @@ -24,6 +24,8 @@ MODELS = { # User-specified plain-HTTP service; the Bearer key crosses the network unencrypted. "qwen3.8-27b-nvfp4": dict(api="openai-responses", path="/v1/responses", https=False, upstream="114.111.18.115:18080", upstream_path="/v1/responses", + # Four identical instances of the same model; a credential may pick one of them. + upstream_ports=(18080, 18081, 18082, 18083), max_tokens=65536, max_tokens_field="max_output_tokens"), } MODEL = "glm-5.3" @@ -44,7 +46,9 @@ PREAMBLE = ("response.created", "response.in_progress") TERMINALS = ("response.completed", "response.incomplete", "response.failed", "error") -def _credential(path): +def _credential(path, route): + """Return (key, upstream host:port) from a private JSON credential; `upstream_port` picks one of the route's + listed instances.""" path = _path(path) descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) with os.fdopen(descriptor, "rb") as stream: @@ -55,12 +59,18 @@ def _credential(path): try: if len(raw) > 64 * 1024: raise ValueError - value = json.loads(raw)["model_api_key"] + data = json.loads(raw) + value = data["model_api_key"] if not isinstance(value, str) or not value.strip() or "\r" in value or "\n" in value: raise ValueError - return value except (ValueError, KeyError, TypeError, UnicodeDecodeError): raise ValueError("credential must contain a nonempty model_api_key") from None + port = data.get("upstream_port") + if port is None: + return value, route["upstream"] + if type(port) is not int or port not in route.get("upstream_ports", ()): + raise ValueError("credential upstream_port is not one of this model's instances") + return value, route["upstream"].rsplit(":", 1)[0] + ":%d" % port def _json_object(pairs): @@ -258,11 +268,12 @@ class _Handler(BaseHTTPRequestHandler): else: route = MODELS[proxy.model] statuses = facts["upstream_statuses"] = [] + facts["upstream"] = proxy.upstream for delay in RETRY_DELAYS + (None,): response = None try: connection = (http.client.HTTPSConnection if route["https"] else - http.client.HTTPConnection)(route["upstream"], timeout=30) + http.client.HTTPConnection)(proxy.upstream, timeout=30) connection.connect() upstream_socket = connection.sock proxy._register(upstream_socket) @@ -343,7 +354,7 @@ class ModelProxy: raise ValueError("port must be 0..65535") self._token = token self._port = port - self._key = _credential(credential_file) + self._key, self.upstream = _credential(credential_file, MODELS[model]) self._bind = str(address) self._log_path = _path(log_path) self._lock = threading.Lock() diff --git a/tests/test_model_proxy.py b/tests/test_model_proxy.py index 07ec4f50..e676a19b 100644 --- a/tests/test_model_proxy.py +++ b/tests/test_model_proxy.py @@ -16,6 +16,7 @@ from urllib.parse import urlsplit from rollout import model_proxy from rollout.model_proxy import MAX_BODY_BYTES, ModelProxy, MODEL, MODELS, QWEN_MODEL, UPSTREAM_IDLE_SECONDS +from rollout.model_proxy import _credential ROOT = Path(__file__).resolve().parents[1] FAST = dict(STALL_SECONDS=0.4, KEEPALIVE_SECONDS=0.1, RETRY_DELAYS=(0, 0)) @@ -412,3 +413,22 @@ console.log(JSON.stringify({complete: summary.complete, error: last?.message?.er if __name__ == "__main__": unittest.main() + + +class CredentialInstanceTests(unittest.TestCase): + def write(self, data): + folder = os.path.realpath(tempfile.mkdtemp()) # macOS /var is a symlink, which credentials refuse + self.addCleanup(shutil.rmtree, folder) + path = Path(folder, "key.json") + path.write_text(json.dumps(data)) + path.chmod(0o600) + return path + + def test_a_credential_may_pick_only_a_listed_instance_of_its_model(self): + qwen, glm = MODELS[QWEN_MODEL], MODELS["glm-5.3"] + self.assertEqual(_credential(self.write(dict(model_api_key="k")), qwen), ("k", "114.111.18.115:18080")) + self.assertEqual(_credential(self.write(dict(model_api_key="k", upstream_port=18083)), qwen), + ("k", "114.111.18.115:18083")) + for port, route in ((18084, qwen), ("18081", qwen), (18081, glm)): + with self.assertRaises(ValueError): + _credential(self.write(dict(model_api_key="k", upstream_port=port)), route)