FEAT: Supports sglang backend (#1161)

setup.cfg

@@ -96,6 +96,7 @@ all =
  orjson
  auto-gptq ; sys_platform!='darwin'
  optimum
+ sglang[all] ; sys_platform=='linux'
 intel =
  torch==2.1.0a0
  intel_extension_for_pytorch==2.1.10+xpu
@@ -118,6 +119,8 @@ transformers =
  peft
 vllm =
  vllm>=0.2.6
+sglang =
+ sglang[all]
 embedding =
  sentence-transformers>=2.3.1
 image =

xinference/constants.py

@@ -25,6 +25,7 @@
 XINFERENCE_ENV_HEALTH_CHECK_TIMEOUT = "XINFERENCE_HEALTH_CHECK_TIMEOUT"
 XINFERENCE_ENV_DISABLE_HEALTH_CHECK = "XINFERENCE_DISABLE_HEALTH_CHECK"
 XINFERENCE_ENV_DISABLE_VLLM = "XINFERENCE_DISABLE_VLLM"
+XINFERENCE_ENV_ENABLE_SGLANG = "XINFERENCE_ENABLE_SGLANG"
 
 
 def get_xinference_home() -> str:
@@ -64,3 +65,4 @@ def get_xinference_home() -> str:
  int(os.environ.get(XINFERENCE_ENV_DISABLE_HEALTH_CHECK, 0))
 )
 XINFERENCE_DISABLE_VLLM = bool(int(os.environ.get(XINFERENCE_ENV_DISABLE_VLLM, 0)))
+XINFERENCE_ENABLE_SGLANG = bool(int(os.environ.get(XINFERENCE_ENV_ENABLE_SGLANG, 0)))

xinference/model/llm/__init__.py

@@ -60,6 +60,7 @@ def _install():
  from .pytorch.qwen_vl import QwenVLChatModel
  from .pytorch.vicuna import VicunaPytorchChatModel
  from .pytorch.yi_vl import YiVLChatModel
+ from .sglang.core import SGLANGChatModel, SGLANGModel
  from .vllm.core import VLLMChatModel, VLLMModel
 
  # register llm classes.
@@ -79,6 +80,7 @@ def _install():
  CtransformersModel,
  ]
  )
+ LLM_CLASSES.extend([SGLANGModel, SGLANGChatModel])
  LLM_CLASSES.extend([VLLMModel, VLLMChatModel])
  LLM_CLASSES.extend(
  [

xinference/model/llm/sglang/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022-2024 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.