ENH: bypass the sampling parameter skip_special_tokens to vLLM backend (

#2655)

xinference/api/restful_api.py

@@ -2044,7 +2044,6 @@ async def create_chat_completion(self, request: Request) -> Response:
                 )
         if body.tools and body.stream:
             is_vllm = await model.is_vllm_backend()
-
             if not (
                 (is_vllm and model_family in QWEN_TOOL_CALL_FAMILY)
                 or (not is_vllm and model_family in GLM4_TOOL_CALL_FAMILY)
@@ -2054,7 +2053,8 @@ async def create_chat_completion(self, request: Request) -> Response:
                     detail="Streaming support for tool calls is available only when using "
                     "Qwen models with vLLM backend or GLM4-chat models without vLLM backend.",
                 )
-
+        if "skip_special_tokens" in raw_kwargs and await model.is_vllm_backend():
+            kwargs["skip_special_tokens"] = raw_kwargs["skip_special_tokens"]
         if body.stream:
 
             async def stream_results():

xinference/model/llm/vllm/core.py

@@ -86,6 +86,7 @@ class VLLMGenerateConfig(TypedDict, total=False):
     stop: Optional[Union[str, List[str]]]
     stream: bool  # non-sampling param, should not be passed to the engine.
     stream_options: Optional[Union[dict, None]]
+    skip_special_tokens: Optional[bool]
     response_format: Optional[dict]
     guided_json: Optional[Union[str, dict]]
     guided_regex: Optional[str]
@@ -373,6 +374,9 @@ def _sanitize_generate_config(
         sanitized.setdefault(
             "stream_options", generate_config.get("stream_options", None)
         )
+        sanitized.setdefault(
+            "skip_special_tokens", generate_config.get("skip_special_tokens", True)
+        )
         sanitized.setdefault(
             "guided_json", generate_config.get("guided_json", guided_json)
         )