add disable_custom_all_reduce for vllm multi-gpu inference

xusenlinzy · Jun 7, 2024 · 509c0ba · 509c0ba
1 parent 016bdff
commit 509c0ba
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 0 deletions.
diff --git a/api/config.py b/api/config.py
@@ -224,6 +224,9 @@ class VLLMSetting(BaseModel):
     lora_modules: Optional[str] = Field(
         default=get_env("LORA_MODULES", ""),
     )
+    disable_custom_all_reduce: Optional[bool] = Field(
+        default=get_bool_env("DISABLE_CUSTOM_ALL_REDUCE"),
+    )
     vllm_disable_log_stats: Optional[bool] = Field(
         default=get_bool_env("VLLM_DISABLE_LOG_STATS", "true"),
     )

diff --git a/api/models.py b/api/models.py
@@ -112,6 +112,7 @@ def create_vllm_engine():
         "max_loras",
         "max_lora_rank",
         "lora_extra_vocab_size",
+        "disable_custom_all_reduce",
     }
 
     if vllm_version >= "0.4.3":