|
|
@@ -44,9 +44,6 @@ class ModelSingleton:
|
|
|
batch_size = 0
|
|
|
if backend in ['transformers', 'vllm-engine', "vllm-async-engine"] and not model_path:
|
|
|
model_path = auto_download_and_get_model_root_path("/","vlm")
|
|
|
-
|
|
|
- custom_logits_processors = enable_custom_logits_processors()
|
|
|
-
|
|
|
if backend == "transformers":
|
|
|
try:
|
|
|
from transformers import (
|
|
|
@@ -99,7 +96,7 @@ class ModelSingleton:
|
|
|
kwargs["gpu_memory_utilization"] = 0.5
|
|
|
if "model" not in kwargs:
|
|
|
kwargs["model"] = model_path
|
|
|
- if custom_logits_processors and "logits_processors" not in kwargs:
|
|
|
+ if enable_custom_logits_processors() and ("logits_processors" not in kwargs):
|
|
|
kwargs["logits_processors"] = [MinerULogitsProcessor]
|
|
|
# 使用kwargs为 vllm初始化参数
|
|
|
vllm_llm = vllm.LLM(**kwargs)
|
|
|
@@ -114,7 +111,7 @@ class ModelSingleton:
|
|
|
kwargs["gpu_memory_utilization"] = 0.5
|
|
|
if "model" not in kwargs:
|
|
|
kwargs["model"] = model_path
|
|
|
- if custom_logits_processors and "logits_processors" not in kwargs:
|
|
|
+ if enable_custom_logits_processors() and ("logits_processors" not in kwargs):
|
|
|
kwargs["logits_processors"] = [MinerULogitsProcessor]
|
|
|
# 使用kwargs为 vllm初始化参数
|
|
|
vllm_async_llm = AsyncLLM.from_engine_args(AsyncEngineArgs(**kwargs))
|