Decode bytes input to utf-8 string before passing to vllm engine (#57)

triton-inference-server · Oct 5, 2023 · 2e29214 · 2e29214
1 parent 512ba53
commit 2e29214
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/Quick_Deploy/vLLM/model_repository/vllm/1/model.py b/Quick_Deploy/vLLM/model_repository/vllm/1/model.py
@@ -167,6 +167,8 @@ async def generate(self, request):
         try:
             request_id = random_uuid()
             prompt = pb_utils.get_input_tensor_by_name(request, "PROMPT").as_numpy()[0]
+            if isinstance(prompt, bytes):
+                prompt = prompt.decode("utf-8")
             stream = pb_utils.get_input_tensor_by_name(request, "STREAM").as_numpy()[0]
 
             # Request parameters are not yet supported via
@@ -184,7 +186,7 @@ async def generate(self, request):
 
             last_output = None
             async for output in self.llm_engine.generate(
-                str(prompt), sampling_params, request_id
+                prompt, sampling_params, request_id
             ):
                 if stream:
                     response_sender.send(self.create_response(output))