refactor: Remove explicit callings to garbage collect (#55)

Co-authored-by: Neelay Shah <[email protected]>

src/model.py

@@ -288,7 +288,6 @@ def response_loop(self):
             if item is None:
                 break
             response_sender, response, response_flag = item
-            del item
             try:
                 response_sender.send(response, response_flag)
             except Exception as e:
@@ -298,9 +297,6 @@ def response_loop(self):
             finally:
                 if response_flag == pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL:
                     self.ongoing_request_count -= 1
-                    del response_sender
-                    if self.ongoing_request_count == 0:
-                        gc.collect()
 
     def create_response(self, vllm_output, prepend_input):
         """
@@ -447,9 +443,6 @@ async def generate(self, request):
         finally:
             if decrement_ongoing_request_count:
                 self.ongoing_request_count -= 1
-                del response_sender
-                if self.ongoing_request_count == 0:
-                    gc.collect()
 
     def verify_loras(self, request):
         # We will check if the requested lora exists here, if not we will send a
@@ -527,3 +520,9 @@ def finalize(self):
         if self._response_thread is not None:
             self._response_thread.join()
             self._response_thread = None
+
+        # When using parallel tensors, the stub process may not shutdown due to
+        # unreleased references, so manually run the garbage collector once.
+        self.logger.log_info("[vllm] Running Garbage Collector on finalize...")
+        gc.collect()
+        self.logger.log_info("[vllm] Garbage Collector on finalize... done")