Fix some issues with the model sometimes not getting patched.

comfy/model_patcher.py

@@ -669,6 +669,8 @@ def partially_unload(self, device_to, memory_to_free=0):
         return memory_freed
 
     def partially_load(self, device_to, extra_memory=0):
+        self.unpatch_model(unpatch_weights=False)
+        self.patch_model(patch_weights=False)
         full_load = False
         if self.model.model_lowvram == False:
             return 0