diff --git a/taggit/managers.py b/taggit/managers.py
index 366337d9..35d49a80 100644
--- a/taggit/managers.py
+++ b/taggit/managers.py
@@ -291,27 +291,29 @@ def set(self, tags, *, through_defaults=None, **kwargs):
             # make sure we're working with a collection of a uniform type
             objs = self._to_tag_model_instances(tags, tag_kwargs)
 
-            # get the existing tag strings
-            old_tag_strs = set(
-                self.through._default_manager.using(db)
+            # get the existing tags
+            old_tags = set(
+                t.tag
+                for t in self.through._default_manager.using(db)
                 .filter(**self._lookup_kwargs())
-                .values_list("tag__name", flat=True)
+                .select_related("tag")
             )
 
             new_objs = []
             for obj in objs:
-                if obj.name in old_tag_strs:
-                    old_tag_strs.remove(obj.name)
+                if obj in old_tags:
+                    old_tags.remove(obj)
                 else:
                     new_objs.append(obj)
 
-            self.remove(*old_tag_strs)
+            self.remove(*old_tags)
             self.add(*new_objs, through_defaults=through_defaults, **kwargs)
 
     @require_instance_manager
     def remove(self, *tags):
         if not tags:
             return
+        tag_objs = self._to_tag_model_instances(tags)
 
         self._remove_prefetched_objects()
         db = router.db_for_write(self.through, instance=self.instance)
@@ -319,7 +321,7 @@ def remove(self, *tags):
         qs = (
             self.through._default_manager.using(db)
             .filter(**self._lookup_kwargs())
-            .filter(tag__name__in=tags)
+            .filter(tag__in=tag_objs)
         )
 
         old_ids = set(qs.values_list("tag_id", flat=True))