diff --git a/source/common/listener_manager/listener_impl.cc b/source/common/listener_manager/listener_impl.cc
index 9db574220cce..0337c153964e 100644
--- a/source/common/listener_manager/listener_impl.cc
+++ b/source/common/listener_manager/listener_impl.cc
@@ -1133,7 +1133,8 @@ bool ListenerImpl::socketOptionsEqual(const ListenerImpl& other) const {
 }
 
 bool ListenerImpl::hasCompatibleAddress(const ListenerImpl& other) const {
-  if ((socket_type_ != other.socket_type_) || (addresses_.size() != other.addresses().size())) {
+  if ((socket_type_ != other.socket_type_) || (addresses_.size() != other.addresses().size()) ||
+      !ListenerMessageUtil::socketOptionsEqual(config(), other.config())) {
     return false;
   }
 
@@ -1157,13 +1158,6 @@ bool ListenerImpl::hasCompatibleAddress(const ListenerImpl& other) const {
 }
 
 bool ListenerImpl::hasDuplicatedAddress(const ListenerImpl& other) const {
-  // Skip the duplicate address check if this is the case of a listener update with new socket
-  // options.
-  if ((name_ == other.name_) &&
-      !ListenerMessageUtil::socketOptionsEqual(config(), other.config())) {
-    return false;
-  }
-
   if (socket_type_ != other.socket_type_) {
     return false;
   }
diff --git a/source/common/listener_manager/listener_manager_impl.cc b/source/common/listener_manager/listener_manager_impl.cc
index 8de7a7da3360..8b5296a5d76d 100644
--- a/source/common/listener_manager/listener_manager_impl.cc
+++ b/source/common/listener_manager/listener_manager_impl.cc
@@ -642,6 +642,11 @@ absl::StatusOr<bool> ListenerManagerImpl::addOrUpdateListenerInternal(
 bool ListenerManagerImpl::hasListenerWithDuplicatedAddress(const ListenerList& list,
                                                            const ListenerImpl& listener) {
   for (const auto& existing_listener : list) {
+    // Skip if the listener is the same. It's because that the listener
+    // with same name was proven to be incompatible.
+    if (existing_listener->name() == listener.name()) {
+      continue;
+    }
     if (existing_listener->hasDuplicatedAddress(listener)) {
       return true;
     }
diff --git a/test/common/listener_manager/listener_manager_impl_test.cc b/test/common/listener_manager/listener_manager_impl_test.cc
index 2ad053736efe..980116694785 100644
--- a/test/common/listener_manager/listener_manager_impl_test.cc
+++ b/test/common/listener_manager/listener_manager_impl_test.cc
@@ -400,6 +400,46 @@ name: bar
   addOrUpdateListener(parseListenerFromV3Yaml(yaml2));
 }
 
+TEST_P(ListenerManagerImplWithRealFiltersTest,
+       AllowUpdateListenerWithSameNumberPortsButPortNumberChanged) {
+  const std::string yaml1 = R"EOF(
+name: foo
+address:
+  socket_address:
+    address: 127.0.0.1
+    port_value: 1000
+additional_addresses:
+- address:
+    socket_address:
+      address: 127.0.0.2
+      port_value: 1000
+filter_chains:
+- filters: []
+  name: foo
+  )EOF";
+
+  const std::string yaml2 = R"EOF(
+name: foo
+address:
+  socket_address:
+    address: 127.0.0.1
+    port_value: 1000
+additional_addresses:
+- address:
+    socket_address:
+      address: 127.0.0.2
+      port_value: 2000
+filter_chains:
+- filters: []
+  name: foo
+  )EOF";
+
+  EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)).Times(2);
+  addOrUpdateListener(parseListenerFromV3Yaml(yaml1));
+  EXPECT_CALL(listener_factory_, createListenSocket(_, _, _, default_bind_type, _, 0)).Times(2);
+  addOrUpdateListener(parseListenerFromV3Yaml(yaml2));
+}
+
 TEST_P(ListenerManagerImplWithRealFiltersTest, SetListenerPerConnectionBufferLimit) {
   const std::string yaml = R"EOF(
 address: