diff --git a/ci/do_ci.sh b/ci/do_ci.sh
index 16ceaa8c56e2..9fb6f268870a 100755
--- a/ci/do_ci.sh
+++ b/ci/do_ci.sh
@@ -490,10 +490,10 @@ case $CI_TARGET in
         # This doesn't go into CI but is available for developer convenience.
         echo "bazel fastbuild build with tests..."
         echo "Building..."
-        bazel_envoy_binary_build fastbuild
+        # bazel_envoy_binary_build fastbuild
         echo "Testing ${TEST_TARGETS[*]}"
         bazel test "${BAZEL_BUILD_OPTIONS[@]}" \
-              -c fastbuild "${TEST_TARGETS[@]}"
+              -c fastbuild //test/common/listener_manager:listener_manager_impl_test
         ;;
 
     dev.contrib)
diff --git a/source/common/listener_manager/listener_manager_impl.cc b/source/common/listener_manager/listener_manager_impl.cc
index 8b5296a5d76d..b44addfbabbe 100644
--- a/source/common/listener_manager/listener_manager_impl.cc
+++ b/source/common/listener_manager/listener_manager_impl.cc
@@ -525,8 +525,12 @@ ListenerManagerImpl::setupSocketFactoryForListener(ListenerImpl& new_listener,
                     "when the reuse port isn't enabled",
                     new_listener.name()));
   }
-
-  if (!(existing_listener.hasCompatibleAddress(new_listener) && same_socket_options)) {
+  if (existing_listener.hasDuplicatedAddress(new_listener) && new_listener.reusePort() == false) {
+    return absl::InvalidArgumentError(fmt::format("Listener {}: doesn't support update addresses "
+                                                  "when the reuse port isn't enabled",
+                                                  new_listener.name()));
+  }
+  if (!existing_listener.hasCompatibleAddress(new_listener)) {
     RETURN_IF_NOT_OK(setNewOrDrainingSocketFactory(new_listener.name(), new_listener));
   } else {
     RETURN_IF_NOT_OK(new_listener.cloneSocketFactoryFrom(existing_listener));