diff --git a/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java b/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java
index 3d80aa66a9e..b3321732737 100644
--- a/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java
+++ b/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java
@@ -118,7 +118,7 @@ final class DefaultServerConfig implements ServerConfig {
 
     @Nullable
     private final Mapping<String, SslContext> sslContexts;
-    private final ServerMetrics serverMetrics = new ServerMetrics();
+    private final ServerMetrics serverMetrics;
 
     @Nullable
     private String strVal;
@@ -266,6 +266,7 @@ final class DefaultServerConfig implements ServerConfig {
         this.absoluteUriTransformer = castAbsoluteUriTransformer;
         this.unloggedExceptionsReportIntervalMillis = unloggedExceptionsReportIntervalMillis;
         this.shutdownSupports = ImmutableList.copyOf(requireNonNull(shutdownSupports, "shutdownSupports"));
+        serverMetrics = new ServerMetrics(meterRegistry);
     }
 
     private static Int2ObjectMap<Mapping<String, VirtualHost>> buildDomainAndPortMapping(
diff --git a/core/src/main/java/com/linecorp/armeria/server/Server.java b/core/src/main/java/com/linecorp/armeria/server/Server.java
index 4dccd937874..6b8b0b5572c 100644
--- a/core/src/main/java/com/linecorp/armeria/server/Server.java
+++ b/core/src/main/java/com/linecorp/armeria/server/Server.java
@@ -562,7 +562,25 @@ private ChannelFuture doStart(ServerPort port) {
             final GracefulShutdownSupport gracefulShutdownSupport = this.gracefulShutdownSupport;
             assert gracefulShutdownSupport != null;
 
-            final ServerPortMetric serverPortMetric = new ServerPortMetric();
+            ServerPortMetric serverPortMetric = null;
+            lock.lock();
+            try {
+                for (ServerPort serverPort : activePorts.values()) {
+                    final InetSocketAddress localAddress = serverPort.localAddress();
+                    if (!(localAddress instanceof DomainSocketAddress) &&
+                        localAddress.getPort() == port.localAddress().getPort()) {
+                        // Because we use the port number for aggregating metrics, use the previous
+                        // serverPortMetric.
+                        serverPortMetric = serverPort.serverPortMetric();
+                        break;
+                    }
+                }
+            } finally {
+                lock.unlock();
+            }
+            if (serverPortMetric == null) {
+                serverPortMetric = new ServerPortMetric();
+            }
             b.group(bossGroup, config.workerGroup());
             b.handler(connectionLimitingHandler.newChildHandler(serverPortMetric));
             b.childHandler(new HttpServerPipelineConfigurator(config, port, gracefulShutdownSupport,
@@ -837,6 +855,9 @@ public void operationComplete(ChannelFuture f) {
                     logger.warn("Unexpected local address type: {}", localAddress.getClass().getName());
                     return;
                 }
+                final ServerPortMetric serverPortMetric = ch.attr(SERVER_PORT_METRIC).get();
+                assert serverPortMetric != null;
+                actualPort.setServerPortMetric(serverPortMetric);
 
                 // Update the boss thread so its name contains the actual port.
                 Thread.currentThread().setName(bossThreadName(actualPort));
@@ -849,10 +870,7 @@ public void operationComplete(ChannelFuture f) {
                     lock.unlock();
                 }
 
-                final ServerPortMetric serverPortMetric = ch.attr(SERVER_PORT_METRIC).get();
-                assert serverPortMetric != null;
-                serverPortMetric.bindTo(config.meterRegistry(), actualPort);
-                config.serverMetrics().addServerPortMetric(serverPortMetric);
+                config.serverMetrics().addServerPort(actualPort);
 
                 if (logger.isInfoEnabled()) {
                     if (isLocalPort(actualPort)) {
diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java b/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java
index 6e59034b3d9..5d45f9e6114 100644
--- a/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java
+++ b/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java
@@ -16,14 +16,16 @@
 
 package com.linecorp.armeria.server;
 
-import java.util.List;
-import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
 
 import com.google.common.base.MoreObjects;
 import com.google.common.primitives.Ints;
 
 import com.linecorp.armeria.common.annotation.UnstableApi;
 
+import io.micrometer.core.instrument.MeterRegistry;
+
 /**
  * A class that holds metrics related server.
  */
@@ -33,12 +35,19 @@ public final class ServerMetrics {
     static final String ALL_REQUESTS_METER_NAME = "armeria.server.all.requests";
     static final String ALL_CONNECTIONS_METER_NAME = "armeria.server.connections";
 
-    private final List<ServerPortMetric> serverPortMetrics = new CopyOnWriteArrayList<>();
+    private final Set<ServerPortMetric> serverPortMetrics = new CopyOnWriteArraySet<>();
+    private final MeterRegistry meterRegistry;
 
-    ServerMetrics() {}
+    ServerMetrics(MeterRegistry meterRegistry) {
+        this.meterRegistry = meterRegistry;
+    }
 
-    void addServerPortMetric(ServerPortMetric serverPortMetric) {
-        serverPortMetrics.add(serverPortMetric);
+    void addServerPort(ServerPort serverPort) {
+        final ServerPortMetric serverPortMetric = serverPort.serverPortMetric();
+        assert serverPortMetric != null;
+        if (serverPortMetrics.add(serverPortMetric)) {
+            serverPortMetric.bindTo(meterRegistry, serverPort);
+        }
     }
 
     /**
diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerPort.java b/core/src/main/java/com/linecorp/armeria/server/ServerPort.java
index f6da639aa7a..ff1580bf5be 100644
--- a/core/src/main/java/com/linecorp/armeria/server/ServerPort.java
+++ b/core/src/main/java/com/linecorp/armeria/server/ServerPort.java
@@ -66,6 +66,8 @@ static long nextPortGroup() {
     private final Set<SessionProtocol> protocols;
     private final long portGroup;
     private int hashCode;
+    @Nullable
+    private ServerPortMetric serverPortMetric;
 
     @Nullable
     private String strVal;
@@ -226,6 +228,16 @@ long portGroup() {
         return portGroup;
     }
 
+    @Nullable
+    ServerPortMetric serverPortMetric() {
+        return serverPortMetric;
+    }
+
+    void setServerPortMetric(ServerPortMetric serverPortMetric) {
+        this.serverPortMetric = serverPortMetric;
+    }
+
+    // Do not take account into serverPortMetric for equality and hashCode.
     @Override
     public int hashCode() {
         int hashCode = this.hashCode;
diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerPortMetric.java b/core/src/main/java/com/linecorp/armeria/server/ServerPortMetric.java
index 3ed3e46da44..5d34f3115b7 100644
--- a/core/src/main/java/com/linecorp/armeria/server/ServerPortMetric.java
+++ b/core/src/main/java/com/linecorp/armeria/server/ServerPortMetric.java
@@ -149,6 +149,8 @@ void bindTo(MeterRegistry meterRegistry, ServerPort actualPort) {
                             this, ServerPortMetric::activeConnections);
     }
 
+    // Use reference equality for comparison.
+
     @Override
     public String toString() {
         return MoreObjects.toStringHelper(this)