From 637bdd20305ab8f1a3d7ded246c3d654dfc470a9 Mon Sep 17 00:00:00 2001 From: minwoox Date: Thu, 13 Feb 2025 19:06:41 +0900 Subject: [PATCH 1/8] Add port tag to server metrics Motivation: A server can be configured with multiple ports, and operators may want to collect connection and request metrics per port for better observability and debugging. Modifications: - Added a `port` tag to server metrics. - If a `DomainSocketAddress` is used, the path of the address is used as the tag value. Result: - Enabled per-port metrics collection for better monitoring and troubleshooting. --- .../server/ConnectionLimitingHandler.java | 14 +- .../armeria/server/DefaultServerConfig.java | 3 +- .../armeria/server/Http1RequestDecoder.java | 10 +- .../armeria/server/Http2RequestDecoder.java | 9 +- .../armeria/server/HttpServerHandler.java | 72 +-- .../HttpServerPipelineConfigurator.java | 6 +- .../com/linecorp/armeria/server/Server.java | 22 +- .../armeria/server/ServerBuilder.java | 6 +- .../armeria/server/ServerMetrics.java | 419 +++++++++++++++--- .../server/ConnectionLimitingHandlerTest.java | 8 +- .../armeria/server/ServerMetricsTest.java | 41 +- 11 files changed, 459 insertions(+), 151 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java b/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java index e750c42972e..3dc9cdcf976 100644 --- a/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java +++ b/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java @@ -16,6 +16,10 @@ package com.linecorp.armeria.server; +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.linecorp.armeria.server.HttpServerHandler.UNKNOWN_ADDR; + +import java.net.InetSocketAddress; import java.util.Collections; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -26,6 +30,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.linecorp.armeria.internal.common.util.ChannelUtil; + import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.Channel; import io.netty.channel.ChannelHandler; @@ -59,17 +65,17 @@ final class ConnectionLimitingHandler extends ChannelInboundHandlerAdapter { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { final Channel child = (Channel) msg; - - final int conn = serverMetrics.increaseActiveConnectionsAndGet(); + final InetSocketAddress localAddress = firstNonNull(ChannelUtil.localAddress(child), UNKNOWN_ADDR); + final int conn = serverMetrics.increaseActiveConnectionsAndGet(localAddress); if (conn > 0 && conn <= maxNumConnections) { childChannels.add(child); child.closeFuture().addListener(future -> { childChannels.remove(child); - serverMetrics.decreaseActiveConnections(); + serverMetrics.decreaseActiveConnections(localAddress); }); super.channelRead(ctx, msg); } else { - serverMetrics.decreaseActiveConnections(); + serverMetrics.decreaseActiveConnections(localAddress); // Set linger option to 0 so that the server doesn't get too many TIME_WAIT states. child.config().setOption(ChannelOption.SO_LINGER, 0); diff --git a/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java b/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java index 3d80aa66a9e..511d65cf8b7 100644 --- a/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java +++ b/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java @@ -118,7 +118,7 @@ final class DefaultServerConfig implements ServerConfig { @Nullable private final Mapping sslContexts; - private final ServerMetrics serverMetrics = new ServerMetrics(); + private final ServerMetrics serverMetrics; @Nullable private String strVal; @@ -266,6 +266,7 @@ final class DefaultServerConfig implements ServerConfig { this.absoluteUriTransformer = castAbsoluteUriTransformer; this.unloggedExceptionsReportIntervalMillis = unloggedExceptionsReportIntervalMillis; this.shutdownSupports = ImmutableList.copyOf(requireNonNull(shutdownSupports, "shutdownSupports")); + serverMetrics = new ServerMetrics(ports); } private static Int2ObjectMap> buildDomainAndPortMapping( diff --git a/core/src/main/java/com/linecorp/armeria/server/Http1RequestDecoder.java b/core/src/main/java/com/linecorp/armeria/server/Http1RequestDecoder.java index 9172ca42ae8..9b8b5d06a99 100644 --- a/core/src/main/java/com/linecorp/armeria/server/Http1RequestDecoder.java +++ b/core/src/main/java/com/linecorp/armeria/server/Http1RequestDecoder.java @@ -16,10 +16,13 @@ package com.linecorp.armeria.server; +import static com.google.common.base.MoreObjects.firstNonNull; import static com.linecorp.armeria.internal.common.websocket.WebSocketUtil.isHttp1WebSocketUpgradeRequest; +import static com.linecorp.armeria.server.HttpServerHandler.UNKNOWN_ADDR; import static com.linecorp.armeria.server.HttpServerPipelineConfigurator.SCHEME_HTTP; import static com.linecorp.armeria.server.ServiceRouteUtil.newRoutingContext; +import java.net.InetSocketAddress; import java.net.URISyntaxException; import org.slf4j.Logger; @@ -46,6 +49,7 @@ import com.linecorp.armeria.internal.common.InitiateConnectionShutdown; import com.linecorp.armeria.internal.common.KeepAliveHandler; import com.linecorp.armeria.internal.common.NoopKeepAliveHandler; +import com.linecorp.armeria.internal.common.util.ChannelUtil; import com.linecorp.armeria.server.HttpServerUpgradeHandler.UpgradeEvent; import com.linecorp.armeria.server.websocket.WebSocketService; @@ -84,6 +88,7 @@ final class Http1RequestDecoder extends ChannelDuplexHandler { private final AsciiString scheme; private SessionProtocol sessionProtocol; private final InboundTrafficController inboundTrafficController; + private final InetSocketAddress localAddress; private ServerHttpObjectEncoder encoder; private final HttpServer httpServer; @@ -96,6 +101,7 @@ final class Http1RequestDecoder extends ChannelDuplexHandler { Http1RequestDecoder(ServerConfig cfg, Channel channel, AsciiString scheme, ServerHttp1ObjectEncoder encoder, HttpServer httpServer) { this.cfg = cfg; + localAddress = firstNonNull(ChannelUtil.localAddress(channel), UNKNOWN_ADDR); this.scheme = scheme; sessionProtocol = scheme == SCHEME_HTTP ? SessionProtocol.H1C : SessionProtocol.H1; inboundTrafficController = InboundTrafficController.ofHttp1(channel); @@ -271,7 +277,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception if (pipeline.get(HttpServerUpgradeHandler.class) != null) { pipeline.remove(HttpServerUpgradeHandler.class); } - cfg.serverMetrics().increasePendingHttp1Requests(); + cfg.serverMetrics().increasePendingHttp1Requests(localAddress); ctx.fireChannelRead(webSocketRequest); return; } @@ -284,7 +290,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception if (maxRequestLength > 0 && contentLength > maxRequestLength) { abortLargeRequest(ctx, req, id, endOfStream, keepAliveHandler, true); } - cfg.serverMetrics().increasePendingHttp1Requests(); + cfg.serverMetrics().increasePendingHttp1Requests(localAddress); ctx.fireChannelRead(req); } else { fail(id, null, HttpStatus.BAD_REQUEST, "Invalid decoder state", null); diff --git a/core/src/main/java/com/linecorp/armeria/server/Http2RequestDecoder.java b/core/src/main/java/com/linecorp/armeria/server/Http2RequestDecoder.java index 72d4078fc91..cb069f7e416 100644 --- a/core/src/main/java/com/linecorp/armeria/server/Http2RequestDecoder.java +++ b/core/src/main/java/com/linecorp/armeria/server/Http2RequestDecoder.java @@ -16,12 +16,16 @@ package com.linecorp.armeria.server; +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.linecorp.armeria.server.HttpServerHandler.UNKNOWN_ADDR; import static com.linecorp.armeria.server.HttpServerPipelineConfigurator.SCHEME_HTTP; import static com.linecorp.armeria.server.ServiceRouteUtil.newRoutingContext; import static io.netty.handler.codec.http2.Http2Error.INTERNAL_ERROR; import static io.netty.handler.codec.http2.Http2Error.PROTOCOL_ERROR; import static io.netty.handler.codec.http2.Http2Exception.connectionError; +import java.net.InetSocketAddress; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,6 +45,7 @@ import com.linecorp.armeria.internal.common.Http2GoAwayHandler; import com.linecorp.armeria.internal.common.InboundTrafficController; import com.linecorp.armeria.internal.common.KeepAliveHandler; +import com.linecorp.armeria.internal.common.util.ChannelUtil; import io.netty.buffer.ByteBuf; import io.netty.channel.Channel; @@ -65,6 +70,7 @@ final class Http2RequestDecoder extends Http2EventAdapter { private final ServerConfig cfg; private final Channel channel; + private final InetSocketAddress localAddress; private final AsciiString scheme; @Nullable private ServerHttp2ObjectEncoder encoder; @@ -79,6 +85,7 @@ final class Http2RequestDecoder extends Http2EventAdapter { AsciiString scheme, KeepAliveHandler keepAliveHandler) { this.cfg = cfg; this.channel = channel; + localAddress = firstNonNull(ChannelUtil.localAddress(channel), UNKNOWN_ADDR); this.scheme = scheme; inboundTrafficController = InboundTrafficController.ofHttp2(channel, cfg.http2InitialConnectionWindowSize()); @@ -211,7 +218,7 @@ public void onHeadersRead(ChannelHandlerContext ctx, int streamId, Http2Headers abortLargeRequest(req, endOfStream, true); } requests.put(streamId, req); - cfg.serverMetrics().increasePendingHttp2Requests(); + cfg.serverMetrics().increasePendingHttp2Requests(localAddress); ctx.fireChannelRead(req); } else { if (!(req instanceof DecodedHttpRequestWriter)) { diff --git a/core/src/main/java/com/linecorp/armeria/server/HttpServerHandler.java b/core/src/main/java/com/linecorp/armeria/server/HttpServerHandler.java index 4eb48090da9..d75e08cd13a 100644 --- a/core/src/main/java/com/linecorp/armeria/server/HttpServerHandler.java +++ b/core/src/main/java/com/linecorp/armeria/server/HttpServerHandler.java @@ -97,7 +97,7 @@ final class HttpServerHandler extends ChannelInboundHandlerAdapter implements Ht private static final String ALLOWED_METHODS_STRING = HttpMethod.knownMethods().stream().map(HttpMethod::name).collect(Collectors.joining(",")); - private static final InetSocketAddress UNKNOWN_ADDR; + static final InetSocketAddress UNKNOWN_ADDR; static { InetAddress unknownAddr; @@ -193,10 +193,8 @@ static void safeClose(Channel ch) { @Nullable private final ProxiedAddresses proxiedAddresses; - @Nullable - private InetSocketAddress remoteAddress; - @Nullable - private InetSocketAddress localAddress; + private final InetSocketAddress remoteAddress; + private final InetSocketAddress localAddress; private final IdentityHashMap unfinishedRequests; private boolean isReading; @@ -205,7 +203,7 @@ static void safeClose(Channel ch) { private boolean handledLastRequest; HttpServerHandler(ServerConfig config, - GracefulShutdownSupport gracefulShutdownSupport, + Channel channel, GracefulShutdownSupport gracefulShutdownSupport, @Nullable ServerHttpObjectEncoder responseEncoder, SessionProtocol protocol, @Nullable ProxiedAddresses proxiedAddresses) { @@ -213,6 +211,8 @@ static void safeClose(Channel ch) { assert protocol == H1 || protocol == H1C || protocol == H2; this.config = requireNonNull(config, "config"); + remoteAddress = firstNonNull(ChannelUtil.remoteAddress(channel), UNKNOWN_ADDR); + localAddress = firstNonNull(ChannelUtil.localAddress(channel), UNKNOWN_ADDR); this.gracefulShutdownSupport = requireNonNull(gracefulShutdownSupport, "gracefulShutdownSupport"); this.protocol = requireNonNull(protocol, "protocol"); @@ -384,6 +384,7 @@ private static void incrementLocalWindowSize(ChannelPipeline pipeline, int delta private void handleRequest(ChannelHandlerContext ctx, DecodedHttpRequest req) throws Exception { final ServerHttpObjectEncoder responseEncoder = this.responseEncoder; assert responseEncoder != null; + final Channel channel = ctx.channel(); // Ignore the request received after the last request, // because we are going to close the connection after sending the last response. @@ -400,11 +401,8 @@ private void handleRequest(ChannelHandlerContext ctx, DecodedHttpRequest req) th responseEncoder.keepAliveHandler().disconnectWhenFinished(); } - final Channel channel = ctx.channel(); final RequestHeaders headers = req.headers(); - final InetSocketAddress remoteAddress = firstNonNull(remoteAddress(channel), UNKNOWN_ADDR); - final InetSocketAddress localAddress = firstNonNull(localAddress(channel), UNKNOWN_ADDR); - final ProxiedAddresses proxiedAddresses = determineProxiedAddresses(remoteAddress, headers); + final ProxiedAddresses proxiedAddresses = determineProxiedAddresses(headers); final InetAddress clientAddress = config.clientAddressMapper().apply(proxiedAddresses).getAddress(); final EventLoop channelEventLoop = channel.eventLoop(); @@ -412,8 +410,7 @@ private void handleRequest(ChannelHandlerContext ctx, DecodedHttpRequest req) th final RoutingStatus routingStatus = routingCtx.status(); if (!routingStatus.routeMustExist()) { final ServiceRequestContext reqCtx = newEarlyRespondingRequestContext( - channel, req, proxiedAddresses, clientAddress, remoteAddress, localAddress, routingCtx, - channelEventLoop); + channel, req, proxiedAddresses, clientAddress, routingCtx, channelEventLoop); // Handle 'OPTIONS * HTTP/1.1'. if (routingStatus == RoutingStatus.OPTIONS) { @@ -454,13 +451,15 @@ private void handleRequest(ChannelHandlerContext ctx, DecodedHttpRequest req) th if (serviceEventLoop.inEventLoop()) { return serve0(req, service, reqCtx, req.isHttp1WebSocket()); } - return serveInServiceEventLoop(req, service, reqCtx, serviceEventLoop, req.isHttp1WebSocket()); + return serveInServiceEventLoop(req, service, reqCtx, serviceEventLoop, + req.isHttp1WebSocket()); })); } else { if (serviceEventLoop.inEventLoop()) { res = serve0(req, service, reqCtx, req.isHttp1WebSocket()); } else { - res = serveInServiceEventLoop(req, service, reqCtx, serviceEventLoop, req.isHttp1WebSocket()); + res = serveInServiceEventLoop(req, service, reqCtx, serviceEventLoop, + req.isHttp1WebSocket()); } } res = res.recover(cause -> { @@ -522,21 +521,21 @@ private void handleRequest(ChannelHandlerContext ctx, DecodedHttpRequest req) th private void decreasePendingRequests() { if (protocol.isExplicitHttp1()) { - config.serverMetrics().decreasePendingHttp1Requests(); + config.serverMetrics().decreasePendingHttp1Requests(localAddress); } else { assert protocol.isExplicitHttp2(); - config.serverMetrics().decreasePendingHttp2Requests(); + config.serverMetrics().decreasePendingHttp2Requests(localAddress); } } private void increaseActiveRequests(boolean isHttp1WebSocket) { if (isHttp1WebSocket) { - config.serverMetrics().increaseActiveHttp1WebSocketRequests(); + config.serverMetrics().increaseActiveHttp1WebSocketRequests(localAddress); } else if (protocol.isExplicitHttp1()) { - config.serverMetrics().increaseActiveHttp1Requests(); + config.serverMetrics().increaseActiveHttp1Requests(localAddress); } else { assert protocol.isExplicitHttp2(); - config.serverMetrics().increaseActiveHttp2Requests(); + config.serverMetrics().increaseActiveHttp2Requests(localAddress); } } @@ -570,8 +569,7 @@ private HttpResponse serveInServiceEventLoop(DecodedHttpRequest req, .subscribeOn(serviceEventLoop); } - private ProxiedAddresses determineProxiedAddresses(InetSocketAddress remoteAddress, - RequestHeaders headers) { + private ProxiedAddresses determineProxiedAddresses(RequestHeaders headers) { if (config.clientAddressTrustedProxyFilter().test(remoteAddress.getAddress())) { return HttpHeaderUtil.determineProxiedAddresses( headers, config.clientAddressSources(), proxiedAddresses, @@ -581,30 +579,6 @@ private ProxiedAddresses determineProxiedAddresses(InetSocketAddress remoteAddre } } - @Nullable - private InetSocketAddress remoteAddress(Channel ch) { - final InetSocketAddress remoteAddress = this.remoteAddress; - if (remoteAddress != null) { - return remoteAddress; - } - - final InetSocketAddress newRemoteAddress = ChannelUtil.remoteAddress(ch); - this.remoteAddress = newRemoteAddress; - return newRemoteAddress; - } - - @Nullable - private InetSocketAddress localAddress(Channel ch) { - final InetSocketAddress localAddress = this.localAddress; - if (localAddress != null) { - return localAddress; - } - - final InetSocketAddress newLocalAddress = ChannelUtil.localAddress(ch); - this.localAddress = newLocalAddress; - return newLocalAddress; - } - private void handleOptions(ChannelHandlerContext ctx, ServiceRequestContext reqCtx) { respond(ctx, reqCtx, ResponseHeaders.builder(HttpStatus.OK) @@ -733,8 +707,6 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws E private ServiceRequestContext newEarlyRespondingRequestContext(Channel channel, DecodedHttpRequest req, ProxiedAddresses proxiedAddresses, InetAddress clientAddress, - InetSocketAddress remoteAddress, - InetSocketAddress localAddress, RoutingContext routingCtx, EventLoop eventLoop) { final ServiceConfig serviceConfig = routingCtx.virtualHost().fallbackServiceConfig(); @@ -849,11 +821,11 @@ private void handleRequestOrResponseComplete() { return; } if (req.isHttp1WebSocket()) { - config.serverMetrics().decreaseActiveHttp1WebSocketRequests(); + config.serverMetrics().decreaseActiveHttp1WebSocketRequests(localAddress); } else if (protocol.isExplicitHttp1()) { - config.serverMetrics().decreaseActiveHttp1Requests(); + config.serverMetrics().decreaseActiveHttp1Requests(localAddress); } else if (protocol.isExplicitHttp2()) { - config.serverMetrics().decreaseActiveHttp2Requests(); + config.serverMetrics().decreaseActiveHttp2Requests(localAddress); } // NB: logBuilder.endResponse() is called by HttpResponseSubscriber. diff --git a/core/src/main/java/com/linecorp/armeria/server/HttpServerPipelineConfigurator.java b/core/src/main/java/com/linecorp/armeria/server/HttpServerPipelineConfigurator.java index 284eff8ca1f..c5f38529ee5 100644 --- a/core/src/main/java/com/linecorp/armeria/server/HttpServerPipelineConfigurator.java +++ b/core/src/main/java/com/linecorp/armeria/server/HttpServerPipelineConfigurator.java @@ -216,7 +216,7 @@ private void configureHttp(ChannelPipeline p, @Nullable ProxiedAddresses proxied p.channel(), H1C, keepAliveHandler, config.http1HeaderNaming() ); p.addLast(TrafficLoggingHandler.SERVER); - final HttpServerHandler httpServerHandler = new HttpServerHandler(config, + final HttpServerHandler httpServerHandler = new HttpServerHandler(config, p.channel(), gracefulShutdownSupport, responseEncoder, H1C, proxiedAddresses); @@ -511,7 +511,7 @@ protected void configurePipeline(ChannelHandlerContext ctx, String protocol) thr private void addHttp2Handlers(ChannelHandlerContext ctx) { final ChannelPipeline p = ctx.pipeline(); p.addLast(newHttp2ConnectionHandler(p, SCHEME_HTTPS)); - p.addLast(new HttpServerHandler(config, + p.addLast(new HttpServerHandler(config, p.channel(), gracefulShutdownSupport, null, H2, proxiedAddresses)); } @@ -541,7 +541,7 @@ private void addHttpHandlers(ChannelHandlerContext ctx) { config.http1MaxInitialLineLength(), config.http1MaxHeaderSize(), config.http1MaxChunkSize())); - final HttpServerHandler httpServerHandler = new HttpServerHandler(config, + final HttpServerHandler httpServerHandler = new HttpServerHandler(config, ch, gracefulShutdownSupport, encoder, H1, proxiedAddresses); p.addLast(new Http1RequestDecoder(config, ch, SCHEME_HTTPS, encoder, httpServerHandler)); diff --git a/core/src/main/java/com/linecorp/armeria/server/Server.java b/core/src/main/java/com/linecorp/armeria/server/Server.java index a535c587a7f..03a01f39cb5 100644 --- a/core/src/main/java/com/linecorp/armeria/server/Server.java +++ b/core/src/main/java/com/linecorp/armeria/server/Server.java @@ -460,7 +460,14 @@ public void reconfigure(ServerConfigurator serverConfigurator) { requireNonNull(serverConfigurator, "serverConfigurator"); final ServerBuilder sb = builder(); serverConfigurator.reconfigure(sb); - final DefaultServerConfig newConfig = sb.buildServerConfig(config()); + final ImmutableList serverPorts; + lock.lock(); + try { + serverPorts = ImmutableList.copyOf(activePorts.values()); + } finally { + lock.unlock(); + } + final DefaultServerConfig newConfig = sb.buildServerConfig(serverPorts); newConfig.setServer(this); config.updateConfig(newConfig); // Invoke the serviceAdded() method in Service so that it can keep the reference to this Server or @@ -522,12 +529,12 @@ protected CompletionStage doStart(@Nullable Void arg) { try { doStart(primary).addListener(new ServerPortStartListener(primary)) .addListener(new NextServerPortStartListener(this, it, future)); - setupServerMetrics(); + // Chain the future to set up server metrics first before server start future is completed. + return future.thenAccept(unused -> bindServerMetricsToMeterRegistry()); } catch (Throwable cause) { future.completeExceptionally(cause); + return future; } - - return future; } private ChannelFuture doStart(ServerPort port) { @@ -582,7 +589,7 @@ private ChannelFuture doStart(ServerPort port) { return b.bind(localAddress); } - private void setupServerMetrics() { + private void bindServerMetricsToMeterRegistry() { final MeterRegistry meterRegistry = config.meterRegistry(); final GracefulShutdownSupport gracefulShutdownSupport = this.gracefulShutdownSupport; assert gracefulShutdownSupport != null; @@ -832,14 +839,17 @@ public void operationComplete(ChannelFuture f) { // Update the boss thread so its name contains the actual port. Thread.currentThread().setName(bossThreadName(actualPort)); + final InetSocketAddress actualLocalAddress = actualPort.localAddress(); lock.lock(); try { // Update the map of active ports. - activePorts.put(actualPort.localAddress(), actualPort); + activePorts.put(actualLocalAddress, actualPort); } finally { lock.unlock(); } + config().serverMetrics().addActivePort(actualPort); + if (logger.isInfoEnabled()) { if (isLocalPort(actualPort)) { port.protocols().forEach(p -> logger.info( diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerBuilder.java b/core/src/main/java/com/linecorp/armeria/server/ServerBuilder.java index 6366bb584f6..9e77ce63c84 100644 --- a/core/src/main/java/com/linecorp/armeria/server/ServerBuilder.java +++ b/core/src/main/java/com/linecorp/armeria/server/ServerBuilder.java @@ -2287,11 +2287,7 @@ public Server build() { return server; } - DefaultServerConfig buildServerConfig(ServerConfig existingConfig) { - return buildServerConfig(existingConfig.ports()); - } - - private DefaultServerConfig buildServerConfig(List serverPorts) { + DefaultServerConfig buildServerConfig(List serverPorts) { final AnnotatedServiceExtensions extensions = virtualHostTemplate.annotatedServiceExtensions(); assert extensions != null; diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java b/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java index 9a57bf65888..a703b8780ac 100644 --- a/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java +++ b/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java @@ -16,17 +16,28 @@ package com.linecorp.armeria.server; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.LongAdder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.linecorp.armeria.common.annotation.UnstableApi; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.binder.MeterBinder; +import io.netty.channel.unix.DomainSocketAddress; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; /** * A class that holds metrics related server. @@ -34,18 +45,190 @@ @UnstableApi public final class ServerMetrics implements MeterBinder { - private final LongAdder pendingHttp1Requests = new LongAdder(); - private final LongAdder pendingHttp2Requests = new LongAdder(); - private final LongAdder activeHttp1WebSocketRequests = new LongAdder(); - private final LongAdder activeHttp1Requests = new LongAdder(); - private final LongAdder activeHttp2Requests = new LongAdder(); + private static final Logger logger = LoggerFactory.getLogger(ServerMetrics.class); + + private static boolean warnedInvalidSocketAddress; + + private final Iterable ports; + private final boolean hasEphemeralPort; + + private final Map pendingHttp1Requests; + private final Map pendingHttp2Requests; + private final Map activeHttp1WebSocketRequests; + private final Map activeHttp1Requests; + private final Map activeHttp2Requests; + private final Map domainSocketPendingHttp1Requests; + private final Map domainSocketPendingHttp2Requests; + private final Map domainSocketActiveHttp1WebSocketRequests; + private final Map domainSocketActiveHttp1Requests; + private final Map domainSocketActiveHttp2Requests; /** * AtomicInteger is used to read the number of active connections frequently. */ - private final AtomicInteger activeConnections = new AtomicInteger(); + private final Map activeConnections; + private final Map domainSocketActiveConnections; + + ServerMetrics(Iterable ports) { + this.ports = ports; + + final ImmutableMap.Builder pendingHttp1RequestsBuilder = + ImmutableMap.builder(); + final ImmutableMap.Builder pendingHttp2RequestsBuilder = + ImmutableMap.builder(); + final ImmutableMap.Builder activeHttp1WebSocketRequestsBuilder = + ImmutableMap.builder(); + final ImmutableMap.Builder activeHttp1RequestsBuilder = + ImmutableMap.builder(); + final ImmutableMap.Builder activeHttp2RequestsBuilder = + ImmutableMap.builder(); + final ImmutableMap.Builder domainSocketPendingHttp1RequestsBuilder = + ImmutableMap.builder(); + final ImmutableMap.Builder domainSocketPendingHttp2RequestsBuilder = + ImmutableMap.builder(); + final ImmutableMap.Builder domainSocketActiveHttp1WebSocketRequestsBuilder = + ImmutableMap.builder(); + final ImmutableMap.Builder domainSocketActiveHttp1RequestsBuilder = + ImmutableMap.builder(); + final ImmutableMap.Builder domainSocketActiveHttp2RequestsBuilder = + ImmutableMap.builder(); + + final ImmutableMap.Builder ActiveConnectionsBuilder = + ImmutableMap.builder(); + final ImmutableMap.Builder domainSocketActiveConnectionsBuilder = + ImmutableMap.builder(); + + boolean hasEphemeralPort = false; + for (ServerPort serverPort : ports) { + final InetSocketAddress address = serverPort.localAddress(); + final int port = address.getPort(); + if (port == 0) { + hasEphemeralPort = true; + } else if (address instanceof com.linecorp.armeria.common.util.DomainSocketAddress) { + final String path = ((com.linecorp.armeria.common.util.DomainSocketAddress) address).path(); + domainSocketPendingHttp1RequestsBuilder.put(path, new LongAdder()); + domainSocketPendingHttp2RequestsBuilder.put(path, new LongAdder()); + domainSocketActiveHttp1WebSocketRequestsBuilder.put(path, new LongAdder()); + domainSocketActiveHttp1RequestsBuilder.put(path, new LongAdder()); + domainSocketActiveHttp2RequestsBuilder.put(path, new LongAdder()); + domainSocketActiveConnectionsBuilder.put(path, new AtomicInteger()); + } else { + pendingHttp1RequestsBuilder.put(port, new LongAdder()); + pendingHttp2RequestsBuilder.put(port, new LongAdder()); + activeHttp1WebSocketRequestsBuilder.put(port, new LongAdder()); + activeHttp1RequestsBuilder.put(port, new LongAdder()); + activeHttp2RequestsBuilder.put(port, new LongAdder()); + ActiveConnectionsBuilder.put(port, new AtomicInteger()); + } + } + // Put a dummy value to avoid NPE in case where an unknown address is returned from the channel. + // See the commit description of https://github.com/line/armeria/pull/5096 + pendingHttp1RequestsBuilder.put(1, new LongAdder()); + pendingHttp2RequestsBuilder.put(1, new LongAdder()); + activeHttp1WebSocketRequestsBuilder.put(1, new LongAdder()); + activeHttp1RequestsBuilder.put(1, new LongAdder()); + activeHttp2RequestsBuilder.put(1, new LongAdder()); + ActiveConnectionsBuilder.put(1, new AtomicInteger()); + + this.hasEphemeralPort = hasEphemeralPort; + + final Map domainSocketPendingHttp1Requests = + domainSocketPendingHttp1RequestsBuilder.build(); + final Map domainSocketPendingHttp2Requests = + domainSocketPendingHttp2RequestsBuilder.build(); + final Map domainSocketActiveHttp1WebSocketRequests = + domainSocketActiveHttp1WebSocketRequestsBuilder.build(); + final Map domainSocketActiveHttp1Requests = + domainSocketActiveHttp1RequestsBuilder.build(); + final Map domainSocketActiveHttp2Requests = + domainSocketActiveHttp2RequestsBuilder.build(); + final Map domainSocketActiveConnections = + domainSocketActiveConnectionsBuilder.build(); + + final Map pendingHttp1Requests = pendingHttp1RequestsBuilder.build(); + final Map pendingHttp2Requests = pendingHttp2RequestsBuilder.build(); + final Map activeHttp1WebSocketRequests = + activeHttp1WebSocketRequestsBuilder.build(); + final Map activeHttp1Requests = activeHttp1RequestsBuilder.build(); + final Map activeHttp2Requests = activeHttp2RequestsBuilder.build(); + final Map activeConnections = ActiveConnectionsBuilder.build(); - ServerMetrics() {} + if (!hasEphemeralPort) { + this.domainSocketPendingHttp1Requests = + new Object2ObjectOpenHashMap<>(domainSocketPendingHttp1Requests); + this.domainSocketPendingHttp2Requests = + new Object2ObjectOpenHashMap<>(domainSocketPendingHttp2Requests); + this.domainSocketActiveHttp1WebSocketRequests = + new Object2ObjectOpenHashMap<>(domainSocketActiveHttp1WebSocketRequests); + this.domainSocketActiveHttp1Requests = + new Object2ObjectOpenHashMap<>(domainSocketActiveHttp1Requests); + this.domainSocketActiveHttp2Requests = + new Object2ObjectOpenHashMap<>(domainSocketActiveHttp2Requests); + this.domainSocketActiveConnections = + new Object2ObjectOpenHashMap<>(domainSocketActiveConnections); + + this.pendingHttp1Requests = new Int2ObjectOpenHashMap<>(pendingHttp1Requests); + this.pendingHttp2Requests = new Int2ObjectOpenHashMap<>(pendingHttp2Requests); + this.activeHttp1WebSocketRequests = new Int2ObjectOpenHashMap<>(activeHttp1WebSocketRequests); + this.activeHttp1Requests = new Int2ObjectOpenHashMap<>(activeHttp1Requests); + this.activeHttp2Requests = new Int2ObjectOpenHashMap<>(activeHttp2Requests); + this.activeConnections = new Int2ObjectOpenHashMap<>(activeConnections); + } else { + // This is mostly for testing if the server has an ephemeral port. + this.domainSocketPendingHttp1Requests = + new ConcurrentHashMap<>(domainSocketPendingHttp1Requests); + this.domainSocketPendingHttp2Requests = + new ConcurrentHashMap<>(domainSocketPendingHttp2Requests); + this.domainSocketActiveHttp1WebSocketRequests = + new ConcurrentHashMap<>(domainSocketActiveHttp1WebSocketRequests); + this.domainSocketActiveHttp1Requests = + new ConcurrentHashMap<>(domainSocketActiveHttp1Requests); + this.domainSocketActiveHttp2Requests = + new ConcurrentHashMap<>(domainSocketActiveHttp2Requests); + this.domainSocketActiveConnections = + new ConcurrentHashMap<>(domainSocketActiveConnections); + + this.pendingHttp1Requests = new ConcurrentHashMap<>(pendingHttp1Requests); + this.pendingHttp2Requests = new ConcurrentHashMap<>(pendingHttp2Requests); + this.activeHttp1WebSocketRequests = new ConcurrentHashMap<>(activeHttp1WebSocketRequests); + this.activeHttp1Requests = new ConcurrentHashMap<>(activeHttp1Requests); + this.activeHttp2Requests = new ConcurrentHashMap<>(activeHttp2Requests); + this.activeConnections = new ConcurrentHashMap<>(activeConnections); + } + } + + void addActivePort(ServerPort actualPort) { + if (!hasEphemeralPort) { + return; + } + + final InetSocketAddress address = actualPort.localAddress(); + if (address instanceof com.linecorp.armeria.common.util.DomainSocketAddress) { + final String path = ((com.linecorp.armeria.common.util.DomainSocketAddress) address).path(); + if (domainSocketPendingHttp1Requests.containsKey(path)) { + return; + } + + domainSocketPendingHttp1Requests.put(path, new LongAdder()); + domainSocketPendingHttp2Requests.put(path, new LongAdder()); + domainSocketActiveHttp1WebSocketRequests.put(path, new LongAdder()); + domainSocketActiveHttp1Requests.put(path, new LongAdder()); + domainSocketActiveHttp2Requests.put(path, new LongAdder()); + domainSocketActiveConnections.put(path, new AtomicInteger()); + } else { + final int port = address.getPort(); + if (pendingHttp1Requests.containsKey(port)) { + return; + } + + pendingHttp1Requests.put(port, new LongAdder()); + pendingHttp2Requests.put(port, new LongAdder()); + activeHttp1WebSocketRequests.put(port, new LongAdder()); + activeHttp1Requests.put(port, new LongAdder()); + activeHttp2Requests.put(port, new LongAdder()); + activeConnections.put(port, new AtomicInteger()); + } + } /** * Returns the number of all pending requests. @@ -58,14 +241,14 @@ public long pendingRequests() { * Returns the number of pending http1 requests. */ public long pendingHttp1Requests() { - return pendingHttp1Requests.longValue(); + return pendingHttp1Requests.values().stream().mapToLong(LongAdder::longValue).sum(); } /** * Returns the number of pending http2 requests. */ public long pendingHttp2Requests() { - return pendingHttp2Requests.longValue(); + return pendingHttp2Requests.values().stream().mapToLong(LongAdder::longValue).sum(); } /** @@ -81,99 +264,218 @@ public long activeRequests() { * Returns the number of active http1 web socket requests. */ public long activeHttp1WebSocketRequests() { - return activeHttp1WebSocketRequests.longValue(); + return activeHttp1WebSocketRequests.values().stream().mapToLong(LongAdder::longValue).sum(); } /** * Returns the number of active http1 requests. */ public long activeHttp1Requests() { - return activeHttp1Requests.longValue(); + return activeHttp1Requests.values().stream().mapToLong(LongAdder::longValue).sum(); } /** * Returns the number of active http2 requests. */ public long activeHttp2Requests() { - return activeHttp2Requests.longValue(); + return activeHttp2Requests.values().stream().mapToLong(LongAdder::longValue).sum(); } /** * Returns the number of open connections. */ public int activeConnections() { - return activeConnections.get(); + return activeConnections.values().stream().mapToInt(AtomicInteger::get).sum(); } - void increasePendingHttp1Requests() { - pendingHttp1Requests.increment(); + void increasePendingHttp1Requests(SocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, pendingHttp1Requests, domainSocketPendingHttp1Requests, true); } - void decreasePendingHttp1Requests() { - pendingHttp1Requests.decrement(); + private static void increaseOrDecreaseRequests(SocketAddress socketAddress, + Map pendingHttp1Requests, + Map domainSocketPendingHttp1Requests, + boolean increase) { + // Armeria's DomainSocketAddress is used. + if (socketAddress instanceof com.linecorp.armeria.common.util.DomainSocketAddress) { + final String path = ((com.linecorp.armeria.common.util.DomainSocketAddress) socketAddress).path(); + final LongAdder longAdder = domainSocketPendingHttp1Requests.get(path); + assert longAdder != null; + if (increase) { + longAdder.increment(); + } else { + longAdder.decrement(); + } + } else if (socketAddress instanceof InetSocketAddress) { + final int port = ((InetSocketAddress) socketAddress).getPort(); + final LongAdder longAdder = pendingHttp1Requests.get(port); + assert longAdder != null; + if (increase) { + longAdder.increment(); + } else { + longAdder.decrement(); + } + } } - void increasePendingHttp2Requests() { - pendingHttp2Requests.increment(); + void decreasePendingHttp1Requests(SocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, pendingHttp1Requests, + domainSocketPendingHttp1Requests, false); } - void decreasePendingHttp2Requests() { - pendingHttp2Requests.decrement(); + void increasePendingHttp2Requests(SocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, pendingHttp2Requests, domainSocketPendingHttp2Requests, true); } - void increaseActiveHttp1Requests() { - activeHttp1Requests.increment(); + void decreasePendingHttp2Requests(SocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, pendingHttp2Requests, + domainSocketPendingHttp2Requests, false); } - void decreaseActiveHttp1Requests() { - activeHttp1Requests.decrement(); + void increaseActiveHttp1Requests(SocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp1Requests, domainSocketActiveHttp1Requests, true); } - void increaseActiveHttp1WebSocketRequests() { - activeHttp1WebSocketRequests.increment(); + void decreaseActiveHttp1Requests(SocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp1Requests, domainSocketActiveHttp1Requests, false); } - void decreaseActiveHttp1WebSocketRequests() { - activeHttp1WebSocketRequests.decrement(); + void increaseActiveHttp1WebSocketRequests(SocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp1WebSocketRequests, + domainSocketActiveHttp1WebSocketRequests, true); } - void increaseActiveHttp2Requests() { - activeHttp2Requests.increment(); + void decreaseActiveHttp1WebSocketRequests(SocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp1WebSocketRequests, + domainSocketActiveHttp1WebSocketRequests, false); } - void decreaseActiveHttp2Requests() { - activeHttp2Requests.decrement(); + void increaseActiveHttp2Requests(SocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp2Requests, domainSocketActiveHttp2Requests, true); } - int increaseActiveConnectionsAndGet() { - return activeConnections.incrementAndGet(); + void decreaseActiveHttp2Requests(SocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp2Requests, domainSocketActiveHttp2Requests, false); } - void decreaseActiveConnections() { - activeConnections.decrementAndGet(); + int increaseActiveConnectionsAndGet(SocketAddress socketAddress) { + if (socketAddress instanceof InetSocketAddress) { + final int port = ((InetSocketAddress) socketAddress).getPort(); + final AtomicInteger atomicInteger = activeConnections.get(port); + assert atomicInteger != null; + return atomicInteger.incrementAndGet(); + // Netty's DomainSocketAddress is used. + } else if (socketAddress instanceof DomainSocketAddress) { + final String path = ((DomainSocketAddress) socketAddress).path(); + final AtomicInteger atomicInteger = domainSocketActiveConnections.get(path); + assert atomicInteger != null; + return atomicInteger.incrementAndGet(); + } else { + if (!warnedInvalidSocketAddress) { + warnedInvalidSocketAddress = true; + logger.warn("Unexpected address type: {}", socketAddress.getClass().getName()); + } + return -1; + } + } + + void decreaseActiveConnections(SocketAddress socketAddress) { + if (socketAddress instanceof InetSocketAddress) { + final int port = ((InetSocketAddress) socketAddress).getPort(); + final AtomicInteger atomicInteger = activeConnections.get(port); + assert atomicInteger != null; + atomicInteger.decrementAndGet(); + } else if (socketAddress instanceof DomainSocketAddress) { + final String path = ((DomainSocketAddress) socketAddress).path(); + final AtomicInteger atomicInteger = domainSocketActiveConnections.get(path); + assert atomicInteger != null; + atomicInteger.decrementAndGet(); + } + // already warned in increaseActiveConnectionsAndGet } @Override public void bindTo(MeterRegistry meterRegistry) { - meterRegistry.gauge("armeria.server.connections", activeConnections); - // pending requests final String allRequestsMeterName = "armeria.server.all.requests"; - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(Tag.of("protocol", "http1"), Tag.of("state", "pending")), - pendingHttp1Requests); - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(Tag.of("protocol", "http2"), Tag.of("state", "pending")), - pendingHttp2Requests); - // Active requests - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(Tag.of("protocol", "http1"), Tag.of("state", "active")), - activeHttp1Requests); - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(Tag.of("protocol", "http2"), Tag.of("state", "active")), - activeHttp2Requests); - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(Tag.of("protocol", "http1.websocket"), Tag.of("state", "active")), - activeHttp1WebSocketRequests); + pendingHttp1Requests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", String.valueOf(port)); + meterRegistry.gauge(allRequestsMeterName, + ImmutableList.of(portTag, Tag.of("protocol", "http1"), + Tag.of("state", "pending")), + value); + }); + pendingHttp2Requests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", String.valueOf(port)); + meterRegistry.gauge(allRequestsMeterName, + ImmutableList.of(portTag, Tag.of("protocol", "http2"), + Tag.of("state", "pending")), + value); + }); + activeHttp1WebSocketRequests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", String.valueOf(port)); + meterRegistry.gauge(allRequestsMeterName, + ImmutableList.of(portTag, Tag.of("protocol", "http1.websocket"), + Tag.of("state", "active")), + value); + }); + activeHttp1Requests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", String.valueOf(port)); + meterRegistry.gauge(allRequestsMeterName, + ImmutableList.of(portTag, Tag.of("protocol", "http1"), + Tag.of("state", "active")), + value); + }); + activeHttp2Requests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", String.valueOf(port)); + meterRegistry.gauge(allRequestsMeterName, + ImmutableList.of(portTag, Tag.of("protocol", "http2"), + Tag.of("state", "active")), + value); + }); + activeConnections.forEach((port, value) -> { + final Tag portTag = Tag.of("port", String.valueOf(port)); + meterRegistry.gauge("armeria.server.connections", ImmutableList.of(portTag), value); + }); + + domainSocketPendingHttp1Requests.forEach((path, value) -> { + final Tag portTag = Tag.of("port", path); + meterRegistry.gauge(allRequestsMeterName, + ImmutableList.of(portTag, Tag.of("protocol", "http1"), + Tag.of("state", "pending")), + value); + }); + domainSocketPendingHttp2Requests.forEach((path, value) -> { + final Tag portTag = Tag.of("port", path); + meterRegistry.gauge(allRequestsMeterName, + ImmutableList.of(portTag, Tag.of("protocol", "http2"), + Tag.of("state", "pending")), + value); + }); + domainSocketActiveHttp1WebSocketRequests.forEach((path, value) -> { + final Tag portTag = Tag.of("port", path); + meterRegistry.gauge(allRequestsMeterName, + ImmutableList.of(portTag, Tag.of("protocol", "http1.websocket"), + Tag.of("state", "active")), + value); + }); + domainSocketActiveHttp1Requests.forEach((path, value) -> { + final Tag portTag = Tag.of("port", path); + meterRegistry.gauge(allRequestsMeterName, + ImmutableList.of(portTag, Tag.of("protocol", "http1"), + Tag.of("state", "active")), + value); + }); + domainSocketActiveHttp2Requests.forEach((path, value) -> { + final Tag portTag = Tag.of("port", path); + meterRegistry.gauge(allRequestsMeterName, + ImmutableList.of(portTag, Tag.of("protocol", "http2"), + Tag.of("state", "active")), + value); + }); + domainSocketActiveConnections.forEach((path, value) -> { + final Tag portTag = Tag.of("port", path); + meterRegistry.gauge(allRequestsMeterName, ImmutableList.of(portTag), value); + }); } @Override @@ -185,6 +487,13 @@ public String toString() { .add("pendingHttp2Requests", pendingHttp2Requests) .add("activeHttp2Requests", activeHttp2Requests) .add("activeConnections", activeConnections) + .add("domainSocketPendingHttp1Requests", domainSocketPendingHttp1Requests) + .add("domainSocketPendingHttp2Requests", domainSocketPendingHttp2Requests) + .add("domainSocketActiveHttp1WebSocketRequests", + domainSocketActiveHttp1WebSocketRequests) + .add("domainSocketActiveHttp1Requests", domainSocketActiveHttp1Requests) + .add("domainSocketActiveHttp2Requests", domainSocketActiveHttp2Requests) + .add("domainSocketActiveConnections", domainSocketActiveConnections) .toString(); } } diff --git a/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java b/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java index c814a955770..e2fded5ef63 100644 --- a/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java +++ b/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java @@ -18,15 +18,12 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import org.junit.jupiter.api.Test; - import io.netty.channel.embedded.EmbeddedChannel; class ConnectionLimitingHandlerTest { - @Test void testExceedMaxNumConnections() { - final ServerMetrics serverMetrics = new ServerMetrics(); + final ServerMetrics serverMetrics = null; final ConnectionLimitingHandler handler = new ConnectionLimitingHandler(1, serverMetrics); @@ -44,9 +41,8 @@ void testExceedMaxNumConnections() { assertThat(handler.numConnections()).isEqualTo(0); } - @Test void testMaxNumConnectionsRange() { - final ServerMetrics serverMetrics = new ServerMetrics(); + final ServerMetrics serverMetrics = null; final ConnectionLimitingHandler handler = new ConnectionLimitingHandler(Integer.MAX_VALUE, serverMetrics); assertThat(handler.maxNumConnections()).isEqualTo(Integer.MAX_VALUE); diff --git a/core/src/test/java/com/linecorp/armeria/server/ServerMetricsTest.java b/core/src/test/java/com/linecorp/armeria/server/ServerMetricsTest.java index 43106ff8cf3..be97d8bfdbc 100644 --- a/core/src/test/java/com/linecorp/armeria/server/ServerMetricsTest.java +++ b/core/src/test/java/com/linecorp/armeria/server/ServerMetricsTest.java @@ -18,6 +18,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import java.net.InetSocketAddress; import java.time.Duration; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -28,6 +29,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import com.google.common.collect.ImmutableList; + import com.linecorp.armeria.client.BlockingWebClient; import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.ClientOptions; @@ -139,41 +142,43 @@ public ExchangeType exchangeType(RoutingContext routingContext) { @Test void pendingRequests() { - final ServerMetrics serverMetrics = new ServerMetrics(); - - serverMetrics.increasePendingHttp1Requests(); + final ServerPort activePort = server.server().activePort(); + final ServerMetrics serverMetrics = new ServerMetrics(ImmutableList.of(activePort)); + final InetSocketAddress localAddress = activePort.localAddress(); + serverMetrics.increasePendingHttp1Requests(localAddress); assertThat(serverMetrics.pendingRequests()).isEqualTo(1); - serverMetrics.increasePendingHttp2Requests(); + serverMetrics.increasePendingHttp2Requests(localAddress); assertThat(serverMetrics.pendingRequests()).isEqualTo(2); - serverMetrics.decreasePendingHttp1Requests(); + serverMetrics.decreasePendingHttp1Requests(localAddress); assertThat(serverMetrics.pendingRequests()).isEqualTo(1); - serverMetrics.decreasePendingHttp2Requests(); + serverMetrics.decreasePendingHttp2Requests(localAddress); assertThat(serverMetrics.pendingRequests()).isZero(); } @Test void activeRequests() { - final ServerMetrics serverMetrics = new ServerMetrics(); - - serverMetrics.increaseActiveHttp1Requests(); + final ServerPort activePort = server.server().activePort(); + final ServerMetrics serverMetrics = new ServerMetrics(ImmutableList.of(activePort)); + final InetSocketAddress localAddress = activePort.localAddress(); + serverMetrics.increaseActiveHttp1Requests(localAddress); assertThat(serverMetrics.activeRequests()).isEqualTo(1); - serverMetrics.increaseActiveHttp1WebSocketRequests(); + serverMetrics.increaseActiveHttp1WebSocketRequests(localAddress); assertThat(serverMetrics.activeRequests()).isEqualTo(2); - serverMetrics.increaseActiveHttp2Requests(); + serverMetrics.increaseActiveHttp2Requests(localAddress); assertThat(serverMetrics.activeRequests()).isEqualTo(3); - serverMetrics.decreaseActiveHttp1WebSocketRequests(); + serverMetrics.decreaseActiveHttp1WebSocketRequests(localAddress); assertThat(serverMetrics.activeRequests()).isEqualTo(2); - serverMetrics.decreaseActiveHttp1Requests(); + serverMetrics.decreaseActiveHttp1Requests(localAddress); assertThat(serverMetrics.activeRequests()).isEqualTo(1); - serverMetrics.decreaseActiveHttp2Requests(); + serverMetrics.decreaseActiveHttp2Requests(localAddress); assertThat(serverMetrics.activeRequests()).isZero(); } @@ -296,10 +301,10 @@ public boolean matches(String key) { final String protocolName = protocol == SessionProtocol.H1C ? "http1" : "http2"; // armeria.server.active.requests.all#value is measured by ServerMetrics - assertThat(meters).containsKey("armeria.server.all.requests#value{protocol=" + protocolName + - ",state=active}"); - assertThat(meters).containsKey("armeria.server.all.requests#value{protocol=" + protocolName + - ",state=pending}"); + assertThat(meters).containsKey("armeria.server.all.requests#value{port=" + server.httpPort() + + ",protocol=" + protocolName + ",state=active}"); + assertThat(meters).containsKey("armeria.server.all.requests#value{port=" + server.httpPort() + + ",protocol=" + protocolName + ",state=pending}"); }); } } From 472b6ce227b607cb330396ac783610d9ec5b11d1 Mon Sep 17 00:00:00 2001 From: minwoox Date: Thu, 20 Feb 2025 13:11:40 +0900 Subject: [PATCH 2/8] Fix --- .../server/ConnectionLimitingHandler.java | 10 ++++- .../armeria/server/HttpServerHandler.java | 8 ++-- .../armeria/server/ServerMetrics.java | 45 +++++++++---------- .../server/ConnectionLimitingHandlerTest.java | 40 ++++++++++++++--- 4 files changed, 66 insertions(+), 37 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java b/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java index 3dc9cdcf976..c0e925edee8 100644 --- a/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java +++ b/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java @@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.LongAdder; import org.slf4j.Logger; @@ -54,6 +55,10 @@ final class ConnectionLimitingHandler extends ChannelInboundHandlerAdapter { private final int maxNumConnections; private final ServerMetrics serverMetrics; + /** + * AtomicInteger is used to read the number of active connections frequently. + */ + private final AtomicInteger activeConnections = new AtomicInteger(); private final AtomicBoolean loggingScheduled = new AtomicBoolean(); private final LongAdder numDroppedConnections = new LongAdder(); @@ -66,8 +71,9 @@ final class ConnectionLimitingHandler extends ChannelInboundHandlerAdapter { public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { final Channel child = (Channel) msg; final InetSocketAddress localAddress = firstNonNull(ChannelUtil.localAddress(child), UNKNOWN_ADDR); - final int conn = serverMetrics.increaseActiveConnectionsAndGet(localAddress); + final int conn = activeConnections.incrementAndGet(); if (conn > 0 && conn <= maxNumConnections) { + serverMetrics.increaseActiveConnections(localAddress); childChannels.add(child); child.closeFuture().addListener(future -> { childChannels.remove(child); @@ -75,7 +81,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception }); super.channelRead(ctx, msg); } else { - serverMetrics.decreaseActiveConnections(localAddress); + activeConnections.decrementAndGet(); // Set linger option to 0 so that the server doesn't get too many TIME_WAIT states. child.config().setOption(ChannelOption.SO_LINGER, 0); diff --git a/core/src/main/java/com/linecorp/armeria/server/HttpServerHandler.java b/core/src/main/java/com/linecorp/armeria/server/HttpServerHandler.java index d75e08cd13a..6e7fcb184dc 100644 --- a/core/src/main/java/com/linecorp/armeria/server/HttpServerHandler.java +++ b/core/src/main/java/com/linecorp/armeria/server/HttpServerHandler.java @@ -384,7 +384,6 @@ private static void incrementLocalWindowSize(ChannelPipeline pipeline, int delta private void handleRequest(ChannelHandlerContext ctx, DecodedHttpRequest req) throws Exception { final ServerHttpObjectEncoder responseEncoder = this.responseEncoder; assert responseEncoder != null; - final Channel channel = ctx.channel(); // Ignore the request received after the last request, // because we are going to close the connection after sending the last response. @@ -401,6 +400,7 @@ private void handleRequest(ChannelHandlerContext ctx, DecodedHttpRequest req) th responseEncoder.keepAliveHandler().disconnectWhenFinished(); } + final Channel channel = ctx.channel(); final RequestHeaders headers = req.headers(); final ProxiedAddresses proxiedAddresses = determineProxiedAddresses(headers); final InetAddress clientAddress = config.clientAddressMapper().apply(proxiedAddresses).getAddress(); @@ -451,15 +451,13 @@ private void handleRequest(ChannelHandlerContext ctx, DecodedHttpRequest req) th if (serviceEventLoop.inEventLoop()) { return serve0(req, service, reqCtx, req.isHttp1WebSocket()); } - return serveInServiceEventLoop(req, service, reqCtx, serviceEventLoop, - req.isHttp1WebSocket()); + return serveInServiceEventLoop(req, service, reqCtx, serviceEventLoop, req.isHttp1WebSocket()); })); } else { if (serviceEventLoop.inEventLoop()) { res = serve0(req, service, reqCtx, req.isHttp1WebSocket()); } else { - res = serveInServiceEventLoop(req, service, reqCtx, serviceEventLoop, - req.isHttp1WebSocket()); + res = serveInServiceEventLoop(req, service, reqCtx, serviceEventLoop, req.isHttp1WebSocket()); } } res = res.recover(cause -> { diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java b/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java index a703b8780ac..b9774f4de41 100644 --- a/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java +++ b/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java @@ -31,11 +31,11 @@ import com.google.common.collect.ImmutableMap; import com.linecorp.armeria.common.annotation.UnstableApi; +import com.linecorp.armeria.common.util.DomainSocketAddress; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.binder.MeterBinder; -import io.netty.channel.unix.DomainSocketAddress; import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; @@ -104,8 +104,8 @@ public final class ServerMetrics implements MeterBinder { final int port = address.getPort(); if (port == 0) { hasEphemeralPort = true; - } else if (address instanceof com.linecorp.armeria.common.util.DomainSocketAddress) { - final String path = ((com.linecorp.armeria.common.util.DomainSocketAddress) address).path(); + } else if (address instanceof DomainSocketAddress) { + final String path = ((DomainSocketAddress) address).path(); domainSocketPendingHttp1RequestsBuilder.put(path, new LongAdder()); domainSocketPendingHttp2RequestsBuilder.put(path, new LongAdder()); domainSocketActiveHttp1WebSocketRequestsBuilder.put(path, new LongAdder()); @@ -203,8 +203,8 @@ void addActivePort(ServerPort actualPort) { } final InetSocketAddress address = actualPort.localAddress(); - if (address instanceof com.linecorp.armeria.common.util.DomainSocketAddress) { - final String path = ((com.linecorp.armeria.common.util.DomainSocketAddress) address).path(); + if (address instanceof DomainSocketAddress) { + final String path = ((DomainSocketAddress) address).path(); if (domainSocketPendingHttp1Requests.containsKey(path)) { return; } @@ -296,9 +296,8 @@ private static void increaseOrDecreaseRequests(SocketAddress socketAddress, Map pendingHttp1Requests, Map domainSocketPendingHttp1Requests, boolean increase) { - // Armeria's DomainSocketAddress is used. - if (socketAddress instanceof com.linecorp.armeria.common.util.DomainSocketAddress) { - final String path = ((com.linecorp.armeria.common.util.DomainSocketAddress) socketAddress).path(); + if (socketAddress instanceof DomainSocketAddress) { + final String path = ((DomainSocketAddress) socketAddress).path(); final LongAdder longAdder = domainSocketPendingHttp1Requests.get(path); assert longAdder != null; if (increase) { @@ -358,38 +357,36 @@ void decreaseActiveHttp2Requests(SocketAddress socketAddress) { increaseOrDecreaseRequests(socketAddress, activeHttp2Requests, domainSocketActiveHttp2Requests, false); } - int increaseActiveConnectionsAndGet(SocketAddress socketAddress) { - if (socketAddress instanceof InetSocketAddress) { - final int port = ((InetSocketAddress) socketAddress).getPort(); - final AtomicInteger atomicInteger = activeConnections.get(port); - assert atomicInteger != null; - return atomicInteger.incrementAndGet(); - // Netty's DomainSocketAddress is used. - } else if (socketAddress instanceof DomainSocketAddress) { + void increaseActiveConnections(SocketAddress socketAddress) { + if (socketAddress instanceof DomainSocketAddress) { final String path = ((DomainSocketAddress) socketAddress).path(); final AtomicInteger atomicInteger = domainSocketActiveConnections.get(path); assert atomicInteger != null; - return atomicInteger.incrementAndGet(); + atomicInteger.incrementAndGet(); + } else if (socketAddress instanceof InetSocketAddress) { + final int port = ((InetSocketAddress) socketAddress).getPort(); + final AtomicInteger atomicInteger = activeConnections.get(port); + assert atomicInteger != null; + atomicInteger.incrementAndGet(); } else { if (!warnedInvalidSocketAddress) { warnedInvalidSocketAddress = true; logger.warn("Unexpected address type: {}", socketAddress.getClass().getName()); } - return -1; } } void decreaseActiveConnections(SocketAddress socketAddress) { - if (socketAddress instanceof InetSocketAddress) { - final int port = ((InetSocketAddress) socketAddress).getPort(); - final AtomicInteger atomicInteger = activeConnections.get(port); - assert atomicInteger != null; - atomicInteger.decrementAndGet(); - } else if (socketAddress instanceof DomainSocketAddress) { + if (socketAddress instanceof DomainSocketAddress) { final String path = ((DomainSocketAddress) socketAddress).path(); final AtomicInteger atomicInteger = domainSocketActiveConnections.get(path); assert atomicInteger != null; atomicInteger.decrementAndGet(); + } else if (socketAddress instanceof InetSocketAddress) { + final int port = ((InetSocketAddress) socketAddress).getPort(); + final AtomicInteger atomicInteger = activeConnections.get(port); + assert atomicInteger != null; + atomicInteger.decrementAndGet(); } // already warned in increaseActiveConnectionsAndGet } diff --git a/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java b/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java index e2fded5ef63..24612f1020f 100644 --- a/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java +++ b/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java @@ -18,21 +18,45 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.net.InetSocketAddress; + +import org.junit.jupiter.api.Test; + +import com.google.common.collect.ImmutableList; + +import com.linecorp.armeria.common.SessionProtocol; + import io.netty.channel.embedded.EmbeddedChannel; class ConnectionLimitingHandlerTest { + @Test void testExceedMaxNumConnections() { - final ServerMetrics serverMetrics = null; - final ConnectionLimitingHandler handler = - new ConnectionLimitingHandler(1, serverMetrics); + // The port is not used in this test. + final InetSocketAddress localAddress1 = new InetSocketAddress(2); + final InetSocketAddress localAddress2 = new InetSocketAddress(3); + final ServerPort serverPort1 = new ServerPort(localAddress1, SessionProtocol.HTTP); + final ServerPort serverPort2 = new ServerPort(localAddress2, SessionProtocol.HTTP); + final ServerMetrics serverMetrics = new ServerMetrics(ImmutableList.of(serverPort1, serverPort2)); + + final ConnectionLimitingHandler handler = new ConnectionLimitingHandler(1, serverMetrics); + final EmbeddedChannel ch1 = new EmbeddedChannel(handler) { + @Override + public InetSocketAddress localAddress() { + return localAddress1; + } + }; - final EmbeddedChannel ch1 = new EmbeddedChannel(handler); ch1.writeInbound(ch1); assertThat(handler.numConnections()).isEqualTo(1); assertThat(ch1.isActive()).isTrue(); - final EmbeddedChannel ch2 = new EmbeddedChannel(handler); + final EmbeddedChannel ch2 = new EmbeddedChannel(handler) { + @Override + public InetSocketAddress localAddress() { + return localAddress2; + } + }; ch2.writeInbound(ch2); assertThat(handler.numConnections()).isEqualTo(1); assertThat(ch2.isActive()).isFalse(); @@ -41,8 +65,12 @@ void testExceedMaxNumConnections() { assertThat(handler.numConnections()).isEqualTo(0); } + @Test void testMaxNumConnectionsRange() { - final ServerMetrics serverMetrics = null; + // The port is not used in this test. + final InetSocketAddress localAddress = new InetSocketAddress(2); + final ServerMetrics serverMetrics = new ServerMetrics(ImmutableList.of( + new ServerPort(localAddress, SessionProtocol.HTTP))); final ConnectionLimitingHandler handler = new ConnectionLimitingHandler(Integer.MAX_VALUE, serverMetrics); assertThat(handler.maxNumConnections()).isEqualTo(Integer.MAX_VALUE); From 73fe06de1f684699377a6bda420c7d1792db1440 Mon Sep 17 00:00:00 2001 From: minwoox Date: Thu, 20 Feb 2025 13:52:28 +0900 Subject: [PATCH 3/8] Fix --- .../linecorp/armeria/server/ConnectionLimitingHandler.java | 5 +++-- .../main/java/com/linecorp/armeria/server/ServerMetrics.java | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java b/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java index c0e925edee8..858efb0dfd1 100644 --- a/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java +++ b/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java @@ -70,13 +70,14 @@ final class ConnectionLimitingHandler extends ChannelInboundHandlerAdapter { @Override public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { final Channel child = (Channel) msg; - final InetSocketAddress localAddress = firstNonNull(ChannelUtil.localAddress(child), UNKNOWN_ADDR); final int conn = activeConnections.incrementAndGet(); if (conn > 0 && conn <= maxNumConnections) { + final InetSocketAddress localAddress = firstNonNull(ChannelUtil.localAddress(child), UNKNOWN_ADDR); serverMetrics.increaseActiveConnections(localAddress); childChannels.add(child); child.closeFuture().addListener(future -> { childChannels.remove(child); + activeConnections.decrementAndGet(); serverMetrics.decreaseActiveConnections(localAddress); }); super.channelRead(ctx, msg); @@ -116,7 +117,7 @@ public int maxNumConnections() { * Returns the number of open connections. */ public int numConnections() { - return serverMetrics.activeConnections(); + return activeConnections.get(); } /** diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java b/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java index b9774f4de41..520ac373654 100644 --- a/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java +++ b/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java @@ -388,7 +388,7 @@ void decreaseActiveConnections(SocketAddress socketAddress) { assert atomicInteger != null; atomicInteger.decrementAndGet(); } - // already warned in increaseActiveConnectionsAndGet + // already warned in increaseActiveConnections } @Override From 8663daedc6971cd947d9e0da8ca189fab2160b78 Mon Sep 17 00:00:00 2001 From: minwoox Date: Thu, 20 Feb 2025 21:36:37 +0900 Subject: [PATCH 4/8] Split to InetSocketServerMetrics and DomainSocketServerMetrics --- .../armeria/server/DefaultServerConfig.java | 2 +- .../armeria/server/DefaultServerMetrics.java | 204 ++++++++ .../server/DomainSocketServerMetrics.java | 238 +++++++++ .../server/InetSocketServerMetrics.java | 278 ++++++++++ .../armeria/server/ServerMetrics.java | 475 +++--------------- .../server/ConnectionLimitingHandlerTest.java | 5 +- .../armeria/server/ServerMetricsTest.java | 4 +- 7 files changed, 786 insertions(+), 420 deletions(-) create mode 100644 core/src/main/java/com/linecorp/armeria/server/DefaultServerMetrics.java create mode 100644 core/src/main/java/com/linecorp/armeria/server/DomainSocketServerMetrics.java create mode 100644 core/src/main/java/com/linecorp/armeria/server/InetSocketServerMetrics.java diff --git a/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java b/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java index 511d65cf8b7..d87e4744618 100644 --- a/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java +++ b/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java @@ -266,7 +266,7 @@ final class DefaultServerConfig implements ServerConfig { this.absoluteUriTransformer = castAbsoluteUriTransformer; this.unloggedExceptionsReportIntervalMillis = unloggedExceptionsReportIntervalMillis; this.shutdownSupports = ImmutableList.copyOf(requireNonNull(shutdownSupports, "shutdownSupports")); - serverMetrics = new ServerMetrics(ports); + serverMetrics = new DefaultServerMetrics(ports); } private static Int2ObjectMap> buildDomainAndPortMapping( diff --git a/core/src/main/java/com/linecorp/armeria/server/DefaultServerMetrics.java b/core/src/main/java/com/linecorp/armeria/server/DefaultServerMetrics.java new file mode 100644 index 00000000000..d6ce9e363bf --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/server/DefaultServerMetrics.java @@ -0,0 +1,204 @@ +/* + * Copyright 2024 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.server; + +import java.net.InetSocketAddress; + +import com.google.common.base.MoreObjects; + +import com.linecorp.armeria.common.util.DomainSocketAddress; + +import io.micrometer.core.instrument.MeterRegistry; + +final class DefaultServerMetrics implements ServerMetrics { + + static final String ALL_REQUESTS_METER_NAME = "armeria.server.all.requests"; + static final String ALL_CONNECTIONS_METER_NAME = "armeria.server.connections"; + + private final InetSocketServerMetrics inetSocketServerMetrics; + private final DomainSocketServerMetrics domainSocketServerMetrics; + + DefaultServerMetrics(Iterable ports) { + inetSocketServerMetrics = new InetSocketServerMetrics(ports); + domainSocketServerMetrics = new DomainSocketServerMetrics(ports); + } + + @Override + public void addActivePort(ServerPort actualPort) { + if (!(actualPort.localAddress() instanceof DomainSocketAddress)) { + inetSocketServerMetrics.addActivePort(actualPort); + } + } + + @Override + public long pendingHttp1Requests() { + return inetSocketServerMetrics.pendingHttp1Requests() + + domainSocketServerMetrics.pendingHttp1Requests(); + } + + @Override + public long pendingHttp2Requests() { + return inetSocketServerMetrics.pendingHttp2Requests() + + domainSocketServerMetrics.pendingHttp2Requests(); + } + + @Override + public long activeHttp1WebSocketRequests() { + return inetSocketServerMetrics.activeHttp1WebSocketRequests() + + domainSocketServerMetrics.activeHttp1WebSocketRequests(); + } + + @Override + public long activeHttp1Requests() { + return inetSocketServerMetrics.activeHttp1Requests() + + domainSocketServerMetrics.activeHttp1Requests(); + } + + @Override + public long activeHttp2Requests() { + return inetSocketServerMetrics.activeHttp2Requests() + + domainSocketServerMetrics.activeHttp2Requests(); + } + + @Override + public long activeConnections() { + return inetSocketServerMetrics.activeConnections() + + domainSocketServerMetrics.activeConnections(); + } + + @Override + public void increasePendingHttp1Requests(InetSocketAddress socketAddress) { + if (socketAddress instanceof DomainSocketAddress) { + domainSocketServerMetrics.increasePendingHttp1Requests(socketAddress); + } else { + inetSocketServerMetrics.increasePendingHttp1Requests(socketAddress); + } + } + + @Override + public void decreasePendingHttp1Requests(InetSocketAddress socketAddress) { + if (socketAddress instanceof DomainSocketAddress) { + domainSocketServerMetrics.decreasePendingHttp1Requests(socketAddress); + } else { + inetSocketServerMetrics.decreasePendingHttp1Requests(socketAddress); + } + } + + @Override + public void increasePendingHttp2Requests(InetSocketAddress socketAddress) { + if (socketAddress instanceof DomainSocketAddress) { + domainSocketServerMetrics.increasePendingHttp2Requests(socketAddress); + } else { + inetSocketServerMetrics.increasePendingHttp2Requests(socketAddress); + } + } + + @Override + public void decreasePendingHttp2Requests(InetSocketAddress socketAddress) { + if (socketAddress instanceof DomainSocketAddress) { + domainSocketServerMetrics.decreasePendingHttp2Requests(socketAddress); + } else { + inetSocketServerMetrics.decreasePendingHttp2Requests(socketAddress); + } + } + + @Override + public void increaseActiveHttp1Requests(InetSocketAddress socketAddress) { + if (socketAddress instanceof DomainSocketAddress) { + domainSocketServerMetrics.increaseActiveHttp1Requests(socketAddress); + } else { + inetSocketServerMetrics.increaseActiveHttp1Requests(socketAddress); + } + } + + @Override + public void decreaseActiveHttp1Requests(InetSocketAddress socketAddress) { + if (socketAddress instanceof DomainSocketAddress) { + domainSocketServerMetrics.decreaseActiveHttp1Requests(socketAddress); + } else { + inetSocketServerMetrics.decreaseActiveHttp1Requests(socketAddress); + } + } + + @Override + public void increaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress) { + if (socketAddress instanceof DomainSocketAddress) { + domainSocketServerMetrics.increaseActiveHttp1WebSocketRequests(socketAddress); + } else { + inetSocketServerMetrics.increaseActiveHttp1WebSocketRequests(socketAddress); + } + } + + @Override + public void decreaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress) { + if (socketAddress instanceof DomainSocketAddress) { + domainSocketServerMetrics.decreaseActiveHttp1WebSocketRequests(socketAddress); + } else { + inetSocketServerMetrics.decreaseActiveHttp1WebSocketRequests(socketAddress); + } + } + + @Override + public void increaseActiveHttp2Requests(InetSocketAddress socketAddress) { + if (socketAddress instanceof DomainSocketAddress) { + domainSocketServerMetrics.increaseActiveHttp2Requests(socketAddress); + } else { + inetSocketServerMetrics.increaseActiveHttp2Requests(socketAddress); + } + } + + @Override + public void decreaseActiveHttp2Requests(InetSocketAddress socketAddress) { + if (socketAddress instanceof DomainSocketAddress) { + domainSocketServerMetrics.decreaseActiveHttp2Requests(socketAddress); + } else { + inetSocketServerMetrics.decreaseActiveHttp2Requests(socketAddress); + } + } + + @Override + public void increaseActiveConnections(InetSocketAddress socketAddress) { + if (socketAddress instanceof DomainSocketAddress) { + domainSocketServerMetrics.increaseActiveConnections(socketAddress); + } else { + inetSocketServerMetrics.increaseActiveConnections(socketAddress); + } + } + + @Override + public void decreaseActiveConnections(InetSocketAddress socketAddress) { + if (socketAddress instanceof DomainSocketAddress) { + domainSocketServerMetrics.decreaseActiveConnections(socketAddress); + } else { + inetSocketServerMetrics.decreaseActiveConnections(socketAddress); + } + } + + @Override + public void bindTo(MeterRegistry meterRegistry) { + inetSocketServerMetrics.bindTo(meterRegistry); + domainSocketServerMetrics.bindTo(meterRegistry); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("inetSocketServerMetrics", inetSocketServerMetrics) + .add("domainSocketServerMetrics", domainSocketServerMetrics) + .toString(); + } +} diff --git a/core/src/main/java/com/linecorp/armeria/server/DomainSocketServerMetrics.java b/core/src/main/java/com/linecorp/armeria/server/DomainSocketServerMetrics.java new file mode 100644 index 00000000000..0bbadc5e486 --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/server/DomainSocketServerMetrics.java @@ -0,0 +1,238 @@ +/* + * Copyright 2025 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.armeria.server; + +import static com.linecorp.armeria.server.DefaultServerMetrics.ALL_REQUESTS_METER_NAME; + +import java.net.InetSocketAddress; +import java.util.Map; +import java.util.concurrent.atomic.LongAdder; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import com.linecorp.armeria.common.util.DomainSocketAddress; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; + +final class DomainSocketServerMetrics implements ServerMetrics { + + private final Map pendingHttp1Requests; + private final Map pendingHttp2Requests; + private final Map activeHttp1WebSocketRequests; + private final Map activeHttp1Requests; + private final Map activeHttp2Requests; + private final Map activeConnections; + + DomainSocketServerMetrics(Iterable serverPorts) { + final ImmutableMap.Builder pendingHttp1RequestsBuilder = ImmutableMap.builder(); + final ImmutableMap.Builder pendingHttp2RequestsBuilder = ImmutableMap.builder(); + final ImmutableMap.Builder activeHttp1WebSocketRequestsBuilder = + ImmutableMap.builder(); + final ImmutableMap.Builder activeHttp1RequestsBuilder = ImmutableMap.builder(); + final ImmutableMap.Builder activeHttp2RequestsBuilder = ImmutableMap.builder(); + final ImmutableMap.Builder activeConnectionsBuilder = ImmutableMap.builder(); + serverPorts.forEach(port -> { + final InetSocketAddress localAddress = port.localAddress(); + if (!(localAddress instanceof DomainSocketAddress)) { + return; + } + final String path = ((DomainSocketAddress) localAddress).path(); + pendingHttp1RequestsBuilder.put(path, new LongAdder()); + pendingHttp2RequestsBuilder.put(path, new LongAdder()); + activeHttp1WebSocketRequestsBuilder.put(path, new LongAdder()); + activeHttp1RequestsBuilder.put(path, new LongAdder()); + activeHttp2RequestsBuilder.put(path, new LongAdder()); + activeConnectionsBuilder.put(path, new LongAdder()); + }); + pendingHttp1Requests = pendingHttp1RequestsBuilder.build(); + pendingHttp2Requests = pendingHttp2RequestsBuilder.build(); + activeHttp1WebSocketRequests = activeHttp1WebSocketRequestsBuilder.build(); + activeHttp1Requests = activeHttp1RequestsBuilder.build(); + activeHttp2Requests = activeHttp2RequestsBuilder.build(); + activeConnections = activeConnectionsBuilder.build(); + } + + @Override + public void addActivePort(ServerPort actualPort) { + // Do nothing because the port is already added in the constructor. + } + + @Override + public long pendingHttp1Requests() { + return sum(pendingHttp1Requests); + } + + private static long sum(Map map) { + return map.values().stream().mapToLong(LongAdder::longValue).sum(); + } + + @Override + public long pendingHttp2Requests() { + return sum(pendingHttp2Requests); + } + + @Override + public long activeHttp1WebSocketRequests() { + return sum(activeHttp1WebSocketRequests); + } + + @Override + public long activeHttp1Requests() { + return sum(activeHttp1Requests); + } + + @Override + public long activeHttp2Requests() { + return sum(activeHttp2Requests); + } + + @Override + public long activeConnections() { + return sum(activeConnections); + } + + @Override + public void increasePendingHttp1Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, pendingHttp1Requests, true); + } + + private static void increaseOrDecreaseRequests(InetSocketAddress socketAddress, + Map requests, + boolean increase) { + assert socketAddress instanceof DomainSocketAddress; + final String path = ((DomainSocketAddress) socketAddress).path(); + final LongAdder longAdder = requests.get(path); + assert longAdder != null; + if (increase) { + longAdder.increment(); + } else { + longAdder.decrement(); + } + } + + @Override + public void decreasePendingHttp1Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, pendingHttp1Requests, false); + } + + @Override + public void increasePendingHttp2Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, pendingHttp2Requests, true); + } + + @Override + public void decreasePendingHttp2Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, pendingHttp2Requests, false); + } + + @Override + public void increaseActiveHttp1Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp1Requests, true); + } + + @Override + public void decreaseActiveHttp1Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp1Requests, false); + } + + @Override + public void increaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp1WebSocketRequests, true); + } + + @Override + public void decreaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp1WebSocketRequests, false); + } + + @Override + public void increaseActiveHttp2Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp2Requests, true); + } + + @Override + public void decreaseActiveHttp2Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp2Requests, false); + } + + @Override + public void increaseActiveConnections(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeConnections, true); + } + + @Override + public void decreaseActiveConnections(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeConnections, false); + } + + @Override + public void bindTo(MeterRegistry meterRegistry) { + pendingHttp1Requests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", port); + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http1"), + Tag.of("state", "pending")), + value); + }); + pendingHttp2Requests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", port); + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http2"), + Tag.of("state", "pending")), + value); + }); + activeHttp1WebSocketRequests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", port); + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http1.websocket"), + Tag.of("state", "active")), + value); + }); + activeHttp1Requests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", port); + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http1"), + Tag.of("state", "active")), + value); + }); + activeHttp2Requests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", port); + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http2"), + Tag.of("state", "active")), + value); + }); + activeConnections.forEach((port, value) -> { + final Tag portTag = Tag.of("port", port); + meterRegistry.gauge("ALL_CONNECTIONS_METER_NAME", ImmutableList.of(portTag), value); + }); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("pendingHttp1Requests", pendingHttp1Requests) + .add("activeHttp1WebSocketRequests", activeHttp1WebSocketRequests) + .add("activeHttp1Requests", activeHttp1Requests) + .add("pendingHttp2Requests", pendingHttp2Requests) + .add("activeHttp2Requests", activeHttp2Requests) + .add("activeConnections", activeConnections) + .toString(); + } +} diff --git a/core/src/main/java/com/linecorp/armeria/server/InetSocketServerMetrics.java b/core/src/main/java/com/linecorp/armeria/server/InetSocketServerMetrics.java new file mode 100644 index 00000000000..78a865b48bf --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/server/InetSocketServerMetrics.java @@ -0,0 +1,278 @@ +/* + * Copyright 2025 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.armeria.server; + +import static com.linecorp.armeria.server.DefaultServerMetrics.ALL_CONNECTIONS_METER_NAME; +import static com.linecorp.armeria.server.DefaultServerMetrics.ALL_REQUESTS_METER_NAME; + +import java.net.InetSocketAddress; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.LongAdder; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; + +import com.linecorp.armeria.common.util.DomainSocketAddress; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; + +final class InetSocketServerMetrics implements ServerMetrics { + + private final boolean hasEphemeralPort; + + private final Map pendingHttp1Requests; + private final Map pendingHttp2Requests; + private final Map activeHttp1WebSocketRequests; + private final Map activeHttp1Requests; + private final Map activeHttp2Requests; + private final Map activeConnections; + + InetSocketServerMetrics(Iterable serverPorts) { + boolean hasEphemeralPort = false; + final Map pendingHttp1Requests = new Int2ObjectOpenHashMap<>(); + final Map pendingHttp2Requests = new Int2ObjectOpenHashMap<>(); + final Map activeHttp1WebSocketRequests = new Int2ObjectOpenHashMap<>(); + final Map activeHttp1Requests = new Int2ObjectOpenHashMap<>(); + final Map activeHttp2Requests = new Int2ObjectOpenHashMap<>(); + final Map activeConnections = new Int2ObjectOpenHashMap<>(); + for (ServerPort serverPort : serverPorts) { + final InetSocketAddress localAddress = serverPort.localAddress(); + if (!(localAddress instanceof DomainSocketAddress)) { + final int port = localAddress.getPort(); + if (port == 0) { + hasEphemeralPort = true; + } else { + pendingHttp1Requests.put(port, new LongAdder()); + pendingHttp2Requests.put(port, new LongAdder()); + activeHttp1WebSocketRequests.put(port, new LongAdder()); + activeHttp1Requests.put(port, new LongAdder()); + activeHttp2Requests.put(port, new LongAdder()); + activeConnections.put(port, new LongAdder()); + } + } + } + // Put a dummy value to avoid NPE in case where an unknown address is returned from the channel. + // See the commit description of https://github.com/line/armeria/pull/5096 + pendingHttp1Requests.put(1, new LongAdder()); + pendingHttp2Requests.put(1, new LongAdder()); + activeHttp1WebSocketRequests.put(1, new LongAdder()); + activeHttp1Requests.put(1, new LongAdder()); + activeHttp2Requests.put(1, new LongAdder()); + activeConnections.put(1, new LongAdder()); + + this.hasEphemeralPort = hasEphemeralPort; + if (hasEphemeralPort) { + // This is mostly for testing if the server has an ephemeral port. + this.pendingHttp1Requests = new ConcurrentHashMap<>(pendingHttp1Requests); + this.pendingHttp2Requests = new ConcurrentHashMap<>(pendingHttp2Requests); + this.activeHttp1WebSocketRequests = new ConcurrentHashMap<>(activeHttp1WebSocketRequests); + this.activeHttp1Requests = new ConcurrentHashMap<>(activeHttp1Requests); + this.activeHttp2Requests = new ConcurrentHashMap<>(activeHttp2Requests); + this.activeConnections = new ConcurrentHashMap<>(activeConnections); + } else { + this.pendingHttp1Requests = pendingHttp1Requests; + this.pendingHttp2Requests = pendingHttp2Requests; + this.activeHttp1WebSocketRequests = activeHttp1WebSocketRequests; + this.activeHttp1Requests = activeHttp1Requests; + this.activeHttp2Requests = activeHttp2Requests; + this.activeConnections = activeConnections; + } + } + + @Override + public void addActivePort(ServerPort actualPort) { + if (!hasEphemeralPort) { + return; + } + + final InetSocketAddress address = actualPort.localAddress(); + final int port = address.getPort(); + if (pendingHttp1Requests.containsKey(port)) { + return; + } + + pendingHttp1Requests.put(port, new LongAdder()); + pendingHttp2Requests.put(port, new LongAdder()); + activeHttp1WebSocketRequests.put(port, new LongAdder()); + activeHttp1Requests.put(port, new LongAdder()); + activeHttp2Requests.put(port, new LongAdder()); + activeConnections.put(port, new LongAdder()); + } + + @Override + public long pendingHttp1Requests() { + return sum(pendingHttp1Requests); + } + + private static long sum(Map map) { + return map.values().stream().mapToLong(LongAdder::longValue).sum(); + } + + @Override + public long pendingHttp2Requests() { + return sum(pendingHttp2Requests); + } + + @Override + public long activeHttp1WebSocketRequests() { + return sum(activeHttp1WebSocketRequests); + } + + @Override + public long activeHttp1Requests() { + return sum(activeHttp1Requests); + } + + @Override + public long activeHttp2Requests() { + return sum(activeHttp2Requests); + } + + @Override + public long activeConnections() { + return sum(activeConnections); + } + + @Override + public void increasePendingHttp1Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, pendingHttp1Requests, true); + } + + private static void increaseOrDecreaseRequests(InetSocketAddress socketAddress, + Map requests, + boolean increase) { + final LongAdder longAdder = requests.get(socketAddress.getPort()); + assert longAdder != null; + if (increase) { + longAdder.increment(); + } else { + longAdder.decrement(); + } + } + + @Override + public void decreasePendingHttp1Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, pendingHttp1Requests, false); + } + + @Override + public void increasePendingHttp2Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, pendingHttp2Requests, true); + } + + @Override + public void decreasePendingHttp2Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, pendingHttp2Requests, false); + } + + @Override + public void increaseActiveHttp1Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp1Requests, true); + } + + @Override + public void decreaseActiveHttp1Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp1Requests, false); + } + + @Override + public void increaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp1WebSocketRequests, true); + } + + @Override + public void decreaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp1WebSocketRequests, false); + } + + @Override + public void increaseActiveHttp2Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp2Requests, true); + } + + @Override + public void decreaseActiveHttp2Requests(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeHttp2Requests, false); + } + + @Override + public void increaseActiveConnections(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeConnections, true); + } + + @Override + public void decreaseActiveConnections(InetSocketAddress socketAddress) { + increaseOrDecreaseRequests(socketAddress, activeConnections, false); + } + + @Override + public void bindTo(MeterRegistry meterRegistry) { + pendingHttp1Requests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", String.valueOf(port)); + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http1"), + Tag.of("state", "pending")), + value); + }); + pendingHttp2Requests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", String.valueOf(port)); + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http2"), + Tag.of("state", "pending")), + value); + }); + activeHttp1WebSocketRequests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", String.valueOf(port)); + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http1.websocket"), + Tag.of("state", "active")), + value); + }); + activeHttp1Requests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", String.valueOf(port)); + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http1"), + Tag.of("state", "active")), + value); + }); + activeHttp2Requests.forEach((port, value) -> { + final Tag portTag = Tag.of("port", String.valueOf(port)); + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http2"), + Tag.of("state", "active")), + value); + }); + activeConnections.forEach((port, value) -> { + final Tag portTag = Tag.of("port", String.valueOf(port)); + meterRegistry.gauge(ALL_CONNECTIONS_METER_NAME, ImmutableList.of(portTag), value); + }); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("pendingHttp1Requests", pendingHttp1Requests) + .add("activeHttp1WebSocketRequests", activeHttp1WebSocketRequests) + .add("activeHttp1Requests", activeHttp1Requests) + .add("pendingHttp2Requests", pendingHttp2Requests) + .add("activeHttp2Requests", activeHttp2Requests) + .add("activeConnections", activeConnections) + .toString(); + } +} diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java b/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java index 520ac373654..92b81c7bb94 100644 --- a/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java +++ b/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java @@ -17,244 +17,43 @@ package com.linecorp.armeria.server; import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.LongAdder; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.base.MoreObjects; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.linecorp.armeria.common.annotation.UnstableApi; -import com.linecorp.armeria.common.util.DomainSocketAddress; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.binder.MeterBinder; -import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; -import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; /** - * A class that holds metrics related server. + * An interface that provides the requests and connections metrics information of the server. */ @UnstableApi -public final class ServerMetrics implements MeterBinder { - - private static final Logger logger = LoggerFactory.getLogger(ServerMetrics.class); - - private static boolean warnedInvalidSocketAddress; - - private final Iterable ports; - private final boolean hasEphemeralPort; - - private final Map pendingHttp1Requests; - private final Map pendingHttp2Requests; - private final Map activeHttp1WebSocketRequests; - private final Map activeHttp1Requests; - private final Map activeHttp2Requests; - private final Map domainSocketPendingHttp1Requests; - private final Map domainSocketPendingHttp2Requests; - private final Map domainSocketActiveHttp1WebSocketRequests; - private final Map domainSocketActiveHttp1Requests; - private final Map domainSocketActiveHttp2Requests; +public interface ServerMetrics extends MeterBinder { /** - * AtomicInteger is used to read the number of active connections frequently. + * Adds the {@link ServerPort} to the {@link ServerMetrics}. */ - private final Map activeConnections; - private final Map domainSocketActiveConnections; - - ServerMetrics(Iterable ports) { - this.ports = ports; - - final ImmutableMap.Builder pendingHttp1RequestsBuilder = - ImmutableMap.builder(); - final ImmutableMap.Builder pendingHttp2RequestsBuilder = - ImmutableMap.builder(); - final ImmutableMap.Builder activeHttp1WebSocketRequestsBuilder = - ImmutableMap.builder(); - final ImmutableMap.Builder activeHttp1RequestsBuilder = - ImmutableMap.builder(); - final ImmutableMap.Builder activeHttp2RequestsBuilder = - ImmutableMap.builder(); - final ImmutableMap.Builder domainSocketPendingHttp1RequestsBuilder = - ImmutableMap.builder(); - final ImmutableMap.Builder domainSocketPendingHttp2RequestsBuilder = - ImmutableMap.builder(); - final ImmutableMap.Builder domainSocketActiveHttp1WebSocketRequestsBuilder = - ImmutableMap.builder(); - final ImmutableMap.Builder domainSocketActiveHttp1RequestsBuilder = - ImmutableMap.builder(); - final ImmutableMap.Builder domainSocketActiveHttp2RequestsBuilder = - ImmutableMap.builder(); - - final ImmutableMap.Builder ActiveConnectionsBuilder = - ImmutableMap.builder(); - final ImmutableMap.Builder domainSocketActiveConnectionsBuilder = - ImmutableMap.builder(); - - boolean hasEphemeralPort = false; - for (ServerPort serverPort : ports) { - final InetSocketAddress address = serverPort.localAddress(); - final int port = address.getPort(); - if (port == 0) { - hasEphemeralPort = true; - } else if (address instanceof DomainSocketAddress) { - final String path = ((DomainSocketAddress) address).path(); - domainSocketPendingHttp1RequestsBuilder.put(path, new LongAdder()); - domainSocketPendingHttp2RequestsBuilder.put(path, new LongAdder()); - domainSocketActiveHttp1WebSocketRequestsBuilder.put(path, new LongAdder()); - domainSocketActiveHttp1RequestsBuilder.put(path, new LongAdder()); - domainSocketActiveHttp2RequestsBuilder.put(path, new LongAdder()); - domainSocketActiveConnectionsBuilder.put(path, new AtomicInteger()); - } else { - pendingHttp1RequestsBuilder.put(port, new LongAdder()); - pendingHttp2RequestsBuilder.put(port, new LongAdder()); - activeHttp1WebSocketRequestsBuilder.put(port, new LongAdder()); - activeHttp1RequestsBuilder.put(port, new LongAdder()); - activeHttp2RequestsBuilder.put(port, new LongAdder()); - ActiveConnectionsBuilder.put(port, new AtomicInteger()); - } - } - // Put a dummy value to avoid NPE in case where an unknown address is returned from the channel. - // See the commit description of https://github.com/line/armeria/pull/5096 - pendingHttp1RequestsBuilder.put(1, new LongAdder()); - pendingHttp2RequestsBuilder.put(1, new LongAdder()); - activeHttp1WebSocketRequestsBuilder.put(1, new LongAdder()); - activeHttp1RequestsBuilder.put(1, new LongAdder()); - activeHttp2RequestsBuilder.put(1, new LongAdder()); - ActiveConnectionsBuilder.put(1, new AtomicInteger()); - - this.hasEphemeralPort = hasEphemeralPort; - - final Map domainSocketPendingHttp1Requests = - domainSocketPendingHttp1RequestsBuilder.build(); - final Map domainSocketPendingHttp2Requests = - domainSocketPendingHttp2RequestsBuilder.build(); - final Map domainSocketActiveHttp1WebSocketRequests = - domainSocketActiveHttp1WebSocketRequestsBuilder.build(); - final Map domainSocketActiveHttp1Requests = - domainSocketActiveHttp1RequestsBuilder.build(); - final Map domainSocketActiveHttp2Requests = - domainSocketActiveHttp2RequestsBuilder.build(); - final Map domainSocketActiveConnections = - domainSocketActiveConnectionsBuilder.build(); - - final Map pendingHttp1Requests = pendingHttp1RequestsBuilder.build(); - final Map pendingHttp2Requests = pendingHttp2RequestsBuilder.build(); - final Map activeHttp1WebSocketRequests = - activeHttp1WebSocketRequestsBuilder.build(); - final Map activeHttp1Requests = activeHttp1RequestsBuilder.build(); - final Map activeHttp2Requests = activeHttp2RequestsBuilder.build(); - final Map activeConnections = ActiveConnectionsBuilder.build(); - - if (!hasEphemeralPort) { - this.domainSocketPendingHttp1Requests = - new Object2ObjectOpenHashMap<>(domainSocketPendingHttp1Requests); - this.domainSocketPendingHttp2Requests = - new Object2ObjectOpenHashMap<>(domainSocketPendingHttp2Requests); - this.domainSocketActiveHttp1WebSocketRequests = - new Object2ObjectOpenHashMap<>(domainSocketActiveHttp1WebSocketRequests); - this.domainSocketActiveHttp1Requests = - new Object2ObjectOpenHashMap<>(domainSocketActiveHttp1Requests); - this.domainSocketActiveHttp2Requests = - new Object2ObjectOpenHashMap<>(domainSocketActiveHttp2Requests); - this.domainSocketActiveConnections = - new Object2ObjectOpenHashMap<>(domainSocketActiveConnections); - - this.pendingHttp1Requests = new Int2ObjectOpenHashMap<>(pendingHttp1Requests); - this.pendingHttp2Requests = new Int2ObjectOpenHashMap<>(pendingHttp2Requests); - this.activeHttp1WebSocketRequests = new Int2ObjectOpenHashMap<>(activeHttp1WebSocketRequests); - this.activeHttp1Requests = new Int2ObjectOpenHashMap<>(activeHttp1Requests); - this.activeHttp2Requests = new Int2ObjectOpenHashMap<>(activeHttp2Requests); - this.activeConnections = new Int2ObjectOpenHashMap<>(activeConnections); - } else { - // This is mostly for testing if the server has an ephemeral port. - this.domainSocketPendingHttp1Requests = - new ConcurrentHashMap<>(domainSocketPendingHttp1Requests); - this.domainSocketPendingHttp2Requests = - new ConcurrentHashMap<>(domainSocketPendingHttp2Requests); - this.domainSocketActiveHttp1WebSocketRequests = - new ConcurrentHashMap<>(domainSocketActiveHttp1WebSocketRequests); - this.domainSocketActiveHttp1Requests = - new ConcurrentHashMap<>(domainSocketActiveHttp1Requests); - this.domainSocketActiveHttp2Requests = - new ConcurrentHashMap<>(domainSocketActiveHttp2Requests); - this.domainSocketActiveConnections = - new ConcurrentHashMap<>(domainSocketActiveConnections); - - this.pendingHttp1Requests = new ConcurrentHashMap<>(pendingHttp1Requests); - this.pendingHttp2Requests = new ConcurrentHashMap<>(pendingHttp2Requests); - this.activeHttp1WebSocketRequests = new ConcurrentHashMap<>(activeHttp1WebSocketRequests); - this.activeHttp1Requests = new ConcurrentHashMap<>(activeHttp1Requests); - this.activeHttp2Requests = new ConcurrentHashMap<>(activeHttp2Requests); - this.activeConnections = new ConcurrentHashMap<>(activeConnections); - } - } - - void addActivePort(ServerPort actualPort) { - if (!hasEphemeralPort) { - return; - } - - final InetSocketAddress address = actualPort.localAddress(); - if (address instanceof DomainSocketAddress) { - final String path = ((DomainSocketAddress) address).path(); - if (domainSocketPendingHttp1Requests.containsKey(path)) { - return; - } - - domainSocketPendingHttp1Requests.put(path, new LongAdder()); - domainSocketPendingHttp2Requests.put(path, new LongAdder()); - domainSocketActiveHttp1WebSocketRequests.put(path, new LongAdder()); - domainSocketActiveHttp1Requests.put(path, new LongAdder()); - domainSocketActiveHttp2Requests.put(path, new LongAdder()); - domainSocketActiveConnections.put(path, new AtomicInteger()); - } else { - final int port = address.getPort(); - if (pendingHttp1Requests.containsKey(port)) { - return; - } - - pendingHttp1Requests.put(port, new LongAdder()); - pendingHttp2Requests.put(port, new LongAdder()); - activeHttp1WebSocketRequests.put(port, new LongAdder()); - activeHttp1Requests.put(port, new LongAdder()); - activeHttp2Requests.put(port, new LongAdder()); - activeConnections.put(port, new AtomicInteger()); - } - } + void addActivePort(ServerPort actualPort); /** * Returns the number of all pending requests. */ - public long pendingRequests() { + default long pendingRequests() { return pendingHttp1Requests() + pendingHttp2Requests(); } /** * Returns the number of pending http1 requests. */ - public long pendingHttp1Requests() { - return pendingHttp1Requests.values().stream().mapToLong(LongAdder::longValue).sum(); - } + long pendingHttp1Requests(); /** * Returns the number of pending http2 requests. */ - public long pendingHttp2Requests() { - return pendingHttp2Requests.values().stream().mapToLong(LongAdder::longValue).sum(); - } + long pendingHttp2Requests(); /** * Returns the number of all active requests. */ - public long activeRequests() { + default long activeRequests() { return activeHttp1WebSocketRequests() + activeHttp1Requests() + activeHttp2Requests(); @@ -263,234 +62,80 @@ public long activeRequests() { /** * Returns the number of active http1 web socket requests. */ - public long activeHttp1WebSocketRequests() { - return activeHttp1WebSocketRequests.values().stream().mapToLong(LongAdder::longValue).sum(); - } + long activeHttp1WebSocketRequests(); /** * Returns the number of active http1 requests. */ - public long activeHttp1Requests() { - return activeHttp1Requests.values().stream().mapToLong(LongAdder::longValue).sum(); - } + long activeHttp1Requests(); /** * Returns the number of active http2 requests. */ - public long activeHttp2Requests() { - return activeHttp2Requests.values().stream().mapToLong(LongAdder::longValue).sum(); - } + long activeHttp2Requests(); /** * Returns the number of open connections. */ - public int activeConnections() { - return activeConnections.values().stream().mapToInt(AtomicInteger::get).sum(); - } - - void increasePendingHttp1Requests(SocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, pendingHttp1Requests, domainSocketPendingHttp1Requests, true); - } - - private static void increaseOrDecreaseRequests(SocketAddress socketAddress, - Map pendingHttp1Requests, - Map domainSocketPendingHttp1Requests, - boolean increase) { - if (socketAddress instanceof DomainSocketAddress) { - final String path = ((DomainSocketAddress) socketAddress).path(); - final LongAdder longAdder = domainSocketPendingHttp1Requests.get(path); - assert longAdder != null; - if (increase) { - longAdder.increment(); - } else { - longAdder.decrement(); - } - } else if (socketAddress instanceof InetSocketAddress) { - final int port = ((InetSocketAddress) socketAddress).getPort(); - final LongAdder longAdder = pendingHttp1Requests.get(port); - assert longAdder != null; - if (increase) { - longAdder.increment(); - } else { - longAdder.decrement(); - } - } - } - - void decreasePendingHttp1Requests(SocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, pendingHttp1Requests, - domainSocketPendingHttp1Requests, false); - } - - void increasePendingHttp2Requests(SocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, pendingHttp2Requests, domainSocketPendingHttp2Requests, true); - } + long activeConnections(); - void decreasePendingHttp2Requests(SocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, pendingHttp2Requests, - domainSocketPendingHttp2Requests, false); - } + /** + * Returns the number of all connections. + */ + void increasePendingHttp1Requests(InetSocketAddress socketAddress); - void increaseActiveHttp1Requests(SocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp1Requests, domainSocketActiveHttp1Requests, true); - } + /** + * Decreases the number of all pending http1 requests. + */ + void decreasePendingHttp1Requests(InetSocketAddress socketAddress); - void decreaseActiveHttp1Requests(SocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp1Requests, domainSocketActiveHttp1Requests, false); - } + /** + * Increases the number of all pending http2 requests. + */ + void increasePendingHttp2Requests(InetSocketAddress socketAddress); - void increaseActiveHttp1WebSocketRequests(SocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp1WebSocketRequests, - domainSocketActiveHttp1WebSocketRequests, true); - } + /** + * Decreases the number of all pending http2 requests. + */ + void decreasePendingHttp2Requests(InetSocketAddress socketAddress); - void decreaseActiveHttp1WebSocketRequests(SocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp1WebSocketRequests, - domainSocketActiveHttp1WebSocketRequests, false); - } + /** + * Increases the number of all active http1 requests. + */ + void increaseActiveHttp1Requests(InetSocketAddress socketAddress); - void increaseActiveHttp2Requests(SocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp2Requests, domainSocketActiveHttp2Requests, true); - } + /** + * Decreases the number of all active http1 requests. + */ + void decreaseActiveHttp1Requests(InetSocketAddress socketAddress); - void decreaseActiveHttp2Requests(SocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp2Requests, domainSocketActiveHttp2Requests, false); - } + /** + * Increases the number of all active http1 web socket requests. + */ + void increaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress); - void increaseActiveConnections(SocketAddress socketAddress) { - if (socketAddress instanceof DomainSocketAddress) { - final String path = ((DomainSocketAddress) socketAddress).path(); - final AtomicInteger atomicInteger = domainSocketActiveConnections.get(path); - assert atomicInteger != null; - atomicInteger.incrementAndGet(); - } else if (socketAddress instanceof InetSocketAddress) { - final int port = ((InetSocketAddress) socketAddress).getPort(); - final AtomicInteger atomicInteger = activeConnections.get(port); - assert atomicInteger != null; - atomicInteger.incrementAndGet(); - } else { - if (!warnedInvalidSocketAddress) { - warnedInvalidSocketAddress = true; - logger.warn("Unexpected address type: {}", socketAddress.getClass().getName()); - } - } - } + /** + * Decreases the number of all active http1 web socket requests. + */ + void decreaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress); - void decreaseActiveConnections(SocketAddress socketAddress) { - if (socketAddress instanceof DomainSocketAddress) { - final String path = ((DomainSocketAddress) socketAddress).path(); - final AtomicInteger atomicInteger = domainSocketActiveConnections.get(path); - assert atomicInteger != null; - atomicInteger.decrementAndGet(); - } else if (socketAddress instanceof InetSocketAddress) { - final int port = ((InetSocketAddress) socketAddress).getPort(); - final AtomicInteger atomicInteger = activeConnections.get(port); - assert atomicInteger != null; - atomicInteger.decrementAndGet(); - } - // already warned in increaseActiveConnections - } + /** + * Increases the number of all active http2 requests. + */ + void increaseActiveHttp2Requests(InetSocketAddress socketAddress); - @Override - public void bindTo(MeterRegistry meterRegistry) { - final String allRequestsMeterName = "armeria.server.all.requests"; - pendingHttp1Requests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", String.valueOf(port)); - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(portTag, Tag.of("protocol", "http1"), - Tag.of("state", "pending")), - value); - }); - pendingHttp2Requests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", String.valueOf(port)); - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(portTag, Tag.of("protocol", "http2"), - Tag.of("state", "pending")), - value); - }); - activeHttp1WebSocketRequests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", String.valueOf(port)); - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(portTag, Tag.of("protocol", "http1.websocket"), - Tag.of("state", "active")), - value); - }); - activeHttp1Requests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", String.valueOf(port)); - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(portTag, Tag.of("protocol", "http1"), - Tag.of("state", "active")), - value); - }); - activeHttp2Requests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", String.valueOf(port)); - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(portTag, Tag.of("protocol", "http2"), - Tag.of("state", "active")), - value); - }); - activeConnections.forEach((port, value) -> { - final Tag portTag = Tag.of("port", String.valueOf(port)); - meterRegistry.gauge("armeria.server.connections", ImmutableList.of(portTag), value); - }); + /** + * Decreases the number of all active http2 requests. + */ + void decreaseActiveHttp2Requests(InetSocketAddress socketAddress); - domainSocketPendingHttp1Requests.forEach((path, value) -> { - final Tag portTag = Tag.of("port", path); - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(portTag, Tag.of("protocol", "http1"), - Tag.of("state", "pending")), - value); - }); - domainSocketPendingHttp2Requests.forEach((path, value) -> { - final Tag portTag = Tag.of("port", path); - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(portTag, Tag.of("protocol", "http2"), - Tag.of("state", "pending")), - value); - }); - domainSocketActiveHttp1WebSocketRequests.forEach((path, value) -> { - final Tag portTag = Tag.of("port", path); - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(portTag, Tag.of("protocol", "http1.websocket"), - Tag.of("state", "active")), - value); - }); - domainSocketActiveHttp1Requests.forEach((path, value) -> { - final Tag portTag = Tag.of("port", path); - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(portTag, Tag.of("protocol", "http1"), - Tag.of("state", "active")), - value); - }); - domainSocketActiveHttp2Requests.forEach((path, value) -> { - final Tag portTag = Tag.of("port", path); - meterRegistry.gauge(allRequestsMeterName, - ImmutableList.of(portTag, Tag.of("protocol", "http2"), - Tag.of("state", "active")), - value); - }); - domainSocketActiveConnections.forEach((path, value) -> { - final Tag portTag = Tag.of("port", path); - meterRegistry.gauge(allRequestsMeterName, ImmutableList.of(portTag), value); - }); - } + /** + * Increases the number of open connections. + */ + void increaseActiveConnections(InetSocketAddress socketAddress); - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("pendingHttp1Requests", pendingHttp1Requests) - .add("activeHttp1WebSocketRequests", activeHttp1WebSocketRequests) - .add("activeHttp1Requests", activeHttp1Requests) - .add("pendingHttp2Requests", pendingHttp2Requests) - .add("activeHttp2Requests", activeHttp2Requests) - .add("activeConnections", activeConnections) - .add("domainSocketPendingHttp1Requests", domainSocketPendingHttp1Requests) - .add("domainSocketPendingHttp2Requests", domainSocketPendingHttp2Requests) - .add("domainSocketActiveHttp1WebSocketRequests", - domainSocketActiveHttp1WebSocketRequests) - .add("domainSocketActiveHttp1Requests", domainSocketActiveHttp1Requests) - .add("domainSocketActiveHttp2Requests", domainSocketActiveHttp2Requests) - .add("domainSocketActiveConnections", domainSocketActiveConnections) - .toString(); - } + /** + * Decreases the number of open connections. + */ + void decreaseActiveConnections(InetSocketAddress socketAddress); } diff --git a/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java b/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java index 24612f1020f..63c16f43adf 100644 --- a/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java +++ b/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java @@ -37,7 +37,8 @@ void testExceedMaxNumConnections() { final InetSocketAddress localAddress2 = new InetSocketAddress(3); final ServerPort serverPort1 = new ServerPort(localAddress1, SessionProtocol.HTTP); final ServerPort serverPort2 = new ServerPort(localAddress2, SessionProtocol.HTTP); - final ServerMetrics serverMetrics = new ServerMetrics(ImmutableList.of(serverPort1, serverPort2)); + final ServerMetrics serverMetrics = + new DefaultServerMetrics(ImmutableList.of(serverPort1, serverPort2)); final ConnectionLimitingHandler handler = new ConnectionLimitingHandler(1, serverMetrics); final EmbeddedChannel ch1 = new EmbeddedChannel(handler) { @@ -69,7 +70,7 @@ public InetSocketAddress localAddress() { void testMaxNumConnectionsRange() { // The port is not used in this test. final InetSocketAddress localAddress = new InetSocketAddress(2); - final ServerMetrics serverMetrics = new ServerMetrics(ImmutableList.of( + final ServerMetrics serverMetrics = new DefaultServerMetrics(ImmutableList.of( new ServerPort(localAddress, SessionProtocol.HTTP))); final ConnectionLimitingHandler handler = new ConnectionLimitingHandler(Integer.MAX_VALUE, serverMetrics); diff --git a/core/src/test/java/com/linecorp/armeria/server/ServerMetricsTest.java b/core/src/test/java/com/linecorp/armeria/server/ServerMetricsTest.java index be97d8bfdbc..48d1d25e5e2 100644 --- a/core/src/test/java/com/linecorp/armeria/server/ServerMetricsTest.java +++ b/core/src/test/java/com/linecorp/armeria/server/ServerMetricsTest.java @@ -143,7 +143,7 @@ public ExchangeType exchangeType(RoutingContext routingContext) { @Test void pendingRequests() { final ServerPort activePort = server.server().activePort(); - final ServerMetrics serverMetrics = new ServerMetrics(ImmutableList.of(activePort)); + final ServerMetrics serverMetrics = new DefaultServerMetrics(ImmutableList.of(activePort)); final InetSocketAddress localAddress = activePort.localAddress(); serverMetrics.increasePendingHttp1Requests(localAddress); assertThat(serverMetrics.pendingRequests()).isEqualTo(1); @@ -161,7 +161,7 @@ void pendingRequests() { @Test void activeRequests() { final ServerPort activePort = server.server().activePort(); - final ServerMetrics serverMetrics = new ServerMetrics(ImmutableList.of(activePort)); + final ServerMetrics serverMetrics = new DefaultServerMetrics(ImmutableList.of(activePort)); final InetSocketAddress localAddress = activePort.localAddress(); serverMetrics.increaseActiveHttp1Requests(localAddress); assertThat(serverMetrics.activeRequests()).isEqualTo(1); From ed85e5b66ac5c4ecda3f6568e8ec63d00a65468f Mon Sep 17 00:00:00 2001 From: minwoox Date: Thu, 20 Feb 2025 21:43:33 +0900 Subject: [PATCH 5/8] Fix --- .../server/DomainSocketServerMetrics.java | 28 ++++++++++--------- .../com/linecorp/armeria/server/Server.java | 4 +-- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/server/DomainSocketServerMetrics.java b/core/src/main/java/com/linecorp/armeria/server/DomainSocketServerMetrics.java index 0bbadc5e486..90868949e09 100644 --- a/core/src/main/java/com/linecorp/armeria/server/DomainSocketServerMetrics.java +++ b/core/src/main/java/com/linecorp/armeria/server/DomainSocketServerMetrics.java @@ -15,6 +15,7 @@ */ package com.linecorp.armeria.server; +import static com.linecorp.armeria.server.DefaultServerMetrics.ALL_CONNECTIONS_METER_NAME; import static com.linecorp.armeria.server.DefaultServerMetrics.ALL_REQUESTS_METER_NAME; import java.net.InetSocketAddress; @@ -183,44 +184,45 @@ public void decreaseActiveConnections(InetSocketAddress socketAddress) { @Override public void bindTo(MeterRegistry meterRegistry) { - pendingHttp1Requests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", port); + pendingHttp1Requests.forEach((path, value) -> { + // The path is used as the 'port' tag. + final Tag portTag = Tag.of("port", path); meterRegistry.gauge(ALL_REQUESTS_METER_NAME, ImmutableList.of(portTag, Tag.of("protocol", "http1"), Tag.of("state", "pending")), value); }); - pendingHttp2Requests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", port); + pendingHttp2Requests.forEach((path, value) -> { + final Tag portTag = Tag.of("port", path); meterRegistry.gauge(ALL_REQUESTS_METER_NAME, ImmutableList.of(portTag, Tag.of("protocol", "http2"), Tag.of("state", "pending")), value); }); - activeHttp1WebSocketRequests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", port); + activeHttp1WebSocketRequests.forEach((path, value) -> { + final Tag portTag = Tag.of("port", path); meterRegistry.gauge(ALL_REQUESTS_METER_NAME, ImmutableList.of(portTag, Tag.of("protocol", "http1.websocket"), Tag.of("state", "active")), value); }); - activeHttp1Requests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", port); + activeHttp1Requests.forEach((path, value) -> { + final Tag portTag = Tag.of("port", path); meterRegistry.gauge(ALL_REQUESTS_METER_NAME, ImmutableList.of(portTag, Tag.of("protocol", "http1"), Tag.of("state", "active")), value); }); - activeHttp2Requests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", port); + activeHttp2Requests.forEach((path, value) -> { + final Tag portTag = Tag.of("port", path); meterRegistry.gauge(ALL_REQUESTS_METER_NAME, ImmutableList.of(portTag, Tag.of("protocol", "http2"), Tag.of("state", "active")), value); }); - activeConnections.forEach((port, value) -> { - final Tag portTag = Tag.of("port", port); - meterRegistry.gauge("ALL_CONNECTIONS_METER_NAME", ImmutableList.of(portTag), value); + activeConnections.forEach((path, value) -> { + final Tag portTag = Tag.of("port", path); + meterRegistry.gauge(ALL_CONNECTIONS_METER_NAME, ImmutableList.of(portTag), value); }); } diff --git a/core/src/main/java/com/linecorp/armeria/server/Server.java b/core/src/main/java/com/linecorp/armeria/server/Server.java index 03a01f39cb5..50e9a4042ab 100644 --- a/core/src/main/java/com/linecorp/armeria/server/Server.java +++ b/core/src/main/java/com/linecorp/armeria/server/Server.java @@ -530,7 +530,7 @@ protected CompletionStage doStart(@Nullable Void arg) { doStart(primary).addListener(new ServerPortStartListener(primary)) .addListener(new NextServerPortStartListener(this, it, future)); // Chain the future to set up server metrics first before server start future is completed. - return future.thenAccept(unused -> bindServerMetricsToMeterRegistry()); + return future.thenAccept(unused -> setupServerMetrics()); } catch (Throwable cause) { future.completeExceptionally(cause); return future; @@ -589,7 +589,7 @@ private ChannelFuture doStart(ServerPort port) { return b.bind(localAddress); } - private void bindServerMetricsToMeterRegistry() { + private void setupServerMetrics() { final MeterRegistry meterRegistry = config.meterRegistry(); final GracefulShutdownSupport gracefulShutdownSupport = this.gracefulShutdownSupport; assert gracefulShutdownSupport != null; From a45483465a62836f1c3aa2a4d13cf7ed364a7f89 Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 24 Feb 2025 19:01:57 +0900 Subject: [PATCH 6/8] Address the comment from @jrhee17 --- .../server/ConnectionLimitingHandler.java | 112 +++---- .../armeria/server/DefaultServerConfig.java | 3 +- .../armeria/server/DefaultServerMetrics.java | 204 ------------- .../server/DomainSocketServerMetrics.java | 240 --------------- .../armeria/server/Http1RequestDecoder.java | 15 +- .../armeria/server/Http2RequestDecoder.java | 14 +- .../armeria/server/HttpServerHandler.java | 23 +- .../server/InetSocketServerMetrics.java | 278 ------------------ .../com/linecorp/armeria/server/Server.java | 25 +- .../armeria/server/ServerMetrics.java | 120 +++----- .../armeria/server/ServerPortMetric.java | 163 ++++++++++ .../server/ConnectionLimitingHandlerTest.java | 43 +-- .../armeria/server/ServerMetricsTest.java | 46 --- 13 files changed, 317 insertions(+), 969 deletions(-) delete mode 100644 core/src/main/java/com/linecorp/armeria/server/DefaultServerMetrics.java delete mode 100644 core/src/main/java/com/linecorp/armeria/server/DomainSocketServerMetrics.java delete mode 100644 core/src/main/java/com/linecorp/armeria/server/InetSocketServerMetrics.java create mode 100644 core/src/main/java/com/linecorp/armeria/server/ServerPortMetric.java diff --git a/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java b/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java index 858efb0dfd1..c26da96e1e2 100644 --- a/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java +++ b/core/src/main/java/com/linecorp/armeria/server/ConnectionLimitingHandler.java @@ -16,10 +16,6 @@ package com.linecorp.armeria.server; -import static com.google.common.base.MoreObjects.firstNonNull; -import static com.linecorp.armeria.server.HttpServerHandler.UNKNOWN_ADDR; - -import java.net.InetSocketAddress; import java.util.Collections; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -31,8 +27,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.linecorp.armeria.internal.common.util.ChannelUtil; - import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.Channel; import io.netty.channel.ChannelHandler; @@ -45,15 +39,13 @@ * Limit the number of open connections to the configured value. * {@link ConnectionLimitingHandler} instance would be set to {@link ServerBootstrap#handler(ChannelHandler)}. */ -@Sharable -final class ConnectionLimitingHandler extends ChannelInboundHandlerAdapter { +final class ConnectionLimitingHandler { private static final Logger logger = LoggerFactory.getLogger(ConnectionLimitingHandler.class); private final Set childChannels = Collections.newSetFromMap(new ConcurrentHashMap<>()); private final Set unmodifiableChildChannels = Collections.unmodifiableSet(childChannels); private final int maxNumConnections; - private final ServerMetrics serverMetrics; /** * AtomicInteger is used to read the number of active connections frequently. @@ -62,78 +54,90 @@ final class ConnectionLimitingHandler extends ChannelInboundHandlerAdapter { private final AtomicBoolean loggingScheduled = new AtomicBoolean(); private final LongAdder numDroppedConnections = new LongAdder(); - ConnectionLimitingHandler(int maxNumConnections, ServerMetrics serverMetrics) { + ConnectionLimitingHandler(int maxNumConnections) { this.maxNumConnections = validateMaxNumConnections(maxNumConnections); - this.serverMetrics = serverMetrics; - } - - @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - final Channel child = (Channel) msg; - final int conn = activeConnections.incrementAndGet(); - if (conn > 0 && conn <= maxNumConnections) { - final InetSocketAddress localAddress = firstNonNull(ChannelUtil.localAddress(child), UNKNOWN_ADDR); - serverMetrics.increaseActiveConnections(localAddress); - childChannels.add(child); - child.closeFuture().addListener(future -> { - childChannels.remove(child); - activeConnections.decrementAndGet(); - serverMetrics.decreaseActiveConnections(localAddress); - }); - super.channelRead(ctx, msg); - } else { - activeConnections.decrementAndGet(); - - // Set linger option to 0 so that the server doesn't get too many TIME_WAIT states. - child.config().setOption(ChannelOption.SO_LINGER, 0); - child.unsafe().closeForcibly(); - - numDroppedConnections.increment(); - - if (loggingScheduled.compareAndSet(false, true)) { - ctx.executor().schedule(this::writeNumDroppedConnectionsLog, 1, TimeUnit.SECONDS); - } - } - } - - private void writeNumDroppedConnectionsLog() { - loggingScheduled.set(false); - - final long dropped = numDroppedConnections.sumThenReset(); - if (dropped > 0) { - logger.warn("Dropped {} connection(s) to limit the number of open connections to {}", - dropped, maxNumConnections); - } } /** * Returns the maximum allowed number of open connections. */ - public int maxNumConnections() { + int maxNumConnections() { return maxNumConnections; } /** * Returns the number of open connections. */ - public int numConnections() { + int numConnections() { return activeConnections.get(); } /** * Returns the immutable set of child {@link Channel}s. */ - public Set children() { + Set children() { return unmodifiableChildChannels; } /** * Validates the maximum allowed number of open connections. It must be a positive number. */ - public static int validateMaxNumConnections(int maxNumConnections) { + static int validateMaxNumConnections(int maxNumConnections) { if (maxNumConnections <= 0) { throw new IllegalArgumentException("maxNumConnections: " + maxNumConnections + " (expected: > 0)"); } return maxNumConnections; } + + ChannelHandler newChildHandler(ServerPortMetric serverPortMetric) { + return new ConnectionLimitingChildHandler(serverPortMetric); + } + + @Sharable + private class ConnectionLimitingChildHandler extends ChannelInboundHandlerAdapter { + + private final ServerPortMetric serverPortMetric; + + ConnectionLimitingChildHandler(ServerPortMetric serverPortMetric) { + this.serverPortMetric = serverPortMetric; + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + final Channel child = (Channel) msg; + final int conn = activeConnections.incrementAndGet(); + if (conn > 0 && conn <= maxNumConnections) { + serverPortMetric.increaseActiveConnections(); + childChannels.add(child); + child.closeFuture().addListener(future -> { + childChannels.remove(child); + activeConnections.decrementAndGet(); + serverPortMetric.decreaseActiveConnections(); + }); + super.channelRead(ctx, msg); + } else { + activeConnections.decrementAndGet(); + + // Set linger option to 0 so that the server doesn't get too many TIME_WAIT states. + child.config().setOption(ChannelOption.SO_LINGER, 0); + child.unsafe().closeForcibly(); + + numDroppedConnections.increment(); + + if (loggingScheduled.compareAndSet(false, true)) { + ctx.executor().schedule(this::writeNumDroppedConnectionsLog, 1, TimeUnit.SECONDS); + } + } + } + + private void writeNumDroppedConnectionsLog() { + loggingScheduled.set(false); + + final long dropped = numDroppedConnections.sumThenReset(); + if (dropped > 0) { + logger.warn("Dropped {} connection(s) to limit the number of open connections to {}", + dropped, maxNumConnections); + } + } + } } diff --git a/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java b/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java index d87e4744618..3d80aa66a9e 100644 --- a/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java +++ b/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java @@ -118,7 +118,7 @@ final class DefaultServerConfig implements ServerConfig { @Nullable private final Mapping sslContexts; - private final ServerMetrics serverMetrics; + private final ServerMetrics serverMetrics = new ServerMetrics(); @Nullable private String strVal; @@ -266,7 +266,6 @@ final class DefaultServerConfig implements ServerConfig { this.absoluteUriTransformer = castAbsoluteUriTransformer; this.unloggedExceptionsReportIntervalMillis = unloggedExceptionsReportIntervalMillis; this.shutdownSupports = ImmutableList.copyOf(requireNonNull(shutdownSupports, "shutdownSupports")); - serverMetrics = new DefaultServerMetrics(ports); } private static Int2ObjectMap> buildDomainAndPortMapping( diff --git a/core/src/main/java/com/linecorp/armeria/server/DefaultServerMetrics.java b/core/src/main/java/com/linecorp/armeria/server/DefaultServerMetrics.java deleted file mode 100644 index d6ce9e363bf..00000000000 --- a/core/src/main/java/com/linecorp/armeria/server/DefaultServerMetrics.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright 2024 LINE Corporation - * - * LINE Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.linecorp.armeria.server; - -import java.net.InetSocketAddress; - -import com.google.common.base.MoreObjects; - -import com.linecorp.armeria.common.util.DomainSocketAddress; - -import io.micrometer.core.instrument.MeterRegistry; - -final class DefaultServerMetrics implements ServerMetrics { - - static final String ALL_REQUESTS_METER_NAME = "armeria.server.all.requests"; - static final String ALL_CONNECTIONS_METER_NAME = "armeria.server.connections"; - - private final InetSocketServerMetrics inetSocketServerMetrics; - private final DomainSocketServerMetrics domainSocketServerMetrics; - - DefaultServerMetrics(Iterable ports) { - inetSocketServerMetrics = new InetSocketServerMetrics(ports); - domainSocketServerMetrics = new DomainSocketServerMetrics(ports); - } - - @Override - public void addActivePort(ServerPort actualPort) { - if (!(actualPort.localAddress() instanceof DomainSocketAddress)) { - inetSocketServerMetrics.addActivePort(actualPort); - } - } - - @Override - public long pendingHttp1Requests() { - return inetSocketServerMetrics.pendingHttp1Requests() + - domainSocketServerMetrics.pendingHttp1Requests(); - } - - @Override - public long pendingHttp2Requests() { - return inetSocketServerMetrics.pendingHttp2Requests() + - domainSocketServerMetrics.pendingHttp2Requests(); - } - - @Override - public long activeHttp1WebSocketRequests() { - return inetSocketServerMetrics.activeHttp1WebSocketRequests() + - domainSocketServerMetrics.activeHttp1WebSocketRequests(); - } - - @Override - public long activeHttp1Requests() { - return inetSocketServerMetrics.activeHttp1Requests() + - domainSocketServerMetrics.activeHttp1Requests(); - } - - @Override - public long activeHttp2Requests() { - return inetSocketServerMetrics.activeHttp2Requests() + - domainSocketServerMetrics.activeHttp2Requests(); - } - - @Override - public long activeConnections() { - return inetSocketServerMetrics.activeConnections() + - domainSocketServerMetrics.activeConnections(); - } - - @Override - public void increasePendingHttp1Requests(InetSocketAddress socketAddress) { - if (socketAddress instanceof DomainSocketAddress) { - domainSocketServerMetrics.increasePendingHttp1Requests(socketAddress); - } else { - inetSocketServerMetrics.increasePendingHttp1Requests(socketAddress); - } - } - - @Override - public void decreasePendingHttp1Requests(InetSocketAddress socketAddress) { - if (socketAddress instanceof DomainSocketAddress) { - domainSocketServerMetrics.decreasePendingHttp1Requests(socketAddress); - } else { - inetSocketServerMetrics.decreasePendingHttp1Requests(socketAddress); - } - } - - @Override - public void increasePendingHttp2Requests(InetSocketAddress socketAddress) { - if (socketAddress instanceof DomainSocketAddress) { - domainSocketServerMetrics.increasePendingHttp2Requests(socketAddress); - } else { - inetSocketServerMetrics.increasePendingHttp2Requests(socketAddress); - } - } - - @Override - public void decreasePendingHttp2Requests(InetSocketAddress socketAddress) { - if (socketAddress instanceof DomainSocketAddress) { - domainSocketServerMetrics.decreasePendingHttp2Requests(socketAddress); - } else { - inetSocketServerMetrics.decreasePendingHttp2Requests(socketAddress); - } - } - - @Override - public void increaseActiveHttp1Requests(InetSocketAddress socketAddress) { - if (socketAddress instanceof DomainSocketAddress) { - domainSocketServerMetrics.increaseActiveHttp1Requests(socketAddress); - } else { - inetSocketServerMetrics.increaseActiveHttp1Requests(socketAddress); - } - } - - @Override - public void decreaseActiveHttp1Requests(InetSocketAddress socketAddress) { - if (socketAddress instanceof DomainSocketAddress) { - domainSocketServerMetrics.decreaseActiveHttp1Requests(socketAddress); - } else { - inetSocketServerMetrics.decreaseActiveHttp1Requests(socketAddress); - } - } - - @Override - public void increaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress) { - if (socketAddress instanceof DomainSocketAddress) { - domainSocketServerMetrics.increaseActiveHttp1WebSocketRequests(socketAddress); - } else { - inetSocketServerMetrics.increaseActiveHttp1WebSocketRequests(socketAddress); - } - } - - @Override - public void decreaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress) { - if (socketAddress instanceof DomainSocketAddress) { - domainSocketServerMetrics.decreaseActiveHttp1WebSocketRequests(socketAddress); - } else { - inetSocketServerMetrics.decreaseActiveHttp1WebSocketRequests(socketAddress); - } - } - - @Override - public void increaseActiveHttp2Requests(InetSocketAddress socketAddress) { - if (socketAddress instanceof DomainSocketAddress) { - domainSocketServerMetrics.increaseActiveHttp2Requests(socketAddress); - } else { - inetSocketServerMetrics.increaseActiveHttp2Requests(socketAddress); - } - } - - @Override - public void decreaseActiveHttp2Requests(InetSocketAddress socketAddress) { - if (socketAddress instanceof DomainSocketAddress) { - domainSocketServerMetrics.decreaseActiveHttp2Requests(socketAddress); - } else { - inetSocketServerMetrics.decreaseActiveHttp2Requests(socketAddress); - } - } - - @Override - public void increaseActiveConnections(InetSocketAddress socketAddress) { - if (socketAddress instanceof DomainSocketAddress) { - domainSocketServerMetrics.increaseActiveConnections(socketAddress); - } else { - inetSocketServerMetrics.increaseActiveConnections(socketAddress); - } - } - - @Override - public void decreaseActiveConnections(InetSocketAddress socketAddress) { - if (socketAddress instanceof DomainSocketAddress) { - domainSocketServerMetrics.decreaseActiveConnections(socketAddress); - } else { - inetSocketServerMetrics.decreaseActiveConnections(socketAddress); - } - } - - @Override - public void bindTo(MeterRegistry meterRegistry) { - inetSocketServerMetrics.bindTo(meterRegistry); - domainSocketServerMetrics.bindTo(meterRegistry); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("inetSocketServerMetrics", inetSocketServerMetrics) - .add("domainSocketServerMetrics", domainSocketServerMetrics) - .toString(); - } -} diff --git a/core/src/main/java/com/linecorp/armeria/server/DomainSocketServerMetrics.java b/core/src/main/java/com/linecorp/armeria/server/DomainSocketServerMetrics.java deleted file mode 100644 index 90868949e09..00000000000 --- a/core/src/main/java/com/linecorp/armeria/server/DomainSocketServerMetrics.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright 2025 LINE Corporation - * - * LINE Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package com.linecorp.armeria.server; - -import static com.linecorp.armeria.server.DefaultServerMetrics.ALL_CONNECTIONS_METER_NAME; -import static com.linecorp.armeria.server.DefaultServerMetrics.ALL_REQUESTS_METER_NAME; - -import java.net.InetSocketAddress; -import java.util.Map; -import java.util.concurrent.atomic.LongAdder; - -import com.google.common.base.MoreObjects; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; - -import com.linecorp.armeria.common.util.DomainSocketAddress; - -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Tag; - -final class DomainSocketServerMetrics implements ServerMetrics { - - private final Map pendingHttp1Requests; - private final Map pendingHttp2Requests; - private final Map activeHttp1WebSocketRequests; - private final Map activeHttp1Requests; - private final Map activeHttp2Requests; - private final Map activeConnections; - - DomainSocketServerMetrics(Iterable serverPorts) { - final ImmutableMap.Builder pendingHttp1RequestsBuilder = ImmutableMap.builder(); - final ImmutableMap.Builder pendingHttp2RequestsBuilder = ImmutableMap.builder(); - final ImmutableMap.Builder activeHttp1WebSocketRequestsBuilder = - ImmutableMap.builder(); - final ImmutableMap.Builder activeHttp1RequestsBuilder = ImmutableMap.builder(); - final ImmutableMap.Builder activeHttp2RequestsBuilder = ImmutableMap.builder(); - final ImmutableMap.Builder activeConnectionsBuilder = ImmutableMap.builder(); - serverPorts.forEach(port -> { - final InetSocketAddress localAddress = port.localAddress(); - if (!(localAddress instanceof DomainSocketAddress)) { - return; - } - final String path = ((DomainSocketAddress) localAddress).path(); - pendingHttp1RequestsBuilder.put(path, new LongAdder()); - pendingHttp2RequestsBuilder.put(path, new LongAdder()); - activeHttp1WebSocketRequestsBuilder.put(path, new LongAdder()); - activeHttp1RequestsBuilder.put(path, new LongAdder()); - activeHttp2RequestsBuilder.put(path, new LongAdder()); - activeConnectionsBuilder.put(path, new LongAdder()); - }); - pendingHttp1Requests = pendingHttp1RequestsBuilder.build(); - pendingHttp2Requests = pendingHttp2RequestsBuilder.build(); - activeHttp1WebSocketRequests = activeHttp1WebSocketRequestsBuilder.build(); - activeHttp1Requests = activeHttp1RequestsBuilder.build(); - activeHttp2Requests = activeHttp2RequestsBuilder.build(); - activeConnections = activeConnectionsBuilder.build(); - } - - @Override - public void addActivePort(ServerPort actualPort) { - // Do nothing because the port is already added in the constructor. - } - - @Override - public long pendingHttp1Requests() { - return sum(pendingHttp1Requests); - } - - private static long sum(Map map) { - return map.values().stream().mapToLong(LongAdder::longValue).sum(); - } - - @Override - public long pendingHttp2Requests() { - return sum(pendingHttp2Requests); - } - - @Override - public long activeHttp1WebSocketRequests() { - return sum(activeHttp1WebSocketRequests); - } - - @Override - public long activeHttp1Requests() { - return sum(activeHttp1Requests); - } - - @Override - public long activeHttp2Requests() { - return sum(activeHttp2Requests); - } - - @Override - public long activeConnections() { - return sum(activeConnections); - } - - @Override - public void increasePendingHttp1Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, pendingHttp1Requests, true); - } - - private static void increaseOrDecreaseRequests(InetSocketAddress socketAddress, - Map requests, - boolean increase) { - assert socketAddress instanceof DomainSocketAddress; - final String path = ((DomainSocketAddress) socketAddress).path(); - final LongAdder longAdder = requests.get(path); - assert longAdder != null; - if (increase) { - longAdder.increment(); - } else { - longAdder.decrement(); - } - } - - @Override - public void decreasePendingHttp1Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, pendingHttp1Requests, false); - } - - @Override - public void increasePendingHttp2Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, pendingHttp2Requests, true); - } - - @Override - public void decreasePendingHttp2Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, pendingHttp2Requests, false); - } - - @Override - public void increaseActiveHttp1Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp1Requests, true); - } - - @Override - public void decreaseActiveHttp1Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp1Requests, false); - } - - @Override - public void increaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp1WebSocketRequests, true); - } - - @Override - public void decreaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp1WebSocketRequests, false); - } - - @Override - public void increaseActiveHttp2Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp2Requests, true); - } - - @Override - public void decreaseActiveHttp2Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp2Requests, false); - } - - @Override - public void increaseActiveConnections(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeConnections, true); - } - - @Override - public void decreaseActiveConnections(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeConnections, false); - } - - @Override - public void bindTo(MeterRegistry meterRegistry) { - pendingHttp1Requests.forEach((path, value) -> { - // The path is used as the 'port' tag. - final Tag portTag = Tag.of("port", path); - meterRegistry.gauge(ALL_REQUESTS_METER_NAME, - ImmutableList.of(portTag, Tag.of("protocol", "http1"), - Tag.of("state", "pending")), - value); - }); - pendingHttp2Requests.forEach((path, value) -> { - final Tag portTag = Tag.of("port", path); - meterRegistry.gauge(ALL_REQUESTS_METER_NAME, - ImmutableList.of(portTag, Tag.of("protocol", "http2"), - Tag.of("state", "pending")), - value); - }); - activeHttp1WebSocketRequests.forEach((path, value) -> { - final Tag portTag = Tag.of("port", path); - meterRegistry.gauge(ALL_REQUESTS_METER_NAME, - ImmutableList.of(portTag, Tag.of("protocol", "http1.websocket"), - Tag.of("state", "active")), - value); - }); - activeHttp1Requests.forEach((path, value) -> { - final Tag portTag = Tag.of("port", path); - meterRegistry.gauge(ALL_REQUESTS_METER_NAME, - ImmutableList.of(portTag, Tag.of("protocol", "http1"), - Tag.of("state", "active")), - value); - }); - activeHttp2Requests.forEach((path, value) -> { - final Tag portTag = Tag.of("port", path); - meterRegistry.gauge(ALL_REQUESTS_METER_NAME, - ImmutableList.of(portTag, Tag.of("protocol", "http2"), - Tag.of("state", "active")), - value); - }); - activeConnections.forEach((path, value) -> { - final Tag portTag = Tag.of("port", path); - meterRegistry.gauge(ALL_CONNECTIONS_METER_NAME, ImmutableList.of(portTag), value); - }); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("pendingHttp1Requests", pendingHttp1Requests) - .add("activeHttp1WebSocketRequests", activeHttp1WebSocketRequests) - .add("activeHttp1Requests", activeHttp1Requests) - .add("pendingHttp2Requests", pendingHttp2Requests) - .add("activeHttp2Requests", activeHttp2Requests) - .add("activeConnections", activeConnections) - .toString(); - } -} diff --git a/core/src/main/java/com/linecorp/armeria/server/Http1RequestDecoder.java b/core/src/main/java/com/linecorp/armeria/server/Http1RequestDecoder.java index 9b8b5d06a99..4311e980e19 100644 --- a/core/src/main/java/com/linecorp/armeria/server/Http1RequestDecoder.java +++ b/core/src/main/java/com/linecorp/armeria/server/Http1RequestDecoder.java @@ -16,13 +16,11 @@ package com.linecorp.armeria.server; -import static com.google.common.base.MoreObjects.firstNonNull; import static com.linecorp.armeria.internal.common.websocket.WebSocketUtil.isHttp1WebSocketUpgradeRequest; -import static com.linecorp.armeria.server.HttpServerHandler.UNKNOWN_ADDR; import static com.linecorp.armeria.server.HttpServerPipelineConfigurator.SCHEME_HTTP; +import static com.linecorp.armeria.server.ServerPortMetric.SERVER_PORT_METRIC; import static com.linecorp.armeria.server.ServiceRouteUtil.newRoutingContext; -import java.net.InetSocketAddress; import java.net.URISyntaxException; import org.slf4j.Logger; @@ -49,7 +47,6 @@ import com.linecorp.armeria.internal.common.InitiateConnectionShutdown; import com.linecorp.armeria.internal.common.KeepAliveHandler; import com.linecorp.armeria.internal.common.NoopKeepAliveHandler; -import com.linecorp.armeria.internal.common.util.ChannelUtil; import com.linecorp.armeria.server.HttpServerUpgradeHandler.UpgradeEvent; import com.linecorp.armeria.server.websocket.WebSocketService; @@ -85,10 +82,10 @@ final class Http1RequestDecoder extends ChannelDuplexHandler { private static final ResponseHeaders CONTINUE_RESPONSE = ResponseHeaders.of(HttpStatus.CONTINUE); private final ServerConfig cfg; + private final ServerPortMetric serverPortMetric; private final AsciiString scheme; private SessionProtocol sessionProtocol; private final InboundTrafficController inboundTrafficController; - private final InetSocketAddress localAddress; private ServerHttpObjectEncoder encoder; private final HttpServer httpServer; @@ -101,7 +98,9 @@ final class Http1RequestDecoder extends ChannelDuplexHandler { Http1RequestDecoder(ServerConfig cfg, Channel channel, AsciiString scheme, ServerHttp1ObjectEncoder encoder, HttpServer httpServer) { this.cfg = cfg; - localAddress = firstNonNull(ChannelUtil.localAddress(channel), UNKNOWN_ADDR); + final ServerPortMetric serverPortMetric = channel.attr(SERVER_PORT_METRIC).get(); + assert serverPortMetric != null; + this.serverPortMetric = serverPortMetric; this.scheme = scheme; sessionProtocol = scheme == SCHEME_HTTP ? SessionProtocol.H1C : SessionProtocol.H1; inboundTrafficController = InboundTrafficController.ofHttp1(channel); @@ -277,7 +276,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception if (pipeline.get(HttpServerUpgradeHandler.class) != null) { pipeline.remove(HttpServerUpgradeHandler.class); } - cfg.serverMetrics().increasePendingHttp1Requests(localAddress); + serverPortMetric.increasePendingHttp1Requests(); ctx.fireChannelRead(webSocketRequest); return; } @@ -290,7 +289,7 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception if (maxRequestLength > 0 && contentLength > maxRequestLength) { abortLargeRequest(ctx, req, id, endOfStream, keepAliveHandler, true); } - cfg.serverMetrics().increasePendingHttp1Requests(localAddress); + serverPortMetric.increasePendingHttp1Requests(); ctx.fireChannelRead(req); } else { fail(id, null, HttpStatus.BAD_REQUEST, "Invalid decoder state", null); diff --git a/core/src/main/java/com/linecorp/armeria/server/Http2RequestDecoder.java b/core/src/main/java/com/linecorp/armeria/server/Http2RequestDecoder.java index cb069f7e416..bdfe7405be8 100644 --- a/core/src/main/java/com/linecorp/armeria/server/Http2RequestDecoder.java +++ b/core/src/main/java/com/linecorp/armeria/server/Http2RequestDecoder.java @@ -16,16 +16,13 @@ package com.linecorp.armeria.server; -import static com.google.common.base.MoreObjects.firstNonNull; -import static com.linecorp.armeria.server.HttpServerHandler.UNKNOWN_ADDR; import static com.linecorp.armeria.server.HttpServerPipelineConfigurator.SCHEME_HTTP; +import static com.linecorp.armeria.server.ServerPortMetric.SERVER_PORT_METRIC; import static com.linecorp.armeria.server.ServiceRouteUtil.newRoutingContext; import static io.netty.handler.codec.http2.Http2Error.INTERNAL_ERROR; import static io.netty.handler.codec.http2.Http2Error.PROTOCOL_ERROR; import static io.netty.handler.codec.http2.Http2Exception.connectionError; -import java.net.InetSocketAddress; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +42,6 @@ import com.linecorp.armeria.internal.common.Http2GoAwayHandler; import com.linecorp.armeria.internal.common.InboundTrafficController; import com.linecorp.armeria.internal.common.KeepAliveHandler; -import com.linecorp.armeria.internal.common.util.ChannelUtil; import io.netty.buffer.ByteBuf; import io.netty.channel.Channel; @@ -70,7 +66,7 @@ final class Http2RequestDecoder extends Http2EventAdapter { private final ServerConfig cfg; private final Channel channel; - private final InetSocketAddress localAddress; + private final ServerPortMetric serverPortMetric; private final AsciiString scheme; @Nullable private ServerHttp2ObjectEncoder encoder; @@ -85,7 +81,9 @@ final class Http2RequestDecoder extends Http2EventAdapter { AsciiString scheme, KeepAliveHandler keepAliveHandler) { this.cfg = cfg; this.channel = channel; - localAddress = firstNonNull(ChannelUtil.localAddress(channel), UNKNOWN_ADDR); + final ServerPortMetric serverPortMetric = channel.attr(SERVER_PORT_METRIC).get(); + assert serverPortMetric != null; + this.serverPortMetric = serverPortMetric; this.scheme = scheme; inboundTrafficController = InboundTrafficController.ofHttp2(channel, cfg.http2InitialConnectionWindowSize()); @@ -218,7 +216,7 @@ public void onHeadersRead(ChannelHandlerContext ctx, int streamId, Http2Headers abortLargeRequest(req, endOfStream, true); } requests.put(streamId, req); - cfg.serverMetrics().increasePendingHttp2Requests(localAddress); + serverPortMetric.increasePendingHttp2Requests(); ctx.fireChannelRead(req); } else { if (!(req instanceof DecodedHttpRequestWriter)) { diff --git a/core/src/main/java/com/linecorp/armeria/server/HttpServerHandler.java b/core/src/main/java/com/linecorp/armeria/server/HttpServerHandler.java index 6e7fcb184dc..1e53d3784b4 100644 --- a/core/src/main/java/com/linecorp/armeria/server/HttpServerHandler.java +++ b/core/src/main/java/com/linecorp/armeria/server/HttpServerHandler.java @@ -25,6 +25,7 @@ import static com.linecorp.armeria.internal.common.HttpHeadersUtil.CLOSE_STRING; import static com.linecorp.armeria.internal.common.RequestContextUtil.NOOP_CONTEXT_HOOK; import static com.linecorp.armeria.server.AccessLogWriterUtil.maybeWriteAccessLog; +import static com.linecorp.armeria.server.ServerPortMetric.SERVER_PORT_METRIC; import static io.netty.handler.codec.http2.Http2CodecUtil.DEFAULT_WINDOW_SIZE; import static java.util.Objects.requireNonNull; @@ -97,7 +98,7 @@ final class HttpServerHandler extends ChannelInboundHandlerAdapter implements Ht private static final String ALLOWED_METHODS_STRING = HttpMethod.knownMethods().stream().map(HttpMethod::name).collect(Collectors.joining(",")); - static final InetSocketAddress UNKNOWN_ADDR; + private static final InetSocketAddress UNKNOWN_ADDR; static { InetAddress unknownAddr; @@ -182,6 +183,7 @@ static void safeClose(Channel ch) { } private final ServerConfig config; + private final ServerPortMetric serverPortMetric; private final GracefulShutdownSupport gracefulShutdownSupport; private SessionProtocol protocol; @@ -211,6 +213,9 @@ static void safeClose(Channel ch) { assert protocol == H1 || protocol == H1C || protocol == H2; this.config = requireNonNull(config, "config"); + final ServerPortMetric serverPortMetric = channel.attr(SERVER_PORT_METRIC).get(); + assert serverPortMetric != null; + this.serverPortMetric = serverPortMetric; remoteAddress = firstNonNull(ChannelUtil.remoteAddress(channel), UNKNOWN_ADDR); localAddress = firstNonNull(ChannelUtil.localAddress(channel), UNKNOWN_ADDR); this.gracefulShutdownSupport = requireNonNull(gracefulShutdownSupport, "gracefulShutdownSupport"); @@ -519,21 +524,21 @@ private void handleRequest(ChannelHandlerContext ctx, DecodedHttpRequest req) th private void decreasePendingRequests() { if (protocol.isExplicitHttp1()) { - config.serverMetrics().decreasePendingHttp1Requests(localAddress); + serverPortMetric.decreasePendingHttp1Requests(); } else { assert protocol.isExplicitHttp2(); - config.serverMetrics().decreasePendingHttp2Requests(localAddress); + serverPortMetric.decreasePendingHttp2Requests(); } } private void increaseActiveRequests(boolean isHttp1WebSocket) { if (isHttp1WebSocket) { - config.serverMetrics().increaseActiveHttp1WebSocketRequests(localAddress); + serverPortMetric.increaseActiveHttp1WebSocketRequests(); } else if (protocol.isExplicitHttp1()) { - config.serverMetrics().increaseActiveHttp1Requests(localAddress); + serverPortMetric.increaseActiveHttp1Requests(); } else { assert protocol.isExplicitHttp2(); - config.serverMetrics().increaseActiveHttp2Requests(localAddress); + serverPortMetric.increaseActiveHttp2Requests(); } } @@ -819,11 +824,11 @@ private void handleRequestOrResponseComplete() { return; } if (req.isHttp1WebSocket()) { - config.serverMetrics().decreaseActiveHttp1WebSocketRequests(localAddress); + serverPortMetric.decreaseActiveHttp1WebSocketRequests(); } else if (protocol.isExplicitHttp1()) { - config.serverMetrics().decreaseActiveHttp1Requests(localAddress); + serverPortMetric.decreaseActiveHttp1Requests(); } else if (protocol.isExplicitHttp2()) { - config.serverMetrics().decreaseActiveHttp2Requests(localAddress); + serverPortMetric.decreaseActiveHttp2Requests(); } // NB: logBuilder.endResponse() is called by HttpResponseSubscriber. diff --git a/core/src/main/java/com/linecorp/armeria/server/InetSocketServerMetrics.java b/core/src/main/java/com/linecorp/armeria/server/InetSocketServerMetrics.java deleted file mode 100644 index 78a865b48bf..00000000000 --- a/core/src/main/java/com/linecorp/armeria/server/InetSocketServerMetrics.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright 2025 LINE Corporation - * - * LINE Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ -package com.linecorp.armeria.server; - -import static com.linecorp.armeria.server.DefaultServerMetrics.ALL_CONNECTIONS_METER_NAME; -import static com.linecorp.armeria.server.DefaultServerMetrics.ALL_REQUESTS_METER_NAME; - -import java.net.InetSocketAddress; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.LongAdder; - -import com.google.common.base.MoreObjects; -import com.google.common.collect.ImmutableList; - -import com.linecorp.armeria.common.util.DomainSocketAddress; - -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Tag; -import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; - -final class InetSocketServerMetrics implements ServerMetrics { - - private final boolean hasEphemeralPort; - - private final Map pendingHttp1Requests; - private final Map pendingHttp2Requests; - private final Map activeHttp1WebSocketRequests; - private final Map activeHttp1Requests; - private final Map activeHttp2Requests; - private final Map activeConnections; - - InetSocketServerMetrics(Iterable serverPorts) { - boolean hasEphemeralPort = false; - final Map pendingHttp1Requests = new Int2ObjectOpenHashMap<>(); - final Map pendingHttp2Requests = new Int2ObjectOpenHashMap<>(); - final Map activeHttp1WebSocketRequests = new Int2ObjectOpenHashMap<>(); - final Map activeHttp1Requests = new Int2ObjectOpenHashMap<>(); - final Map activeHttp2Requests = new Int2ObjectOpenHashMap<>(); - final Map activeConnections = new Int2ObjectOpenHashMap<>(); - for (ServerPort serverPort : serverPorts) { - final InetSocketAddress localAddress = serverPort.localAddress(); - if (!(localAddress instanceof DomainSocketAddress)) { - final int port = localAddress.getPort(); - if (port == 0) { - hasEphemeralPort = true; - } else { - pendingHttp1Requests.put(port, new LongAdder()); - pendingHttp2Requests.put(port, new LongAdder()); - activeHttp1WebSocketRequests.put(port, new LongAdder()); - activeHttp1Requests.put(port, new LongAdder()); - activeHttp2Requests.put(port, new LongAdder()); - activeConnections.put(port, new LongAdder()); - } - } - } - // Put a dummy value to avoid NPE in case where an unknown address is returned from the channel. - // See the commit description of https://github.com/line/armeria/pull/5096 - pendingHttp1Requests.put(1, new LongAdder()); - pendingHttp2Requests.put(1, new LongAdder()); - activeHttp1WebSocketRequests.put(1, new LongAdder()); - activeHttp1Requests.put(1, new LongAdder()); - activeHttp2Requests.put(1, new LongAdder()); - activeConnections.put(1, new LongAdder()); - - this.hasEphemeralPort = hasEphemeralPort; - if (hasEphemeralPort) { - // This is mostly for testing if the server has an ephemeral port. - this.pendingHttp1Requests = new ConcurrentHashMap<>(pendingHttp1Requests); - this.pendingHttp2Requests = new ConcurrentHashMap<>(pendingHttp2Requests); - this.activeHttp1WebSocketRequests = new ConcurrentHashMap<>(activeHttp1WebSocketRequests); - this.activeHttp1Requests = new ConcurrentHashMap<>(activeHttp1Requests); - this.activeHttp2Requests = new ConcurrentHashMap<>(activeHttp2Requests); - this.activeConnections = new ConcurrentHashMap<>(activeConnections); - } else { - this.pendingHttp1Requests = pendingHttp1Requests; - this.pendingHttp2Requests = pendingHttp2Requests; - this.activeHttp1WebSocketRequests = activeHttp1WebSocketRequests; - this.activeHttp1Requests = activeHttp1Requests; - this.activeHttp2Requests = activeHttp2Requests; - this.activeConnections = activeConnections; - } - } - - @Override - public void addActivePort(ServerPort actualPort) { - if (!hasEphemeralPort) { - return; - } - - final InetSocketAddress address = actualPort.localAddress(); - final int port = address.getPort(); - if (pendingHttp1Requests.containsKey(port)) { - return; - } - - pendingHttp1Requests.put(port, new LongAdder()); - pendingHttp2Requests.put(port, new LongAdder()); - activeHttp1WebSocketRequests.put(port, new LongAdder()); - activeHttp1Requests.put(port, new LongAdder()); - activeHttp2Requests.put(port, new LongAdder()); - activeConnections.put(port, new LongAdder()); - } - - @Override - public long pendingHttp1Requests() { - return sum(pendingHttp1Requests); - } - - private static long sum(Map map) { - return map.values().stream().mapToLong(LongAdder::longValue).sum(); - } - - @Override - public long pendingHttp2Requests() { - return sum(pendingHttp2Requests); - } - - @Override - public long activeHttp1WebSocketRequests() { - return sum(activeHttp1WebSocketRequests); - } - - @Override - public long activeHttp1Requests() { - return sum(activeHttp1Requests); - } - - @Override - public long activeHttp2Requests() { - return sum(activeHttp2Requests); - } - - @Override - public long activeConnections() { - return sum(activeConnections); - } - - @Override - public void increasePendingHttp1Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, pendingHttp1Requests, true); - } - - private static void increaseOrDecreaseRequests(InetSocketAddress socketAddress, - Map requests, - boolean increase) { - final LongAdder longAdder = requests.get(socketAddress.getPort()); - assert longAdder != null; - if (increase) { - longAdder.increment(); - } else { - longAdder.decrement(); - } - } - - @Override - public void decreasePendingHttp1Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, pendingHttp1Requests, false); - } - - @Override - public void increasePendingHttp2Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, pendingHttp2Requests, true); - } - - @Override - public void decreasePendingHttp2Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, pendingHttp2Requests, false); - } - - @Override - public void increaseActiveHttp1Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp1Requests, true); - } - - @Override - public void decreaseActiveHttp1Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp1Requests, false); - } - - @Override - public void increaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp1WebSocketRequests, true); - } - - @Override - public void decreaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp1WebSocketRequests, false); - } - - @Override - public void increaseActiveHttp2Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp2Requests, true); - } - - @Override - public void decreaseActiveHttp2Requests(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeHttp2Requests, false); - } - - @Override - public void increaseActiveConnections(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeConnections, true); - } - - @Override - public void decreaseActiveConnections(InetSocketAddress socketAddress) { - increaseOrDecreaseRequests(socketAddress, activeConnections, false); - } - - @Override - public void bindTo(MeterRegistry meterRegistry) { - pendingHttp1Requests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", String.valueOf(port)); - meterRegistry.gauge(ALL_REQUESTS_METER_NAME, - ImmutableList.of(portTag, Tag.of("protocol", "http1"), - Tag.of("state", "pending")), - value); - }); - pendingHttp2Requests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", String.valueOf(port)); - meterRegistry.gauge(ALL_REQUESTS_METER_NAME, - ImmutableList.of(portTag, Tag.of("protocol", "http2"), - Tag.of("state", "pending")), - value); - }); - activeHttp1WebSocketRequests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", String.valueOf(port)); - meterRegistry.gauge(ALL_REQUESTS_METER_NAME, - ImmutableList.of(portTag, Tag.of("protocol", "http1.websocket"), - Tag.of("state", "active")), - value); - }); - activeHttp1Requests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", String.valueOf(port)); - meterRegistry.gauge(ALL_REQUESTS_METER_NAME, - ImmutableList.of(portTag, Tag.of("protocol", "http1"), - Tag.of("state", "active")), - value); - }); - activeHttp2Requests.forEach((port, value) -> { - final Tag portTag = Tag.of("port", String.valueOf(port)); - meterRegistry.gauge(ALL_REQUESTS_METER_NAME, - ImmutableList.of(portTag, Tag.of("protocol", "http2"), - Tag.of("state", "active")), - value); - }); - activeConnections.forEach((port, value) -> { - final Tag portTag = Tag.of("port", String.valueOf(port)); - meterRegistry.gauge(ALL_CONNECTIONS_METER_NAME, ImmutableList.of(portTag), value); - }); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("pendingHttp1Requests", pendingHttp1Requests) - .add("activeHttp1WebSocketRequests", activeHttp1WebSocketRequests) - .add("activeHttp1Requests", activeHttp1Requests) - .add("pendingHttp2Requests", pendingHttp2Requests) - .add("activeHttp2Requests", activeHttp2Requests) - .add("activeConnections", activeConnections) - .toString(); - } -} diff --git a/core/src/main/java/com/linecorp/armeria/server/Server.java b/core/src/main/java/com/linecorp/armeria/server/Server.java index 50e9a4042ab..666f54ce6d7 100644 --- a/core/src/main/java/com/linecorp/armeria/server/Server.java +++ b/core/src/main/java/com/linecorp/armeria/server/Server.java @@ -18,6 +18,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.linecorp.armeria.server.ServerPortMetric.SERVER_PORT_METRIC; import static com.linecorp.armeria.server.ServerSslContextUtil.validateSslContext; import static java.util.Objects.requireNonNull; @@ -133,8 +134,7 @@ public static ServerBuilder builder() { serverConfig.setServer(this); config = new UpdatableServerConfig(requireNonNull(serverConfig, "serverConfig")); startStop = new ServerStartStopSupport(config.startStopExecutor()); - connectionLimitingHandler = new ConnectionLimitingHandler(config.maxNumConnections(), - config.serverMetrics()); + connectionLimitingHandler = new ConnectionLimitingHandler(config.maxNumConnections()); // Server-wide metrics. RequestTargetCache.registerServerMetrics(config.meterRegistry()); @@ -530,7 +530,7 @@ protected CompletionStage doStart(@Nullable Void arg) { doStart(primary).addListener(new ServerPortStartListener(primary)) .addListener(new NextServerPortStartListener(this, it, future)); // Chain the future to set up server metrics first before server start future is completed. - return future.thenAccept(unused -> setupServerMetrics()); + return future.thenAccept(unused -> setupPendingResponsesMetrics()); } catch (Throwable cause) { future.completeExceptionally(cause); return future; @@ -562,10 +562,13 @@ private ChannelFuture doStart(ServerPort port) { final GracefulShutdownSupport gracefulShutdownSupport = this.gracefulShutdownSupport; assert gracefulShutdownSupport != null; + final ServerPortMetric serverPortMetric = new ServerPortMetric(); b.group(bossGroup, config.workerGroup()); - b.handler(connectionLimitingHandler); + b.handler(connectionLimitingHandler.newChildHandler(serverPortMetric)); b.childHandler(new HttpServerPipelineConfigurator(config, port, gracefulShutdownSupport, hasWebSocketService)); + b.attr(SERVER_PORT_METRIC, serverPortMetric); + b.childAttr(SERVER_PORT_METRIC, serverPortMetric); final SocketAddress localAddress; final Class channelType; @@ -589,14 +592,13 @@ private ChannelFuture doStart(ServerPort port) { return b.bind(localAddress); } - private void setupServerMetrics() { - final MeterRegistry meterRegistry = config.meterRegistry(); + private void setupPendingResponsesMetrics() { final GracefulShutdownSupport gracefulShutdownSupport = this.gracefulShutdownSupport; assert gracefulShutdownSupport != null; - meterRegistry.gauge("armeria.server.pending.responses", gracefulShutdownSupport, - GracefulShutdownSupport::pendingResponses); - config.serverMetrics().bindTo(meterRegistry); + // Move to ServerMetrics. + config.meterRegistry().gauge("armeria.server.pending.responses", gracefulShutdownSupport, + GracefulShutdownSupport::pendingResponses); } @Override @@ -848,7 +850,10 @@ public void operationComplete(ChannelFuture f) { lock.unlock(); } - config().serverMetrics().addActivePort(actualPort); + final ServerPortMetric serverPortMetric = ch.attr(SERVER_PORT_METRIC).get(); + assert serverPortMetric != null; + serverPortMetric.bindTo(config.meterRegistry(), actualPort); + config.serverMetrics().addServerPortMetric(serverPortMetric); if (logger.isInfoEnabled()) { if (isLocalPort(actualPort)) { diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java b/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java index 92b81c7bb94..6e59034b3d9 100644 --- a/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java +++ b/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java @@ -16,44 +16,56 @@ package com.linecorp.armeria.server; -import java.net.InetSocketAddress; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; -import com.linecorp.armeria.common.annotation.UnstableApi; +import com.google.common.base.MoreObjects; +import com.google.common.primitives.Ints; -import io.micrometer.core.instrument.binder.MeterBinder; +import com.linecorp.armeria.common.annotation.UnstableApi; /** - * An interface that provides the requests and connections metrics information of the server. + * A class that holds metrics related server. */ @UnstableApi -public interface ServerMetrics extends MeterBinder { +public final class ServerMetrics { - /** - * Adds the {@link ServerPort} to the {@link ServerMetrics}. - */ - void addActivePort(ServerPort actualPort); + static final String ALL_REQUESTS_METER_NAME = "armeria.server.all.requests"; + static final String ALL_CONNECTIONS_METER_NAME = "armeria.server.connections"; + + private final List serverPortMetrics = new CopyOnWriteArrayList<>(); + + ServerMetrics() {} + + void addServerPortMetric(ServerPortMetric serverPortMetric) { + serverPortMetrics.add(serverPortMetric); + } /** * Returns the number of all pending requests. */ - default long pendingRequests() { + public long pendingRequests() { return pendingHttp1Requests() + pendingHttp2Requests(); } /** * Returns the number of pending http1 requests. */ - long pendingHttp1Requests(); + public long pendingHttp1Requests() { + return serverPortMetrics.stream().mapToLong(ServerPortMetric::pendingHttp1Requests).sum(); + } /** * Returns the number of pending http2 requests. */ - long pendingHttp2Requests(); + public long pendingHttp2Requests() { + return serverPortMetrics.stream().mapToLong(ServerPortMetric::pendingHttp2Requests).sum(); + } /** * Returns the number of all active requests. */ - default long activeRequests() { + public long activeRequests() { return activeHttp1WebSocketRequests() + activeHttp1Requests() + activeHttp2Requests(); @@ -62,80 +74,36 @@ default long activeRequests() { /** * Returns the number of active http1 web socket requests. */ - long activeHttp1WebSocketRequests(); + public long activeHttp1WebSocketRequests() { + return serverPortMetrics.stream().mapToLong(ServerPortMetric::activeHttp1WebSocketRequests).sum(); + } /** * Returns the number of active http1 requests. */ - long activeHttp1Requests(); + public long activeHttp1Requests() { + return serverPortMetrics.stream().mapToLong(ServerPortMetric::activeHttp1Requests).sum(); + } /** * Returns the number of active http2 requests. */ - long activeHttp2Requests(); + public long activeHttp2Requests() { + return serverPortMetrics.stream().mapToLong(ServerPortMetric::activeHttp2Requests).sum(); + } /** * Returns the number of open connections. */ - long activeConnections(); - - /** - * Returns the number of all connections. - */ - void increasePendingHttp1Requests(InetSocketAddress socketAddress); - - /** - * Decreases the number of all pending http1 requests. - */ - void decreasePendingHttp1Requests(InetSocketAddress socketAddress); - - /** - * Increases the number of all pending http2 requests. - */ - void increasePendingHttp2Requests(InetSocketAddress socketAddress); - - /** - * Decreases the number of all pending http2 requests. - */ - void decreasePendingHttp2Requests(InetSocketAddress socketAddress); - - /** - * Increases the number of all active http1 requests. - */ - void increaseActiveHttp1Requests(InetSocketAddress socketAddress); - - /** - * Decreases the number of all active http1 requests. - */ - void decreaseActiveHttp1Requests(InetSocketAddress socketAddress); - - /** - * Increases the number of all active http1 web socket requests. - */ - void increaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress); - - /** - * Decreases the number of all active http1 web socket requests. - */ - void decreaseActiveHttp1WebSocketRequests(InetSocketAddress socketAddress); - - /** - * Increases the number of all active http2 requests. - */ - void increaseActiveHttp2Requests(InetSocketAddress socketAddress); - - /** - * Decreases the number of all active http2 requests. - */ - void decreaseActiveHttp2Requests(InetSocketAddress socketAddress); - - /** - * Increases the number of open connections. - */ - void increaseActiveConnections(InetSocketAddress socketAddress); + public int activeConnections() { + return Ints.saturatedCast(serverPortMetrics.stream().mapToLong(ServerPortMetric::activeConnections) + .sum()); + } - /** - * Decreases the number of open connections. - */ - void decreaseActiveConnections(InetSocketAddress socketAddress); + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("serverPortMetrics", serverPortMetrics) + .toString(); + } } diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerPortMetric.java b/core/src/main/java/com/linecorp/armeria/server/ServerPortMetric.java new file mode 100644 index 00000000000..3ed3e46da44 --- /dev/null +++ b/core/src/main/java/com/linecorp/armeria/server/ServerPortMetric.java @@ -0,0 +1,163 @@ +/* + * Copyright 2025 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ +package com.linecorp.armeria.server; + +import static com.linecorp.armeria.server.ServerMetrics.ALL_CONNECTIONS_METER_NAME; +import static com.linecorp.armeria.server.ServerMetrics.ALL_REQUESTS_METER_NAME; + +import java.net.InetSocketAddress; +import java.util.concurrent.atomic.LongAdder; + +import com.google.common.base.MoreObjects; +import com.google.common.collect.ImmutableList; + +import com.linecorp.armeria.common.util.DomainSocketAddress; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.netty.util.AttributeKey; + +final class ServerPortMetric { + + static final AttributeKey SERVER_PORT_METRIC = + AttributeKey.valueOf(ServerPortMetric.class, "SERVER_PORT_METRIC"); + + private final LongAdder pendingHttp1Requests = new LongAdder(); + private final LongAdder pendingHttp2Requests = new LongAdder(); + private final LongAdder activeHttp1WebSocketRequests = new LongAdder(); + private final LongAdder activeHttp1Requests = new LongAdder(); + private final LongAdder activeHttp2Requests = new LongAdder(); + private final LongAdder activeConnections = new LongAdder(); + + void increasePendingHttp1Requests() { + pendingHttp1Requests.increment(); + } + + void decreasePendingHttp1Requests() { + pendingHttp1Requests.decrement(); + } + + void increasePendingHttp2Requests() { + pendingHttp2Requests.increment(); + } + + void decreasePendingHttp2Requests() { + pendingHttp2Requests.decrement(); + } + + void increaseActiveHttp1WebSocketRequests() { + activeHttp1WebSocketRequests.increment(); + } + + void decreaseActiveHttp1WebSocketRequests() { + activeHttp1WebSocketRequests.decrement(); + } + + void increaseActiveHttp1Requests() { + activeHttp1Requests.increment(); + } + + void decreaseActiveHttp1Requests() { + activeHttp1Requests.decrement(); + } + + void increaseActiveHttp2Requests() { + activeHttp2Requests.increment(); + } + + void decreaseActiveHttp2Requests() { + activeHttp2Requests.decrement(); + } + + void increaseActiveConnections() { + activeConnections.increment(); + } + + void decreaseActiveConnections() { + activeConnections.decrement(); + } + + long pendingHttp1Requests() { + return pendingHttp1Requests.sum(); + } + + long pendingHttp2Requests() { + return pendingHttp2Requests.sum(); + } + + long activeHttp1WebSocketRequests() { + return activeHttp1WebSocketRequests.sum(); + } + + long activeHttp1Requests() { + return activeHttp1Requests.sum(); + } + + long activeHttp2Requests() { + return activeHttp2Requests.sum(); + } + + long activeConnections() { + return activeConnections.sum(); + } + + void bindTo(MeterRegistry meterRegistry, ServerPort actualPort) { + final InetSocketAddress address = actualPort.localAddress(); + final Tag portTag; + if (address instanceof DomainSocketAddress) { + final String path = ((DomainSocketAddress) address).path(); + // The path is used as the 'port' tag. + portTag = Tag.of("port", path); + } else { + portTag = Tag.of("port", String.valueOf(address.getPort())); + } + + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http1"), + Tag.of("state", "pending")), + this, ServerPortMetric::pendingHttp1Requests); + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http2"), + Tag.of("state", "pending")), + this, ServerPortMetric::pendingHttp2Requests); + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http1"), + Tag.of("state", "active")), + this, ServerPortMetric::activeHttp1Requests); + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http2"), + Tag.of("state", "active")), + this, ServerPortMetric::activeHttp2Requests); + meterRegistry.gauge(ALL_REQUESTS_METER_NAME, + ImmutableList.of(portTag, Tag.of("protocol", "http1"), + Tag.of("state", "active_websocket")), + this, ServerPortMetric::activeHttp1WebSocketRequests); + meterRegistry.gauge(ALL_CONNECTIONS_METER_NAME, ImmutableList.of(portTag), + this, ServerPortMetric::activeConnections); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("pendingHttp1Requests", pendingHttp1Requests) + .add("pendingHttp2Requests", pendingHttp2Requests) + .add("activeHttp1WebSocketRequests", activeHttp1WebSocketRequests) + .add("activeHttp1Requests", activeHttp1Requests) + .add("activeHttp2Requests", activeHttp2Requests) + .add("activeConnections", activeConnections) + .toString(); + } +} diff --git a/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java b/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java index 63c16f43adf..9a5e1b9c7e5 100644 --- a/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java +++ b/core/src/test/java/com/linecorp/armeria/server/ConnectionLimitingHandlerTest.java @@ -18,46 +18,26 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import java.net.InetSocketAddress; - import org.junit.jupiter.api.Test; -import com.google.common.collect.ImmutableList; - -import com.linecorp.armeria.common.SessionProtocol; - +import io.netty.channel.ChannelHandler; import io.netty.channel.embedded.EmbeddedChannel; class ConnectionLimitingHandlerTest { @Test void testExceedMaxNumConnections() { - // The port is not used in this test. - final InetSocketAddress localAddress1 = new InetSocketAddress(2); - final InetSocketAddress localAddress2 = new InetSocketAddress(3); - final ServerPort serverPort1 = new ServerPort(localAddress1, SessionProtocol.HTTP); - final ServerPort serverPort2 = new ServerPort(localAddress2, SessionProtocol.HTTP); - final ServerMetrics serverMetrics = - new DefaultServerMetrics(ImmutableList.of(serverPort1, serverPort2)); + final ServerPortMetric serverPortMetric = new ServerPortMetric(); + final ConnectionLimitingHandler handler = new ConnectionLimitingHandler(1); - final ConnectionLimitingHandler handler = new ConnectionLimitingHandler(1, serverMetrics); - final EmbeddedChannel ch1 = new EmbeddedChannel(handler) { - @Override - public InetSocketAddress localAddress() { - return localAddress1; - } - }; + final ChannelHandler channelHandler = handler.newChildHandler(serverPortMetric); + final EmbeddedChannel ch1 = new EmbeddedChannel(channelHandler); ch1.writeInbound(ch1); assertThat(handler.numConnections()).isEqualTo(1); assertThat(ch1.isActive()).isTrue(); - final EmbeddedChannel ch2 = new EmbeddedChannel(handler) { - @Override - public InetSocketAddress localAddress() { - return localAddress2; - } - }; + final EmbeddedChannel ch2 = new EmbeddedChannel(channelHandler); ch2.writeInbound(ch2); assertThat(handler.numConnections()).isEqualTo(1); assertThat(ch2.isActive()).isFalse(); @@ -68,18 +48,13 @@ public InetSocketAddress localAddress() { @Test void testMaxNumConnectionsRange() { - // The port is not used in this test. - final InetSocketAddress localAddress = new InetSocketAddress(2); - final ServerMetrics serverMetrics = new DefaultServerMetrics(ImmutableList.of( - new ServerPort(localAddress, SessionProtocol.HTTP))); - final ConnectionLimitingHandler handler = new ConnectionLimitingHandler(Integer.MAX_VALUE, - serverMetrics); + final ConnectionLimitingHandler handler = new ConnectionLimitingHandler(Integer.MAX_VALUE); assertThat(handler.maxNumConnections()).isEqualTo(Integer.MAX_VALUE); - assertThatThrownBy(() -> new ConnectionLimitingHandler(0, serverMetrics)) + assertThatThrownBy(() -> new ConnectionLimitingHandler(0)) .isInstanceOf(IllegalArgumentException.class); - assertThatThrownBy(() -> new ConnectionLimitingHandler(-1, serverMetrics)) + assertThatThrownBy(() -> new ConnectionLimitingHandler(-1)) .isInstanceOf(IllegalArgumentException.class); } } diff --git a/core/src/test/java/com/linecorp/armeria/server/ServerMetricsTest.java b/core/src/test/java/com/linecorp/armeria/server/ServerMetricsTest.java index 48d1d25e5e2..e3e36a625fe 100644 --- a/core/src/test/java/com/linecorp/armeria/server/ServerMetricsTest.java +++ b/core/src/test/java/com/linecorp/armeria/server/ServerMetricsTest.java @@ -18,19 +18,15 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; -import java.net.InetSocketAddress; import java.time.Duration; import java.util.Map; import java.util.concurrent.CompletableFuture; import org.assertj.core.api.Condition; -import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import com.google.common.collect.ImmutableList; - import com.linecorp.armeria.client.BlockingWebClient; import com.linecorp.armeria.client.ClientFactory; import com.linecorp.armeria.client.ClientOptions; @@ -140,48 +136,6 @@ public ExchangeType exchangeType(RoutingContext routingContext) { } }; - @Test - void pendingRequests() { - final ServerPort activePort = server.server().activePort(); - final ServerMetrics serverMetrics = new DefaultServerMetrics(ImmutableList.of(activePort)); - final InetSocketAddress localAddress = activePort.localAddress(); - serverMetrics.increasePendingHttp1Requests(localAddress); - assertThat(serverMetrics.pendingRequests()).isEqualTo(1); - - serverMetrics.increasePendingHttp2Requests(localAddress); - assertThat(serverMetrics.pendingRequests()).isEqualTo(2); - - serverMetrics.decreasePendingHttp1Requests(localAddress); - assertThat(serverMetrics.pendingRequests()).isEqualTo(1); - - serverMetrics.decreasePendingHttp2Requests(localAddress); - assertThat(serverMetrics.pendingRequests()).isZero(); - } - - @Test - void activeRequests() { - final ServerPort activePort = server.server().activePort(); - final ServerMetrics serverMetrics = new DefaultServerMetrics(ImmutableList.of(activePort)); - final InetSocketAddress localAddress = activePort.localAddress(); - serverMetrics.increaseActiveHttp1Requests(localAddress); - assertThat(serverMetrics.activeRequests()).isEqualTo(1); - - serverMetrics.increaseActiveHttp1WebSocketRequests(localAddress); - assertThat(serverMetrics.activeRequests()).isEqualTo(2); - - serverMetrics.increaseActiveHttp2Requests(localAddress); - assertThat(serverMetrics.activeRequests()).isEqualTo(3); - - serverMetrics.decreaseActiveHttp1WebSocketRequests(localAddress); - assertThat(serverMetrics.activeRequests()).isEqualTo(2); - - serverMetrics.decreaseActiveHttp1Requests(localAddress); - assertThat(serverMetrics.activeRequests()).isEqualTo(1); - - serverMetrics.decreaseActiveHttp2Requests(localAddress); - assertThat(serverMetrics.activeRequests()).isZero(); - } - @CsvSource({ "H1C, 1, 0", "H2C, 0, 1" }) @ParameterizedTest void checkWhenOk(SessionProtocol sessionProtocol, long expectedPendingHttp1Request, From 0aae5c184254d48f526119edf34e81fcc8841916 Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 24 Feb 2025 19:05:58 +0900 Subject: [PATCH 7/8] misc --- core/src/main/java/com/linecorp/armeria/server/Server.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/server/Server.java b/core/src/main/java/com/linecorp/armeria/server/Server.java index 666f54ce6d7..4dccd937874 100644 --- a/core/src/main/java/com/linecorp/armeria/server/Server.java +++ b/core/src/main/java/com/linecorp/armeria/server/Server.java @@ -841,11 +841,10 @@ public void operationComplete(ChannelFuture f) { // Update the boss thread so its name contains the actual port. Thread.currentThread().setName(bossThreadName(actualPort)); - final InetSocketAddress actualLocalAddress = actualPort.localAddress(); lock.lock(); try { // Update the map of active ports. - activePorts.put(actualLocalAddress, actualPort); + activePorts.put(actualPort.localAddress(), actualPort); } finally { lock.unlock(); } From e2ca3d5d29fb9a8b929dec88da6ba6ec48caf870 Mon Sep 17 00:00:00 2001 From: minwoox Date: Mon, 24 Feb 2025 21:37:53 +0900 Subject: [PATCH 8/8] Consider same port --- .../armeria/server/DefaultServerConfig.java | 3 +- .../com/linecorp/armeria/server/Server.java | 28 +++++++++++++++---- .../armeria/server/ServerMetrics.java | 21 ++++++++++---- .../linecorp/armeria/server/ServerPort.java | 12 ++++++++ .../armeria/server/ServerPortMetric.java | 2 ++ 5 files changed, 54 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java b/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java index 3d80aa66a9e..b3321732737 100644 --- a/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java +++ b/core/src/main/java/com/linecorp/armeria/server/DefaultServerConfig.java @@ -118,7 +118,7 @@ final class DefaultServerConfig implements ServerConfig { @Nullable private final Mapping sslContexts; - private final ServerMetrics serverMetrics = new ServerMetrics(); + private final ServerMetrics serverMetrics; @Nullable private String strVal; @@ -266,6 +266,7 @@ final class DefaultServerConfig implements ServerConfig { this.absoluteUriTransformer = castAbsoluteUriTransformer; this.unloggedExceptionsReportIntervalMillis = unloggedExceptionsReportIntervalMillis; this.shutdownSupports = ImmutableList.copyOf(requireNonNull(shutdownSupports, "shutdownSupports")); + serverMetrics = new ServerMetrics(meterRegistry); } private static Int2ObjectMap> buildDomainAndPortMapping( diff --git a/core/src/main/java/com/linecorp/armeria/server/Server.java b/core/src/main/java/com/linecorp/armeria/server/Server.java index 4dccd937874..6b8b0b5572c 100644 --- a/core/src/main/java/com/linecorp/armeria/server/Server.java +++ b/core/src/main/java/com/linecorp/armeria/server/Server.java @@ -562,7 +562,25 @@ private ChannelFuture doStart(ServerPort port) { final GracefulShutdownSupport gracefulShutdownSupport = this.gracefulShutdownSupport; assert gracefulShutdownSupport != null; - final ServerPortMetric serverPortMetric = new ServerPortMetric(); + ServerPortMetric serverPortMetric = null; + lock.lock(); + try { + for (ServerPort serverPort : activePorts.values()) { + final InetSocketAddress localAddress = serverPort.localAddress(); + if (!(localAddress instanceof DomainSocketAddress) && + localAddress.getPort() == port.localAddress().getPort()) { + // Because we use the port number for aggregating metrics, use the previous + // serverPortMetric. + serverPortMetric = serverPort.serverPortMetric(); + break; + } + } + } finally { + lock.unlock(); + } + if (serverPortMetric == null) { + serverPortMetric = new ServerPortMetric(); + } b.group(bossGroup, config.workerGroup()); b.handler(connectionLimitingHandler.newChildHandler(serverPortMetric)); b.childHandler(new HttpServerPipelineConfigurator(config, port, gracefulShutdownSupport, @@ -837,6 +855,9 @@ public void operationComplete(ChannelFuture f) { logger.warn("Unexpected local address type: {}", localAddress.getClass().getName()); return; } + final ServerPortMetric serverPortMetric = ch.attr(SERVER_PORT_METRIC).get(); + assert serverPortMetric != null; + actualPort.setServerPortMetric(serverPortMetric); // Update the boss thread so its name contains the actual port. Thread.currentThread().setName(bossThreadName(actualPort)); @@ -849,10 +870,7 @@ public void operationComplete(ChannelFuture f) { lock.unlock(); } - final ServerPortMetric serverPortMetric = ch.attr(SERVER_PORT_METRIC).get(); - assert serverPortMetric != null; - serverPortMetric.bindTo(config.meterRegistry(), actualPort); - config.serverMetrics().addServerPortMetric(serverPortMetric); + config.serverMetrics().addServerPort(actualPort); if (logger.isInfoEnabled()) { if (isLocalPort(actualPort)) { diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java b/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java index 6e59034b3d9..5d45f9e6114 100644 --- a/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java +++ b/core/src/main/java/com/linecorp/armeria/server/ServerMetrics.java @@ -16,14 +16,16 @@ package com.linecorp.armeria.server; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArraySet; import com.google.common.base.MoreObjects; import com.google.common.primitives.Ints; import com.linecorp.armeria.common.annotation.UnstableApi; +import io.micrometer.core.instrument.MeterRegistry; + /** * A class that holds metrics related server. */ @@ -33,12 +35,19 @@ public final class ServerMetrics { static final String ALL_REQUESTS_METER_NAME = "armeria.server.all.requests"; static final String ALL_CONNECTIONS_METER_NAME = "armeria.server.connections"; - private final List serverPortMetrics = new CopyOnWriteArrayList<>(); + private final Set serverPortMetrics = new CopyOnWriteArraySet<>(); + private final MeterRegistry meterRegistry; - ServerMetrics() {} + ServerMetrics(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } - void addServerPortMetric(ServerPortMetric serverPortMetric) { - serverPortMetrics.add(serverPortMetric); + void addServerPort(ServerPort serverPort) { + final ServerPortMetric serverPortMetric = serverPort.serverPortMetric(); + assert serverPortMetric != null; + if (serverPortMetrics.add(serverPortMetric)) { + serverPortMetric.bindTo(meterRegistry, serverPort); + } } /** diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerPort.java b/core/src/main/java/com/linecorp/armeria/server/ServerPort.java index f6da639aa7a..ff1580bf5be 100644 --- a/core/src/main/java/com/linecorp/armeria/server/ServerPort.java +++ b/core/src/main/java/com/linecorp/armeria/server/ServerPort.java @@ -66,6 +66,8 @@ static long nextPortGroup() { private final Set protocols; private final long portGroup; private int hashCode; + @Nullable + private ServerPortMetric serverPortMetric; @Nullable private String strVal; @@ -226,6 +228,16 @@ long portGroup() { return portGroup; } + @Nullable + ServerPortMetric serverPortMetric() { + return serverPortMetric; + } + + void setServerPortMetric(ServerPortMetric serverPortMetric) { + this.serverPortMetric = serverPortMetric; + } + + // Do not take account into serverPortMetric for equality and hashCode. @Override public int hashCode() { int hashCode = this.hashCode; diff --git a/core/src/main/java/com/linecorp/armeria/server/ServerPortMetric.java b/core/src/main/java/com/linecorp/armeria/server/ServerPortMetric.java index 3ed3e46da44..5d34f3115b7 100644 --- a/core/src/main/java/com/linecorp/armeria/server/ServerPortMetric.java +++ b/core/src/main/java/com/linecorp/armeria/server/ServerPortMetric.java @@ -149,6 +149,8 @@ void bindTo(MeterRegistry meterRegistry, ServerPort actualPort) { this, ServerPortMetric::activeConnections); } + // Use reference equality for comparison. + @Override public String toString() { return MoreObjects.toStringHelper(this)