throttle request metrics (#541)

Netflix · Aug 25, 2023 · b155b8b · b155b8b
1 parent d0ead57
commit b155b8b
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 1 deletion.
diff --git a/...ontrol-plane-server/src/main/java/io/mantisrx/master/api/akka/route/MasterApiMetrics.java b/...ontrol-plane-server/src/main/java/io/mantisrx/master/api/akka/route/MasterApiMetrics.java
@@ -25,6 +25,7 @@ public class MasterApiMetrics {
     private final Counter resp4xx;
     private final Counter resp5xx;
     private final Counter incomingRequestCount;
+    private final Counter throttledRequestCount;
     private final Counter askTimeOutCount;
 
     private static final MasterApiMetrics INSTANCE = new MasterApiMetrics();
@@ -33,6 +34,7 @@ private MasterApiMetrics() {
         Metrics m = new Metrics.Builder()
                 .id("MasterApiMetrics")
                 .addCounter("incomingRequestCount")
+                .addCounter("throttledRequestCount")
                 .addCounter("resp2xx")
                 .addCounter("resp4xx")
                 .addCounter("resp5xx")
@@ -44,6 +46,7 @@ private MasterApiMetrics() {
         this.resp4xx = metrics.getCounter("resp4xx");
         this.resp5xx = metrics.getCounter("resp5xx");
         this.incomingRequestCount = metrics.getCounter("incomingRequestCount");
+        this.throttledRequestCount = metrics.getCounter("throttledRequestCount");
     }
 
     public static final MasterApiMetrics getInstance() {
@@ -69,4 +72,8 @@ public void incrementAskTimeOutCount() {
     public void incrementIncomingRequestCount() {
         incomingRequestCount.increment();
     }
+
+    public void incrementThrottledRequestCount() {
+        throttledRequestCount.increment();
+    }
 }
diff --git a/...is-control-plane-server/src/main/java/io/mantisrx/master/api/akka/route/v1/BaseRoute.java b/...is-control-plane-server/src/main/java/io/mantisrx/master/api/akka/route/v1/BaseRoute.java
@@ -332,6 +332,7 @@ protected  <T> Route withFuture(CompletableFuture<T> tFuture) {
 
                     if (throwable instanceof RequestThrottledException) {
                         MasterApiMetrics.getInstance().incrementResp4xx();
+                        MasterApiMetrics.getInstance().incrementThrottledRequestCount();
                         return complete(StatusCodes.TOO_MANY_REQUESTS);
                     }
 

diff --git a/...rver/src/main/java/io/mantisrx/master/resourcecluster/ResourceClusterGatewayAkkaImpl.java b/...rver/src/main/java/io/mantisrx/master/resourcecluster/ResourceClusterGatewayAkkaImpl.java
@@ -53,7 +53,7 @@ class ResourceClusterGatewayAkkaImpl implements ResourceClusterGateway {
 
     private <In, Out> Function<In, CompletableFuture<Out>> withThrottle(Function<In, CompletableFuture<Out>> func) {
         return in -> {
-            if (rateLimiter.tryAcquire(1, TimeUnit.SECONDS)) {
+            if (rateLimiter.tryAcquire(200, TimeUnit.MILLISECONDS)) {
                 return func.apply(in);
             } else {
                 return CompletableFutures.exceptionallyCompletedFuture(