Rate limit for listStatus API

(cherry picked from commit 87a5cb1)
pfnet · Jan 23, 2024 · 5e009a5 · 5e009a5
1 parent e0377b8
commit 5e009a5
Show file tree

Hide file tree

Showing 3 changed files with 63 additions and 0 deletions.
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -722,6 +722,32 @@
       short may cause retry storm from S3Gateway to OM.
     </description>
   </property>
+  <property>
+    <name>ozone.om.liststatus.ratelimit</name>
+    <value>0</value>
+    <tag>OZONE, OM, PERFORMANCE</tag>
+    <description>
+      Rate limit in listStatus in seconds. listStatus operation in OM is
+      relatively a heavy operation, especially in case many keys in a bucket
+      or in a prefix.
+
+      Should be positive to turn on this feature. Otherwise, 0 or negative
+      integer will disable rate limiting.
+    </description>
+  </property>
+  <property>
+    <name>ozone.om.liststatus.ratelimit-timeout</name>
+    <value>8</value>
+    <tag>OZONE, OM, PERFORMANCE</tag>
+    <description>
+      The duration of a wait time in seconds when a client try to enter the
+      rate-limited section of listStatus operation.
+
+      The shorter this is, the less the OM gets loaded. The longer this
+      is, the less errors clients will get. Caveat: also, making this too
+      short may cause retry storm from S3Gateway to OM.
+    </description>
+  </property>
   <property>
     <name>ozone.metadata.dirs</name>
     <value/>

diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
@@ -603,4 +603,13 @@ private OMConfigKeys() {
   public static final String OZONE_OM_LISTKEYS_RATELIMIT_TIMEOUT_KEY =
           "ozone.om.listkeys.ratelimit-timeout";
   public static final int OZONE_OM_LISTKEYS_RATELIMIT_TIMEOUT_DEFAULT = 8; // seconds
+
+  // Rate limit listStatus in OzoneManager (req/sec)
+  public static final String OZONE_OM_LISTSTATUS_RATELIMIT_KEY =
+          "ozone.om.liststatus.ratelimit";
+  public static final int OZONE_OM_LISTSTATUS_RATELIMIT_DEFAULT = 0;
+
+  public static final String OZONE_OM_LISTSTATUS_RATELIMIT_TIMEOUT_KEY =
+          "ozone.om.liststatus.ratelimit-timeout";
+  public static final int OZONE_OM_LISTSTATUS_RATELIMIT_TIMEOUT_DEFAULT = 8; // seconds
 }
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -376,6 +376,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
   private final UpgradeFinalizer<OzoneManager> upgradeFinalizer;
   private Optional<RateLimiter> listKeysRateLimiter;
   private Duration listKeysRateLimiterTimeout;
+  private Optional<RateLimiter> listStatusRateLimiter;
+  private Duration listStatusRateLimiterTimeout;
 
   /**
    * OM super user / admin list.
@@ -604,6 +606,7 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption)
     // Validates the default server-side replication configs.
     this.defaultReplicationConfig = getDefaultReplicationConfig();
 
+    // Ratelimit for listKeys
     int listKeysRateLimit = configuration.getInt(
         OMConfigKeys.OZONE_OM_LISTKEYS_RATELIMIT_KEY,
         OMConfigKeys.OZONE_OM_LISTKEYS_RATELIMIT_DEFAULT);
@@ -621,6 +624,24 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption)
       LOG.info("ratelimit disabled: ratelimit={}", listKeysRateLimit);
     }
 
+    // Ratelimit for listStatus
+    int listStatusRateLimit = configuration.getInt(
+            OMConfigKeys.OZONE_OM_LISTSTATUS_RATELIMIT_KEY,
+            OMConfigKeys.OZONE_OM_LISTSTATUS_RATELIMIT_DEFAULT);
+    if (listStatusRateLimit > 0) {
+      listStatusRateLimiter = Optional.of(RateLimiter.create(listStatusRateLimit));
+
+      int timeout = configuration.getInt(
+              OMConfigKeys.OZONE_OM_LISTSTATUS_RATELIMIT_TIMEOUT_KEY,
+              OMConfigKeys.OZONE_OM_LISTSTATUS_RATELIMIT_TIMEOUT_DEFAULT);
+      listStatusRateLimiterTimeout = Duration.ofSeconds(timeout);
+      LOG.info("ratelimit enbled: ratelimit={}s timeout={}",
+              listStatusRateLimit, listStatusRateLimiterTimeout);
+    } else {
+      listStatusRateLimiter = Optional.absent();
+      LOG.info("ratelimit disabled: ratelimit={}", listStatusRateLimit);
+    }
+
     InetSocketAddress omNodeRpcAddr = omNodeDetails.getRpcAddress();
     // Honor property 'hadoop.security.token.service.use_ip'
     omRpcAddressTxt = new Text(SecurityUtil.buildTokenService(omNodeRpcAddr));
@@ -3697,6 +3718,13 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
   public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
       String startKey, long numEntries, boolean allowPartialPrefixes)
       throws IOException {
+    if (listStatusRateLimiter.isPresent()) {
+      boolean ok = listStatusRateLimiter.get().tryAcquire(listStatusRateLimiterTimeout);
+      if (!ok) {
+        throw new RetriableException("Rate limit exceeded for listStatus");
+      }
+    }
+
     try (ReferenceCounted<IOmMetadataReader, SnapshotCache> rcReader =
         getReader(args)) {
       return rcReader.get().listStatus(