[AMORO-3192] Add the "clean-orphan-file.interval-time-minutes" parame…

…ter (#3193) * Add the "clean-orphan-file.interval-time-minutes" parameter * Rename the parameter to "clean-orphan-files.interval" and use the new parameter type. * rollback configurations doc and add example to config.yaml
apache · Sep 11, 2024 · d4ea2fa · d4ea2fa
1 parent 1f4f90a
commit d4ea2fa
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 5 deletions.
diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java
@@ -102,6 +102,12 @@ public class AmoroManagementConf {
           .defaultValue(10)
           .withDescription("The number of threads used for orphan files cleaning.");
 
+  public static final ConfigOption<Duration> CLEAN_ORPHAN_FILES_INTERVAL =
+      ConfigOptions.key("clean-orphan-files.interval")
+          .durationType()
+          .defaultValue(Duration.ofDays(1))
+          .withDescription("Interval for cleaning orphan files.");
+
   public static final ConfigOption<Boolean> CLEAN_DANGLING_DELETE_FILES_ENABLED =
       ConfigOptions.key("clean-dangling-delete-files.enabled")
           .booleanType()

diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/AsyncTableExecutors.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/AsyncTableExecutors.java
@@ -49,7 +49,9 @@ public void setup(TableManager tableManager, Configurations conf) {
     if (conf.getBoolean(AmoroManagementConf.CLEAN_ORPHAN_FILES_ENABLED)) {
       this.orphanFilesCleaningExecutor =
           new OrphanFilesCleaningExecutor(
-              tableManager, conf.getInteger(AmoroManagementConf.CLEAN_ORPHAN_FILES_THREAD_COUNT));
+              tableManager,
+              conf.getInteger(AmoroManagementConf.CLEAN_ORPHAN_FILES_THREAD_COUNT),
+              conf.get(AmoroManagementConf.CLEAN_ORPHAN_FILES_INTERVAL));
     }
     if (conf.getBoolean(AmoroManagementConf.CLEAN_DANGLING_DELETE_FILES_ENABLED)) {
       this.danglingDeleteFilesCleaningExecutor =

diff --git a/...ams/src/main/java/org/apache/amoro/server/table/executor/OrphanFilesCleaningExecutor.java b/...ams/src/main/java/org/apache/amoro/server/table/executor/OrphanFilesCleaningExecutor.java
@@ -26,18 +26,20 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.time.Duration;
+
 public class OrphanFilesCleaningExecutor extends BaseTableExecutor {
   private static final Logger LOG = LoggerFactory.getLogger(OrphanFilesCleaningExecutor.class);
+  private final Duration interval;
 
-  private static final long INTERVAL = 24 * 60 * 60 * 1000L;
-
-  public OrphanFilesCleaningExecutor(TableManager tableRuntimes, int poolSize) {
+  public OrphanFilesCleaningExecutor(TableManager tableRuntimes, int poolSize, Duration interval) {
     super(tableRuntimes, poolSize);
+    this.interval = interval;
   }
 
   @Override
   protected long getNextExecutingTime(TableRuntime tableRuntime) {
-    return INTERVAL;
+    return interval.toMillis();
   }
 
   @Override

diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/table/TableProperties.java
@@ -173,6 +173,7 @@ private TableProperties() {}
   public static final String MIN_ORPHAN_FILE_EXISTING_TIME =
       "clean-orphan-file.min-existing-time-minutes";
   public static final long MIN_ORPHAN_FILE_EXISTING_TIME_DEFAULT = 2880; // 2 Days
+
   public static final String ENABLE_DANGLING_DELETE_FILES_CLEAN =
       "clean-dangling-delete-files.enabled";
   public static final boolean ENABLE_DANGLING_DELETE_FILES_CLEAN_DEFAULT = true;

diff --git a/dist/src/main/amoro-bin/conf/config.yaml b/dist/src/main/amoro-bin/conf/config.yaml
@@ -66,6 +66,7 @@ ams:
   clean-orphan-files:
     enabled: true
     thread-count: 10
+    interval: 1d
 
   clean-dangling-delete-files:
     enabled: true