change expire logic

shendanfengg · Sep 14, 2023 · 73ce73e · 73ce73e
1 parent 32d3856
commit 73ce73e
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 6 deletions.
diff --git a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/mapper/OptimizeHistoryMapper.java b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/mapper/OptimizeHistoryMapper.java
@@ -150,7 +150,16 @@ public interface OptimizeHistoryMapper {
   @Delete("delete from " + TABLE_NAME + " where " +
       "catalog_name = #{tableIdentifier.catalog} and db_name = #{tableIdentifier.database} " +
       "and table_name = #{tableIdentifier.tableName} " +
-      "and commit_time < #{expireTime, typeHandler=com.netease.arctic.ams.server.mybatis.Long2TsConvertor}")
+      "and commit_time < #{expireTime, typeHandler=com.netease.arctic.ams.server.mybatis.Long2TsConvertor} " +
+      "order by commit_time asc limit #{expireCount}")
   void expireOptimizeHistory(@Param("tableIdentifier") TableIdentifier tableIdentifier,
-                             @Param("expireTime") long expireTime);
+                             @Param("expireTime") long expireTime,
+                             @Param("expireCount") int expireCount);
+
+  @Select("select count(*) from " + TABLE_NAME + " where " +
+      "catalog_name = #{tableIdentifier.catalog} and db_name = #{tableIdentifier.database} " +
+      "and table_name = #{tableIdentifier.tableName} " +
+      "and commit_time < #{expireTime, typeHandler=com.netease.arctic.ams.server.mybatis.Long2TsConvertor}")
+  int selectOptimizeHistoryCount(@Param("tableIdentifier") TableIdentifier tableIdentifier,
+                                 @Param("expireTime") long expireTime);
 }
diff --git a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/mapper/SnapInfoCacheMapper.java b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/mapper/SnapInfoCacheMapper.java
@@ -57,8 +57,19 @@ Boolean snapshotIsCached(
       ".server.mybatis.Long2TsConvertor} and table_identifier=#{tableIdentifier, typeHandler=com.netease.arctic.ams" +
       ".server.mybatis.TableIdentifier2StringConverter} and inner_table = #{type} and snapshot_id not in (select " +
       "add_snapshot_id from file_info_cache where delete_snapshot_id is null and table_identifier=#{tableIdentifier, " +
-      "typeHandler=com.netease.arctic.ams.server.mybatis.TableIdentifier2StringConverter} and inner_table = #{type})")
+      "typeHandler=com.netease.arctic.ams.server.mybatis.TableIdentifier2StringConverter} and inner_table = #{type}) " +
+      "order by commit_time asc limit #{expireCount}")
   void expireCache(
+      @Param("expiredTime") long expiredTime, @Param("tableIdentifier") TableIdentifier tableIdentifier,
+      @Param("type") String tableType, @Param("expireCount") int expireCount);
+
+  @Select("select count(*) from " + TABLE_NAME +
+      " where commit_time < #{expiredTime, typeHandler=com.netease.arctic.ams" +
+      ".server.mybatis.Long2TsConvertor} and table_identifier=#{tableIdentifier, typeHandler=com.netease.arctic.ams" +
+      ".server.mybatis.TableIdentifier2StringConverter} and inner_table = #{type} and snapshot_id not in (select " +
+      "add_snapshot_id from file_info_cache where delete_snapshot_id is null and table_identifier=#{tableIdentifier, " +
+      "typeHandler=com.netease.arctic.ams.server.mybatis.TableIdentifier2StringConverter} and inner_table = #{type})")
+  int countExpiredCache(
       @Param("expiredTime") long expiredTime, @Param("tableIdentifier") TableIdentifier tableIdentifier,
       @Param("type") String tableType);
 

diff --git a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/optimize/OptimizeService.java b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/optimize/OptimizeService.java
@@ -649,7 +649,10 @@ public void expireOptimizeHistory(TableIdentifier tableIdentifier, long expireTi
     try (SqlSession sqlSession = getSqlSession(true)) {
       OptimizeHistoryMapper optimizeHistoryMapper =
           getMapper(sqlSession, OptimizeHistoryMapper.class);
-      optimizeHistoryMapper.expireOptimizeHistory(tableIdentifier, expireTime);
+      int expireCount = optimizeHistoryMapper.selectOptimizeHistoryCount(tableIdentifier, expireTime);
+      if (expireCount > 30) {
+        optimizeHistoryMapper.expireOptimizeHistory(tableIdentifier, expireTime, expireCount - 30);
+      }
     }
   }
 

diff --git a/...server/src/main/java/com/netease/arctic/ams/server/service/impl/FileInfoCacheService.java b/...server/src/main/java/com/netease/arctic/ams/server/service/impl/FileInfoCacheService.java
@@ -174,8 +174,19 @@ public void expiredCache(long time) {
       SnapInfoCacheMapper snapInfoCacheMapper = getMapper(sqlSession, SnapInfoCacheMapper.class);
       List<TableMetadata> tableMetadata = ServiceContainer.getMetaService().listTables();
       tableMetadata.forEach(meta -> {
-        snapInfoCacheMapper.expireCache(time, meta.getTableIdentifier().buildTableIdentifier(), "base");
-        snapInfoCacheMapper.expireCache(time, meta.getTableIdentifier().buildTableIdentifier(), "change");
+        LOG.info("start expire file info cache of table {}", meta.getTableIdentifier());
+        int baseExpireCount = snapInfoCacheMapper.countExpiredCache(time,
+            meta.getTableIdentifier().buildTableIdentifier(), "base");
+        if (baseExpireCount > 30) {
+          snapInfoCacheMapper.expireCache(time, meta.getTableIdentifier().buildTableIdentifier(), "base",
+              baseExpireCount - 30);
+        }
+        int changeExpireCount = snapInfoCacheMapper.countExpiredCache(time,
+            meta.getTableIdentifier().buildTableIdentifier(), "change");
+        if (changeExpireCount > 30) {
+          snapInfoCacheMapper.expireCache(time, meta.getTableIdentifier().buildTableIdentifier(), "change",
+              changeExpireCount - 30);
+        }
       });
     }
   }