diff --git a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/mapper/OptimizeHistoryMapper.java b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/mapper/OptimizeHistoryMapper.java index 06ccd5e928..1188a6d4eb 100644 --- a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/mapper/OptimizeHistoryMapper.java +++ b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/mapper/OptimizeHistoryMapper.java @@ -150,7 +150,16 @@ public interface OptimizeHistoryMapper { @Delete("delete from " + TABLE_NAME + " where " + "catalog_name = #{tableIdentifier.catalog} and db_name = #{tableIdentifier.database} " + "and table_name = #{tableIdentifier.tableName} " + - "and commit_time < #{expireTime, typeHandler=com.netease.arctic.ams.server.mybatis.Long2TsConvertor}") + "and commit_time < #{expireTime, typeHandler=com.netease.arctic.ams.server.mybatis.Long2TsConvertor} " + + "order by commit_time asc limit #{expireCount}") void expireOptimizeHistory(@Param("tableIdentifier") TableIdentifier tableIdentifier, - @Param("expireTime") long expireTime); + @Param("expireTime") long expireTime, + @Param("expireCount") int expireCount); + + @Select("select count(*) from " + TABLE_NAME + " where " + + "catalog_name = #{tableIdentifier.catalog} and db_name = #{tableIdentifier.database} " + + "and table_name = #{tableIdentifier.tableName} " + + "and commit_time < #{expireTime, typeHandler=com.netease.arctic.ams.server.mybatis.Long2TsConvertor}") + int selectOptimizeHistoryCount(@Param("tableIdentifier") TableIdentifier tableIdentifier, + @Param("expireTime") long expireTime); } diff --git a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/mapper/SnapInfoCacheMapper.java b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/mapper/SnapInfoCacheMapper.java index e0e20df017..121574e7fc 100644 --- a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/mapper/SnapInfoCacheMapper.java +++ b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/mapper/SnapInfoCacheMapper.java @@ -57,8 +57,19 @@ Boolean snapshotIsCached( ".server.mybatis.Long2TsConvertor} and table_identifier=#{tableIdentifier, typeHandler=com.netease.arctic.ams" + ".server.mybatis.TableIdentifier2StringConverter} and inner_table = #{type} and snapshot_id not in (select " + "add_snapshot_id from file_info_cache where delete_snapshot_id is null and table_identifier=#{tableIdentifier, " + - "typeHandler=com.netease.arctic.ams.server.mybatis.TableIdentifier2StringConverter} and inner_table = #{type})") + "typeHandler=com.netease.arctic.ams.server.mybatis.TableIdentifier2StringConverter} and inner_table = #{type}) " + + "order by commit_time asc limit #{expireCount}") void expireCache( + @Param("expiredTime") long expiredTime, @Param("tableIdentifier") TableIdentifier tableIdentifier, + @Param("type") String tableType, @Param("expireCount") int expireCount); + + @Select("select count(*) from " + TABLE_NAME + + " where commit_time < #{expiredTime, typeHandler=com.netease.arctic.ams" + + ".server.mybatis.Long2TsConvertor} and table_identifier=#{tableIdentifier, typeHandler=com.netease.arctic.ams" + + ".server.mybatis.TableIdentifier2StringConverter} and inner_table = #{type} and snapshot_id not in (select " + + "add_snapshot_id from file_info_cache where delete_snapshot_id is null and table_identifier=#{tableIdentifier, " + + "typeHandler=com.netease.arctic.ams.server.mybatis.TableIdentifier2StringConverter} and inner_table = #{type})") + int countExpiredCache( @Param("expiredTime") long expiredTime, @Param("tableIdentifier") TableIdentifier tableIdentifier, @Param("type") String tableType); diff --git a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/optimize/OptimizeService.java b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/optimize/OptimizeService.java index 52abe1532a..b0db9c7542 100644 --- a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/optimize/OptimizeService.java +++ b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/optimize/OptimizeService.java @@ -649,7 +649,10 @@ public void expireOptimizeHistory(TableIdentifier tableIdentifier, long expireTi try (SqlSession sqlSession = getSqlSession(true)) { OptimizeHistoryMapper optimizeHistoryMapper = getMapper(sqlSession, OptimizeHistoryMapper.class); - optimizeHistoryMapper.expireOptimizeHistory(tableIdentifier, expireTime); + int expireCount = optimizeHistoryMapper.selectOptimizeHistoryCount(tableIdentifier, expireTime); + if (expireCount > 30) { + optimizeHistoryMapper.expireOptimizeHistory(tableIdentifier, expireTime, expireCount - 30); + } } } diff --git a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/service/impl/FileInfoCacheService.java b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/service/impl/FileInfoCacheService.java index 7c5a120afc..056904e4e5 100644 --- a/ams/ams-server/src/main/java/com/netease/arctic/ams/server/service/impl/FileInfoCacheService.java +++ b/ams/ams-server/src/main/java/com/netease/arctic/ams/server/service/impl/FileInfoCacheService.java @@ -174,8 +174,19 @@ public void expiredCache(long time) { SnapInfoCacheMapper snapInfoCacheMapper = getMapper(sqlSession, SnapInfoCacheMapper.class); List tableMetadata = ServiceContainer.getMetaService().listTables(); tableMetadata.forEach(meta -> { - snapInfoCacheMapper.expireCache(time, meta.getTableIdentifier().buildTableIdentifier(), "base"); - snapInfoCacheMapper.expireCache(time, meta.getTableIdentifier().buildTableIdentifier(), "change"); + LOG.info("start expire file info cache of table {}", meta.getTableIdentifier()); + int baseExpireCount = snapInfoCacheMapper.countExpiredCache(time, + meta.getTableIdentifier().buildTableIdentifier(), "base"); + if (baseExpireCount > 30) { + snapInfoCacheMapper.expireCache(time, meta.getTableIdentifier().buildTableIdentifier(), "base", + baseExpireCount - 30); + } + int changeExpireCount = snapInfoCacheMapper.countExpiredCache(time, + meta.getTableIdentifier().buildTableIdentifier(), "change"); + if (changeExpireCount > 30) { + snapInfoCacheMapper.expireCache(time, meta.getTableIdentifier().buildTableIdentifier(), "change", + changeExpireCount - 30); + } }); } }