diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/CommonPartitionEvaluator.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/CommonPartitionEvaluator.java index 013f11f87a..8038e97618 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/CommonPartitionEvaluator.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/CommonPartitionEvaluator.java @@ -24,6 +24,7 @@ import org.apache.amoro.shade.guava32.com.google.common.base.MoreObjects; import org.apache.amoro.shade.guava32.com.google.common.base.Preconditions; import org.apache.amoro.shade.guava32.com.google.common.collect.Sets; +import org.apache.amoro.utils.TableFileUtil; import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.FileContent; @@ -226,8 +227,12 @@ public boolean fileShouldRewrite(DataFile dataFile, List> deletes public boolean segmentShouldRewritePos(DataFile dataFile, List> deletes) { Preconditions.checkArgument(!isFragmentFile(dataFile), "Unsupported fragment file."); - if (deletes.stream().filter(delete -> delete.content() == FileContent.POSITION_DELETES).count() - >= 2) { + long posDeleteFileCount = + deletes.stream().filter(delete -> delete.content() == FileContent.POSITION_DELETES).count(); + if (posDeleteFileCount == 1) { + return !TableFileUtil.isOptimizingPosDeleteFile( + dataFile.path().toString(), deletes.get(0).path().toString()); + } else if (posDeleteFileCount > 1) { combinePosSegmentFileCount++; return true; } diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/io/writer/IcebergFanoutPosDeleteWriter.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/io/writer/IcebergFanoutPosDeleteWriter.java index b709e18f21..ea3b7ef89d 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/io/writer/IcebergFanoutPosDeleteWriter.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/io/writer/IcebergFanoutPosDeleteWriter.java @@ -157,7 +157,10 @@ private void flushDeletes() { String fileDir = TableFileUtil.getFileDir(filePath.get().toString()); String deleteFilePath = format.addExtension( - String.format("%s/%s-delete-%s", fileDir, fileName, fileNameSuffix)); + String.format( + "%s/%s", + fileDir, + TableFileUtil.optimizingPosDeleteFileName(fileName, fileNameSuffix))); EncryptedOutputFile outputFile = encryptionManager.encrypt(fileIO.newOutputFile(deleteFilePath)); diff --git a/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/TableFileUtil.java b/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/TableFileUtil.java index e1b57e45dd..e140c79ebf 100644 --- a/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/TableFileUtil.java +++ b/amoro-format-iceberg/src/main/java/org/apache/amoro/utils/TableFileUtil.java @@ -33,6 +33,7 @@ public class TableFileUtil { private static final Logger LOG = LoggerFactory.getLogger(TableFileUtil.class); + private static final String POS_DELETE_FILE_IDENTIFIER = "delete"; /** * Parse file name form file path @@ -192,4 +193,13 @@ public static String getParent(String path) { Path p = new Path(path); return p.getParent().toString(); } + + public static String optimizingPosDeleteFileName(String dataFileName, String suffix) { + return String.format("%s-%s-%s", dataFileName, POS_DELETE_FILE_IDENTIFIER, suffix); + } + + public static boolean isOptimizingPosDeleteFile(String dataFilePath, String posDeleteFilePath) { + return getFileName(posDeleteFilePath) + .startsWith(String.format("%s-%s", getFileName(dataFilePath), POS_DELETE_FILE_IDENTIFIER)); + } }