From bdd982c3139e82ce7fd82ed02c8067f5115d8cb2 Mon Sep 17 00:00:00 2001 From: Ismail Simsek Date: Wed, 8 Jan 2025 13:15:45 +0100 Subject: [PATCH] Use SupportsPrefixOperations for Remove OrphanFile Procedure on Spark 3.5, improve naming --- .../actions/TestRemoveOrphanFilesAction.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java index 9b854a56f8ba..226dab05f90f 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java @@ -843,41 +843,41 @@ public void testCompareToFileList() throws IOException { .withColumnRenamed("filePath", "file_path") .withColumnRenamed("lastModified", "last_modified"); - DeleteOrphanFiles.Result result1 = + DeleteOrphanFiles.Result deletedOrphanFiles1 = actions .deleteOrphanFiles(table) .compareToFileList(compareToFileList) .deleteWith(s -> {}) .execute(); - assertThat(result1.orphanFileLocations()) + assertThat(deletedOrphanFiles1.orphanFileLocations()) .as("Default olderThan interval should be safe") .isEmpty(); - DeleteOrphanFiles.Result result2 = + DeleteOrphanFiles.Result deletedOrphanFiles2 = actions .deleteOrphanFiles(table) .compareToFileList(compareToFileList) .olderThan(System.currentTimeMillis()) .deleteWith(s -> {}) .execute(); - assertThat(result2.orphanFileLocations()) + assertThat(deletedOrphanFiles2.orphanFileLocations()) .as("Action should find 1 file") .isEqualTo(invalidFilePaths); assertThat(fs.exists(new Path(invalidFilePaths.get(0)))) .as("Invalid file should be present") .isTrue(); - DeleteOrphanFilesSparkAction deleteOrphanFilesSparkAction = + DeleteOrphanFilesSparkAction deleteOrphanFilesSparkAction3 = actions .deleteOrphanFiles(table) .compareToFileList(compareToFileList) .olderThan(System.currentTimeMillis()); assertThatDatasetsAreEqualIgnoringOrder( - deleteOrphanFilesSparkAction.listWithPrefix(), - deleteOrphanFilesSparkAction.listWithoutPrefix()); + deleteOrphanFilesSparkAction3.listWithPrefix(), + deleteOrphanFilesSparkAction3.listWithoutPrefix()); - DeleteOrphanFiles.Result result3 = deleteOrphanFilesSparkAction.execute(); - assertThat(result3.orphanFileLocations()) + DeleteOrphanFiles.Result deletedOrphanFiles3 = deleteOrphanFilesSparkAction3.execute(); + assertThat(deletedOrphanFiles3.orphanFileLocations()) .as("Action should delete 1 file") .isEqualTo(invalidFilePaths); assertThat(fs.exists(new Path(invalidFilePaths.get(0)))) @@ -902,17 +902,19 @@ public void testCompareToFileList() throws IOException { .withColumnRenamed("filePath", "file_path") .withColumnRenamed("lastModified", "last_modified"); - deleteOrphanFilesSparkAction = + DeleteOrphanFilesSparkAction deleteOrphanFilesSparkAction4 = actions .deleteOrphanFiles(table) .compareToFileList(compareToFileListWithOutsideLocation) .deleteWith(s -> {}); assertThatDatasetsAreEqualIgnoringOrder( - deleteOrphanFilesSparkAction.listWithPrefix(), - deleteOrphanFilesSparkAction.listWithoutPrefix()); + deleteOrphanFilesSparkAction4.listWithPrefix(), + deleteOrphanFilesSparkAction4.listWithoutPrefix()); - DeleteOrphanFiles.Result result4 = deleteOrphanFilesSparkAction.execute(); - assertThat(result4.orphanFileLocations()).as("Action should find nothing").isEmpty(); + DeleteOrphanFiles.Result deletedOrphanFiles4 = deleteOrphanFilesSparkAction4.execute(); + assertThat(deletedOrphanFiles4.orphanFileLocations()) + .as("Action should find nothing") + .isEmpty(); } protected long waitUntilAfter(long timestampMillis) {