From 941d8e9495527b764525abc1d9b0f16adb94aacd Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Fri, 10 Jan 2025 11:33:51 +0100 Subject: [PATCH] Fix NesQuEIT to pass against recent Iceberg changes There are two Iceberg PRs that "broke" NesQuEIT: * https://github.com/apache/iceberg/pull/11478 caused `testRewriteManifests` to fail due to the changed outcome of the `rewrite_manifests` procedure * https://github.com/apache/iceberg/pull/11520 caused a class-path issue w/ Scala 2.13 --- .github/workflows/ci.yml | 7 +++++-- .../AbstractNessieSparkSqlExtensionTest.java | 10 +++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6227ff590a..b2328d7c2b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -950,8 +950,11 @@ jobs: - name: Nessie Spark 3.4 / 2.13 Extensions test run: ./gradlew -DscalaVersion=2.13 :nessie:nessie-iceberg:nessie-spark-extensions-3.4_2.13:test :nessie:nessie-iceberg:nessie-spark-extensions-3.4_2.13:intTest --scan - - name: Nessie Spark 3.5 / 2.13 Extensions test - run: ./gradlew -DscalaVersion=2.13 :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.13:test :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.13:intTest --scan + - name: Nessie Spark 3.5 / 2.12 Extensions test + run: ./gradlew -DscalaVersion=2.12 :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.12:test :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.12:intTest --scan + # TODO Revert the change to Scala 2.12 after we have an Iceberg release with https://github.com/apache/iceberg/pull/11520 (Iceberg 1.8 probably) + #- name: Nessie Spark 3.5 / 2.13 Extensions test + # run: ./gradlew -DscalaVersion=2.13 :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.13:test :nessie:nessie-iceberg:nessie-spark-extensions-3.5_2.13:intTest --scan #- name: Publish Nessie + Iceberg to local Maven repo # run: ./gradlew publishLocal --scan diff --git a/integrations/spark-extensions-basetests/src/main/java/org/projectnessie/spark/extensions/AbstractNessieSparkSqlExtensionTest.java b/integrations/spark-extensions-basetests/src/main/java/org/projectnessie/spark/extensions/AbstractNessieSparkSqlExtensionTest.java index 9eb6b7fe01..6646aca8ca 100644 --- a/integrations/spark-extensions-basetests/src/main/java/org/projectnessie/spark/extensions/AbstractNessieSparkSqlExtensionTest.java +++ b/integrations/spark-extensions-basetests/src/main/java/org/projectnessie/spark/extensions/AbstractNessieSparkSqlExtensionTest.java @@ -755,8 +755,16 @@ void testRewriteManifests(String branchName, String tableName) String version = spark.version(); if (version.startsWith("3.3.")) { soft.assertThat(rewriteResult.get(0)).startsWith(5, 1); - } else { + } else if (version.startsWith("3.4.")) { soft.assertThat(rewriteResult.get(0)).startsWith(11, 2); + } else { + // 3.5 onwards + soft.assertThat(rewriteResult.get(0)) + .satisfiesAnyOf( + result -> assertThat(result).startsWith(11, 2), + // Since https://github.com/apache/iceberg/pull/11478 (Spark: Change Delete + // granularity to file for Spark 3.5) + result -> assertThat(result).startsWith(8, 2)); } validateContentAfterMaintenance(branchName, tableName);