From 74ef6601e465498877d273b1b9b58d0cef7aac6f Mon Sep 17 00:00:00 2001 From: FANNG Date: Fri, 10 Jan 2025 15:12:42 +0800 Subject: [PATCH] [#6165] feat(core): Use Gravitino cloud jar without hadoop packages for Iceberg REST server (#6168) ### What changes were proposed in this pull request? 1. use Gravitino cloud jar without hadoop packages for Iceberg REST server credential vending in test and document 2. For OSS, use Gravitino Aliyun bundle jar in test and docker image because Iceberg doesn't provide Iceberg Aliyun bundle jar ### Why are the changes needed? Fix: #6165 ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? test S3 OSS GCS ADLS Iceberg REST test in local --- .../iceberg-rest-server-dependency.sh | 18 +++++++--------- docs/iceberg-rest-service.md | 8 ++++++- docs/security/credential-vending.md | 21 ++++++++++++++----- iceberg/iceberg-rest-server/build.gradle.kts | 3 ++- .../test/IcebergRESTADLSTokenIT.java | 2 +- .../test/IcebergRESTAzureAccountKeyIT.java | 2 +- .../integration/test/IcebergRESTGCSIT.java | 2 +- .../integration/test/IcebergRESTOSSIT.java | 2 ++ .../test/IcebergRESTOSSSecretIT.java | 2 ++ ...ESTS3IT.java => IcebergRESTS3TokenIT.java} | 4 ++-- 10 files changed, 41 insertions(+), 23 deletions(-) rename iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/{IcebergRESTS3IT.java => IcebergRESTS3TokenIT.java} (98%) diff --git a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh index 852b55b0206..aced0224f48 100755 --- a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh +++ b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh @@ -35,17 +35,18 @@ tar xfz gravitino-iceberg-rest-server-*.tar.gz cp -r gravitino-iceberg-rest-server*-bin ${iceberg_rest_server_dir}/packages/gravitino-iceberg-rest-server cd ${gravitino_home} -./gradlew :bundles:gcp-bundle:jar -./gradlew :bundles:aws-bundle:jar -./gradlew :bundles:azure-bundle:jar +./gradlew :bundles:gcp:jar +./gradlew :bundles:aws:jar +./gradlew :bundles:azure:jar +## Iceberg doesn't provide Iceberg Aliyun bundle jar, so use Gravitino aliyun bundle to provide OSS packages. ./gradlew :bundles:aliyun-bundle:jar # prepare bundle jar cd ${iceberg_rest_server_dir} mkdir -p bundles -cp ${gravitino_home}/bundles/gcp-bundle/build/libs/gravitino-gcp-bundle-*.jar bundles/ -cp ${gravitino_home}/bundles/aws-bundle/build/libs/gravitino-aws-bundle-*.jar bundles/ -cp ${gravitino_home}/bundles/azure-bundle/build/libs/gravitino-azure-bundle-*.jar bundles/ +cp ${gravitino_home}/bundles/gcp/build/libs/gravitino-gcp-*.jar bundles/ +cp ${gravitino_home}/bundles/aws/build/libs/gravitino-aws-*.jar bundles/ +cp ${gravitino_home}/bundles/azure/build/libs/gravitino-azure-*.jar bundles/ cp ${gravitino_home}/bundles/aliyun-bundle/build/libs/gravitino-aliyun-bundle-*.jar bundles/ iceberg_gcp_bundle="iceberg-gcp-bundle-1.5.2.jar" @@ -63,11 +64,6 @@ if [ ! -f "bundles/${iceberg_azure_bundle}" ]; then curl -L -s -o bundles/${iceberg_azure_bundle} https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/1.5.2/${iceberg_azure_bundle} fi -iceberg_aliyun_bundle="iceberg-aliyun-bundle-1.5.2.jar" -if [ ! -f "bundles/${iceberg_aliyun_bundle}" ]; then - curl -L -s -o bundles/${iceberg_aliyun_bundle} https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aliyun-bundle/1.5.2/${iceberg_aliyun_bundle} -fi - # download jdbc driver curl -L -s -o bundles/sqlite-jdbc-3.42.0.0.jar https://repo1.maven.org/maven2/org/xerial/sqlite-jdbc/3.42.0.0/sqlite-jdbc-3.42.0.0.jar diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index d42fc98b4dd..a4846d0e0dd 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -134,8 +134,14 @@ For other Iceberg OSS properties not managed by Gravitino like `client.security- Please refer to [OSS credentials](./security/credential-vending.md#oss-credentials) for credential related configurations. +Additionally, Iceberg doesn't provide Iceberg Aliyun bundle jar which contains OSS packages, there are two alternatives to use OSS packages: +1. Use [Gravitino Aliyun bundle jar with hadoop packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun-bundle). +2. Use [Aliyun JAVA SDK](https://gosspublic.alicdn.com/sdks/java/aliyun_java_sdk_3.10.2.zip) and extract `aliyun-sdk-oss-3.10.2.jar`, `hamcrest-core-1.1.jar`, `jdom2-2.0.6.jar` jars. + +Please place the above jars in the classpath of Iceberg REST server, please refer to [server management](#server-management) for classpath details. + :::info -Please set the `gravitino.iceberg-rest.warehouse` parameter to `oss://{bucket_name}/${prefix_name}`. Additionally, download the [Aliyun OSS SDK](https://gosspublic.alicdn.com/sdks/java/aliyun_java_sdk_3.10.2.zip) and copy `aliyun-sdk-oss-3.10.2.jar`, `hamcrest-core-1.1.jar`, `jdom2-2.0.6.jar` in the classpath of Iceberg REST server, `iceberg-rest-server/libs` for the auxiliary server, `libs` for the standalone server. +Please set the `gravitino.iceberg-rest.warehouse` parameter to `oss://{bucket_name}/${prefix_name}`. ::: #### GCS diff --git a/docs/security/credential-vending.md b/docs/security/credential-vending.md index 92370f4315d..b5391ac3152 100644 --- a/docs/security/credential-vending.md +++ b/docs/security/credential-vending.md @@ -132,12 +132,23 @@ Gravitino supports custom credentials, you can implement the `org.apache.graviti Besides setting credentials related configuration, please download Gravitino cloud bundle jar and place it in the classpath of Iceberg REST server or Hadoop catalog. -Gravitino cloud bundle jar: +For Hadoop catalog, please use Gravitino cloud bundle jar with Hadoop and cloud packages: -- [Gravitino AWS bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aws-bundle) -- [Gravitino Aliyun bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun-bundle) -- [Gravitino GCP bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-gcp-bundle) -- [Gravitino Azure bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-azure-bundle) +- [Gravitino AWS bundle jar with Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aws-bundle) +- [Gravitino Aliyun bundle jar with Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun-bundle) +- [Gravitino GCP bundle jar with Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-gcp-bundle) +- [Gravitino Azure bundle jar with Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-azure-bundle) + +For Iceberg REST catalog server, please use Gravitino cloud bundle jar without Hadoop and cloud packages: + +- [Gravitino AWS bundle jar without Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aws) +- [Gravitino Aliyun bundle jar without Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun) +- [Gravitino GCP bundle jar without Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-gcp) +- [Gravitino Azure bundle jar without Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-azure) + +:::note +For OSS, Iceberg doesn't provide Iceberg Aliyun bundle jar which contains OSS packages, you could provide the OSS jar by yourself or use [Gravitino Aliyun bundle jar with Hadoop and cloud packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun-bundle), please refer to [OSS configuration](../iceberg-rest-service.md#oss-configuration) for more details. +::: The classpath of the server: diff --git a/iceberg/iceberg-rest-server/build.gradle.kts b/iceberg/iceberg-rest-server/build.gradle.kts index fe35c4e7789..925ad900762 100644 --- a/iceberg/iceberg-rest-server/build.gradle.kts +++ b/iceberg/iceberg-rest-server/build.gradle.kts @@ -62,7 +62,8 @@ dependencies { annotationProcessor(libs.lombok) compileOnly(libs.lombok) - testImplementation(project(":bundles:aliyun")) + // Iceberg doesn't provide Aliyun bundle jar, use Gravitino Aliyun bundle to provide OSS packages + testImplementation(project(":bundles:aliyun-bundle")) testImplementation(project(":bundles:aws")) testImplementation(project(":bundles:gcp", configuration = "shadow")) testImplementation(project(":bundles:azure", configuration = "shadow")) diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java index 52ccb876df9..bf718d601d7 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java @@ -130,7 +130,7 @@ private void downloadIcebergAzureBundleJar() throws IOException { private void copyAzureBundleJar() { String gravitinoHome = System.getenv("GRAVITINO_HOME"); String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome); - BaseIT.copyBundleJarsToDirectory("azure-bundle", targetDir); + BaseIT.copyBundleJarsToDirectory("azure", targetDir); } @Test diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java index f999f84f58d..4f3c608fe3b 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java @@ -113,6 +113,6 @@ private void downloadIcebergAzureBundleJar() throws IOException { private void copyAzureBundleJar() { String gravitinoHome = System.getenv("GRAVITINO_HOME"); String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome); - BaseIT.copyBundleJarsToDirectory("azure-bundle", targetDir); + BaseIT.copyBundleJarsToDirectory("azure", targetDir); } } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java index 3396b60e1fd..74bf55edc09 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java @@ -88,7 +88,7 @@ private Map getGCSConfig() { private void copyGCSBundleJar() { String gravitinoHome = System.getenv("GRAVITINO_HOME"); String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome); - BaseIT.copyBundleJarsToDirectory("gcp-bundle", targetDir); + BaseIT.copyBundleJarsToDirectory("gcp", targetDir); } private void downloadIcebergBundleJar() throws IOException { diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java index 4c4b4a953bc..8e72ce33f1b 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java @@ -126,6 +126,8 @@ private void downloadIcebergForAliyunJar() throws IOException { private void copyAliyunOSSJar() { String gravitinoHome = System.getenv("GRAVITINO_HOME"); String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome); + // Iceberg doesn't provide Iceberg Aliyun bundle jar, so use Gravitino aliyun bundle to provide + // OSS packages. BaseIT.copyBundleJarsToDirectory("aliyun-bundle", targetDir); } } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java index 0be69cbe3d7..3b198c9d29e 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java @@ -111,6 +111,8 @@ private void downloadIcebergForAliyunJar() throws IOException { private void copyAliyunOSSJar() { String gravitinoHome = System.getenv("GRAVITINO_HOME"); String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome); + // Iceberg doesn't provide Iceberg Aliyun bundle jar, so use Gravitino aliyun bundle to provide + // OSS packages. BaseIT.copyBundleJarsToDirectory("aliyun-bundle", targetDir); } } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java similarity index 98% rename from iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java rename to iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java index e906018f525..ef1551f91b4 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java @@ -40,7 +40,7 @@ @SuppressWarnings("FormatStringAnnotation") @EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches = "true") -public class IcebergRESTS3IT extends IcebergRESTJdbcCatalogIT { +public class IcebergRESTS3TokenIT extends IcebergRESTJdbcCatalogIT { private String s3Warehouse; private String accessKey; @@ -124,7 +124,7 @@ private void downloadIcebergAwsBundleJar() throws IOException { private void copyS3BundleJar() { String gravitinoHome = System.getenv("GRAVITINO_HOME"); String targetDir = String.format("%s/iceberg-rest-server/libs/", gravitinoHome); - BaseIT.copyBundleJarsToDirectory("aws-bundle", targetDir); + BaseIT.copyBundleJarsToDirectory("aws", targetDir); } /**