diff --git a/api/src/main/java/org/apache/gravitino/exceptions/IllegalMetadataObjectException.java b/api/src/main/java/org/apache/gravitino/exceptions/IllegalMetadataObjectException.java new file mode 100644 index 00000000000..7a955f268e1 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/IllegalMetadataObjectException.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** An exception thrown when a metadata object is invalid. */ +public class IllegalMetadataObjectException extends IllegalArgumentException { + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public IllegalMetadataObjectException(@FormatString String message, Object... args) { + super(String.format(message, args)); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * @param cause the cause. + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public IllegalMetadataObjectException( + Throwable cause, @FormatString String message, Object... args) { + super(String.format(message, args), cause); + } + + /** + * Constructs a new exception with the specified cause. + * + * @param cause the cause. + */ + public IllegalMetadataObjectException(Throwable cause) { + super(cause); + } + + /** Constructs a new exception with the specified detail message and cause. */ + public IllegalMetadataObjectException() { + super(); + } +} diff --git a/api/src/main/java/org/apache/gravitino/exceptions/IllegalRoleException.java b/api/src/main/java/org/apache/gravitino/exceptions/IllegalRoleException.java new file mode 100644 index 00000000000..d5a81fe4492 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/IllegalRoleException.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** An exception thrown when a role is invalid. */ +public class IllegalRoleException extends IllegalArgumentException { + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public IllegalRoleException(@FormatString String message, Object... args) { + super(String.format(message, args)); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * @param cause the cause. + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public IllegalRoleException(Throwable cause, @FormatString String message, Object... args) { + super(String.format(message, args), cause); + } + + /** + * Constructs a new exception with the specified cause. + * + * @param cause the cause. + */ + public IllegalRoleException(Throwable cause) { + super(cause); + } + + /** Constructs a new exception with the specified detail message and cause. */ + public IllegalRoleException() { + super(); + } +} diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java index 21462b9ca91..004bde0bd7e 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergConstants.java @@ -68,7 +68,10 @@ public class IcebergConstants { public static final String ICEBERG_REST_CATALOG_CACHE_EVICTION_INTERVAL = "catalog-cache-eviction-interval-ms"; - public static final String ICEBERG_REST_CATALOG_PROVIDER = "catalog-provider"; + public static final String ICEBERG_REST_CATALOG_CONFIG_PROVIDER = "catalog-config-provider"; + public static final String STATIC_ICEBERG_CATALOG_CONFIG_PROVIDER_NAME = "static-config-provider"; + public static final String DYNAMIC_ICEBERG_CATALOG_CONFIG_PROVIDER_NAME = + "dynamic-config-provider"; public static final String GRAVITINO_URI = "gravitino-uri"; diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java new file mode 100644 index 00000000000..596268395e3 --- /dev/null +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +public class CredentialConstants { + public static final String CREDENTIAL_PROVIDER_TYPE = "credential-provider-type"; + + private CredentialConstants() {} +} diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java index c4bac4df124..37124dc5f33 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/ops/TestIcebergTableUpdate.java @@ -18,9 +18,11 @@ */ package org.apache.gravitino.catalog.lakehouse.iceberg.ops; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper.IcebergTableChange; import org.apache.gravitino.rel.TableChange; @@ -79,7 +81,7 @@ public class TestIcebergTableUpdate { @BeforeEach public void init() { - icebergCatalogWrapper = new IcebergCatalogWrapper(); + icebergCatalogWrapper = new IcebergCatalogWrapper(new IcebergConfig(Collections.emptyMap())); icebergCatalogWrapperHelper = new IcebergCatalogWrapperHelper(icebergCatalogWrapper.getCatalog()); createNamespace(TEST_NAMESPACE_NAME); diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java b/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java index a2ff07e27ad..db45b643612 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java @@ -34,7 +34,9 @@ import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.ForbiddenException; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; +import org.apache.gravitino.exceptions.IllegalMetadataObjectException; import org.apache.gravitino.exceptions.IllegalPrivilegeException; +import org.apache.gravitino.exceptions.IllegalRoleException; import org.apache.gravitino.exceptions.InUseException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchCatalogException; @@ -706,6 +708,10 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.ILLEGAL_ARGUMENTS_CODE: if (errorResponse.getType().equals(IllegalPrivilegeException.class.getSimpleName())) { throw new IllegalPrivilegeException(errorMessage); + } else if (errorResponse + .getType() + .equals(IllegalMetadataObjectException.class.getSimpleName())) { + throw new IllegalMetadataObjectException(errorMessage); } else { throw new IllegalArgumentException(errorMessage); } @@ -756,6 +762,8 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.ILLEGAL_ARGUMENTS_CODE: if (errorResponse.getType().equals(IllegalPrivilegeException.class.getSimpleName())) { throw new IllegalPrivilegeException(errorMessage); + } else if (errorResponse.getType().equals(IllegalRoleException.class.getSimpleName())) { + throw new IllegalRoleException(errorMessage); } else { throw new IllegalArgumentException(errorMessage); } diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoClient.java b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoClient.java index 0f3b88133d2..c0310f23873 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoClient.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoClient.java @@ -36,7 +36,9 @@ import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; import org.apache.gravitino.exceptions.CatalogInUseException; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; +import org.apache.gravitino.exceptions.IllegalMetadataObjectException; import org.apache.gravitino.exceptions.IllegalPrivilegeException; +import org.apache.gravitino.exceptions.IllegalRoleException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchGroupException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; @@ -297,12 +299,12 @@ public boolean deleteRole(String role) throws NoSuchMetalakeException { * @return The created Role instance. * @throws RoleAlreadyExistsException If a Role with the same name already exists. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. - * @throws NoSuchMetadataObjectException If securable object doesn't exist + * @throws IllegalMetadataObjectException If securable object is invalid * @throws RuntimeException If creating the Role encounters storage issues. */ public Role createRole( String role, Map properties, List securableObjects) - throws RoleAlreadyExistsException, NoSuchMetalakeException, NoSuchMetadataObjectException { + throws RoleAlreadyExistsException, NoSuchMetalakeException, IllegalMetadataObjectException { return getMetalake().createRole(role, properties, securableObjects); } /** @@ -312,12 +314,12 @@ public Role createRole( * @param roles The names of the Role. * @return The Group after granted. * @throws NoSuchUserException If the User with the given name does not exist. - * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws IllegalRoleException If the Role with the given name is invalid. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If granting roles to a user encounters storage issues. */ public User grantRolesToUser(List roles, String user) - throws NoSuchUserException, NoSuchRoleException, NoSuchMetalakeException { + throws NoSuchUserException, IllegalRoleException, NoSuchMetalakeException { return getMetalake().grantRolesToUser(roles, user); } @@ -328,12 +330,12 @@ public User grantRolesToUser(List roles, String user) * @param roles The names of the Role. * @return The Group after granted. * @throws NoSuchGroupException If the Group with the given name does not exist. - * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws IllegalRoleException If the Role with the given name is invalid. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If granting roles to a group encounters storage issues. */ public Group grantRolesToGroup(List roles, String group) - throws NoSuchGroupException, NoSuchRoleException, NoSuchMetalakeException { + throws NoSuchGroupException, IllegalRoleException, NoSuchMetalakeException { return getMetalake().grantRolesToGroup(roles, group); } @@ -344,12 +346,12 @@ public Group grantRolesToGroup(List roles, String group) * @param roles The names of the Role. * @return The User after revoked. * @throws NoSuchUserException If the User with the given name does not exist. - * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws IllegalRoleException If the Role with the given name is invalid. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If revoking roles from a user encounters storage issues. */ public User revokeRolesFromUser(List roles, String user) - throws NoSuchUserException, NoSuchRoleException, NoSuchMetalakeException { + throws NoSuchUserException, IllegalRoleException, NoSuchMetalakeException { return getMetalake().revokeRolesFromUser(roles, user); } @@ -360,12 +362,12 @@ public User revokeRolesFromUser(List roles, String user) * @param roles The names of the Role. * @return The Group after revoked. * @throws NoSuchGroupException If the Group with the given name does not exist. - * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws IllegalRoleException If the Role with the given name is invalid. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If revoking roles from a group encounters storage issues. */ public Group revokeRolesFromGroup(List roles, String group) - throws NoSuchGroupException, NoSuchRoleException, NoSuchMetalakeException { + throws NoSuchGroupException, IllegalRoleException, NoSuchMetalakeException { return getMetalake().revokeRolesFromGroup(roles, group); } diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoMetalake.java b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoMetalake.java index 441833bd49d..47f42d3ad22 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoMetalake.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoMetalake.java @@ -80,7 +80,9 @@ import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; import org.apache.gravitino.exceptions.CatalogInUseException; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; +import org.apache.gravitino.exceptions.IllegalMetadataObjectException; import org.apache.gravitino.exceptions.IllegalPrivilegeException; +import org.apache.gravitino.exceptions.IllegalRoleException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchGroupException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; @@ -785,12 +787,12 @@ public boolean deleteRole(String role) throws NoSuchMetalakeException { * @return The created Role instance. * @throws RoleAlreadyExistsException If a Role with the same name already exists. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. - * @throws NoSuchMetadataObjectException If the securable object doesn't exist + * @throws IllegalMetadataObjectException If the securable object is invalid * @throws RuntimeException If creating the Role encounters storage issues. */ public Role createRole( String role, Map properties, List securableObjects) - throws RoleAlreadyExistsException, NoSuchMetalakeException, NoSuchMetadataObjectException { + throws RoleAlreadyExistsException, NoSuchMetalakeException, IllegalMetadataObjectException { RoleCreateRequest req = new RoleCreateRequest( role, @@ -837,12 +839,12 @@ public String[] listRoleNames() { * @param roles The names of the Role. * @return The Group after granted. * @throws NoSuchUserException If the User with the given name does not exist. - * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws IllegalRoleException If the Role with the given name is invalid. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If granting roles to a user encounters storage issues. */ public User grantRolesToUser(List roles, String user) - throws NoSuchUserException, NoSuchRoleException, NoSuchMetalakeException { + throws NoSuchUserException, IllegalRoleException, NoSuchMetalakeException { RoleGrantRequest request = new RoleGrantRequest(roles); request.validate(); @@ -868,7 +870,7 @@ public User grantRolesToUser(List roles, String user) * @param roles The names of the Role. * @return The Group after granted. * @throws NoSuchGroupException If the Group with the given name does not exist. - * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws IllegalRoleException If the Role with the given name is invalid. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If granting roles to a group encounters storage issues. */ @@ -899,7 +901,7 @@ public Group grantRolesToGroup(List roles, String group) * @param roles The names of the Role. * @return The User after revoked. * @throws NoSuchUserException If the User with the given name does not exist. - * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws IllegalRoleException If the Role with the given name is invalid. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If revoking roles from a user encounters storage issues. */ @@ -930,12 +932,12 @@ public User revokeRolesFromUser(List roles, String user) * @param roles The names of the Role. * @return The Group after revoked. * @throws NoSuchGroupException If the Group with the given name does not exist. - * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws IllegalRoleException If the Role with the given name is invalid. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If revoking roles from a group encounters storage issues. */ public Group revokeRolesFromGroup(List roles, String group) - throws NoSuchGroupException, NoSuchRoleException, NoSuchMetalakeException { + throws NoSuchGroupException, IllegalRoleException, NoSuchMetalakeException { RoleRevokeRequest request = new RoleRevokeRequest(roles); request.validate(); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlIT.java index 685f465970b..78c29433439 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlIT.java @@ -42,9 +42,10 @@ import org.apache.gravitino.authorization.User; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; +import org.apache.gravitino.exceptions.IllegalMetadataObjectException; import org.apache.gravitino.exceptions.IllegalPrivilegeException; +import org.apache.gravitino.exceptions.IllegalRoleException; import org.apache.gravitino.exceptions.NoSuchGroupException; -import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; import org.apache.gravitino.exceptions.NoSuchRoleException; import org.apache.gravitino.exceptions.NoSuchUserException; import org.apache.gravitino.exceptions.UserAlreadyExistsException; @@ -214,7 +215,7 @@ void testManageRoles() { "not-existed", Lists.newArrayList(Privileges.UseCatalog.allow())); Assertions.assertThrows( - NoSuchMetadataObjectException.class, + IllegalMetadataObjectException.class, () -> metalake.createRole("not-existed", properties, Lists.newArrayList(catalogObject))); // Create a role with duplicated securable objects @@ -359,12 +360,12 @@ void testManageUserPermissions() { // Grant a not-existed role Assertions.assertThrows( - NoSuchRoleException.class, + IllegalRoleException.class, () -> metalake.grantRolesToUser(Lists.newArrayList("not-existed"), username)); // Revoke a not-existed role Assertions.assertThrows( - NoSuchRoleException.class, + IllegalRoleException.class, () -> metalake.revokeRolesFromUser(Lists.newArrayList("not-existed"), username)); // Grant to a not-existed user @@ -414,12 +415,12 @@ void testManageGroupPermissions() { // Grant a not-existed role Assertions.assertThrows( - NoSuchRoleException.class, + IllegalRoleException.class, () -> metalake.grantRolesToGroup(Lists.newArrayList("not-existed"), groupName)); // Revoke a not-existed role Assertions.assertThrows( - NoSuchRoleException.class, + IllegalRoleException.class, () -> metalake.revokeRolesFromGroup(Lists.newArrayList("not-existed"), groupName)); // Grant to a not-existed group diff --git a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java new file mode 100644 index 00000000000..255e54fbf3d --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import java.util.Map; + +/** + * Helper class to generate specific credential properties for different table format and engine. + */ +public class CredentialPropertyUtils { + /** + * Transforms a specific credential into a map of Iceberg properties. + * + * @param credential the credential to be transformed into Iceberg properties + * @return a map of Iceberg properties derived from the credential + */ + public static Map toIcebergProperties(Credential credential) { + // todo: transform specific credential to iceberg properties + return credential.toProperties(); + } +} diff --git a/core/src/main/java/org/apache/gravitino/authorization/AccessControlDispatcher.java b/core/src/main/java/org/apache/gravitino/authorization/AccessControlDispatcher.java index 73004280b7e..f5625d9d69e 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/AccessControlDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/authorization/AccessControlDispatcher.java @@ -22,6 +22,7 @@ import java.util.Map; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; +import org.apache.gravitino.exceptions.IllegalRoleException; import org.apache.gravitino.exceptions.NoSuchGroupException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; @@ -155,12 +156,12 @@ Group getGroup(String metalake, String group) * @param roles The names of the Role. * @return The User after granted. * @throws NoSuchUserException If the User with the given name does not exist. - * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws IllegalRoleException If the Role with the given name does not exist. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If granting roles to a user encounters storage issues. */ User grantRolesToUser(String metalake, List roles, String user) - throws NoSuchUserException, NoSuchRoleException, NoSuchMetalakeException; + throws NoSuchUserException, IllegalRoleException, NoSuchMetalakeException; /** * Grant roles to a group. @@ -170,12 +171,12 @@ User grantRolesToUser(String metalake, List roles, String user) * @param roles The names of the Role. * @return The Group after granted. * @throws NoSuchGroupException If the Group with the given name does not exist. - * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws IllegalRoleException If the Role with the given name does not exist. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If granting roles to a group encounters storage issues. */ Group grantRolesToGroup(String metalake, List roles, String group) - throws NoSuchGroupException, NoSuchRoleException, NoSuchMetalakeException; + throws NoSuchGroupException, IllegalRoleException, NoSuchMetalakeException; /** * Revoke roles from a group. @@ -185,12 +186,12 @@ Group grantRolesToGroup(String metalake, List roles, String group) * @param roles The name of the Role. * @return The Group after revoked. * @throws NoSuchGroupException If the Group with the given name does not exist. - * @throws NoSuchRoleException If the Role with the given name does not exist. + * @throws IllegalRoleException If the Role with the given name does not exist. * @throws NoSuchMetalakeException If the Metalake with the given name does not exist. * @throws RuntimeException If revoking roles from a group encounters storage issues. */ Group revokeRolesFromGroup(String metalake, List roles, String group) - throws NoSuchGroupException, NoSuchRoleException, NoSuchMetalakeException; + throws NoSuchGroupException, IllegalRoleException, NoSuchMetalakeException; /** * Revoke roles from a user. @@ -205,7 +206,7 @@ Group revokeRolesFromGroup(String metalake, List roles, String group) * @throws RuntimeException If revoking roles from a user encounters storage issues. */ User revokeRolesFromUser(String metalake, List roles, String user) - throws NoSuchUserException, NoSuchRoleException, NoSuchMetalakeException; + throws NoSuchUserException, IllegalRoleException, NoSuchMetalakeException; /** * Judges whether the user is the service admin. diff --git a/core/src/main/java/org/apache/gravitino/authorization/AccessControlManager.java b/core/src/main/java/org/apache/gravitino/authorization/AccessControlManager.java index c9adf314a87..798285806f5 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/AccessControlManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/AccessControlManager.java @@ -25,6 +25,7 @@ import org.apache.gravitino.EntityStore; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; +import org.apache.gravitino.exceptions.IllegalRoleException; import org.apache.gravitino.exceptions.NoSuchGroupException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; @@ -107,25 +108,25 @@ public String[] listGroupNames(String metalake) throws NoSuchMetalakeException { @Override public User grantRolesToUser(String metalake, List roles, String user) - throws NoSuchUserException, NoSuchRoleException, NoSuchMetalakeException { + throws NoSuchUserException, IllegalRoleException, NoSuchMetalakeException { return permissionManager.grantRolesToUser(metalake, roles, user); } @Override public Group grantRolesToGroup(String metalake, List roles, String group) - throws NoSuchGroupException, NoSuchRoleException, NoSuchMetalakeException { + throws NoSuchGroupException, IllegalRoleException, NoSuchMetalakeException { return permissionManager.grantRolesToGroup(metalake, roles, group); } @Override public Group revokeRolesFromGroup(String metalake, List roles, String group) - throws NoSuchGroupException, NoSuchRoleException, NoSuchMetalakeException { + throws NoSuchGroupException, IllegalRoleException, NoSuchMetalakeException { return permissionManager.revokeRolesFromGroup(metalake, roles, group); } @Override public User revokeRolesFromUser(String metalake, List roles, String user) - throws NoSuchUserException, NoSuchRoleException, NoSuchMetalakeException { + throws NoSuchUserException, IllegalRoleException, NoSuchMetalakeException { return permissionManager.revokeRolesFromUser(metalake, roles, user); } diff --git a/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java b/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java index 056b18f4045..02c240f30a9 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/PermissionManager.java @@ -33,6 +33,7 @@ import org.apache.gravitino.Entity; import org.apache.gravitino.EntityStore; import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.exceptions.IllegalRoleException; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchGroupException; import org.apache.gravitino.exceptions.NoSuchRoleException; @@ -129,6 +130,8 @@ User grantRolesToUser(String metalake, List roles, String user) { } catch (NoSuchEntityException nse) { LOG.warn("Failed to grant, user {} does not exist in the metalake {}", user, metalake, nse); throw new NoSuchUserException(USER_DOES_NOT_EXIST_MSG, user, metalake); + } catch (NoSuchRoleException nsr) { + throw new IllegalRoleException(nsr); } catch (IOException ioe) { LOG.error( "Failed to grant role {} to user {} in the metalake {} due to storage issues", @@ -208,6 +211,8 @@ Group grantRolesToGroup(String metalake, List roles, String group) { } catch (NoSuchEntityException nse) { LOG.warn("Failed to grant, group {} does not exist in the metalake {}", group, metalake, nse); throw new NoSuchGroupException(GROUP_DOES_NOT_EXIST_MSG, group, metalake); + } catch (NoSuchRoleException nsr) { + throw new IllegalRoleException(nsr); } catch (IOException ioe) { LOG.error( "Failed to grant role {} to group {} in the metalake {} due to storage issues", @@ -288,6 +293,8 @@ Group revokeRolesFromGroup(String metalake, List roles, String group) { LOG.warn( "Failed to revoke, group {} does not exist in the metalake {}", group, metalake, nse); throw new NoSuchGroupException(GROUP_DOES_NOT_EXIST_MSG, group, metalake); + } catch (NoSuchRoleException nsr) { + throw new IllegalRoleException(nsr); } catch (IOException ioe) { LOG.error( "Failed to revoke role {} from group {} in the metalake {} due to storage issues", @@ -366,6 +373,8 @@ User revokeRolesFromUser(String metalake, List roles, String user) { } catch (NoSuchEntityException nse) { LOG.warn("Failed to revoke, user {} does not exist in the metalake {}", user, metalake, nse); throw new NoSuchUserException(USER_DOES_NOT_EXIST_MSG, user, metalake); + } catch (NoSuchRoleException nsr) { + throw new IllegalRoleException(nsr); } catch (IOException ioe) { LOG.error( "Failed to revoke role {} from user {} in the metalake {} due to storage issues", diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialProviderManager.java b/core/src/main/java/org/apache/gravitino/credential/CredentialProviderManager.java new file mode 100644 index 00000000000..b583bedcfdf --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialProviderManager.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.base.Preconditions; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CredentialProviderManager { + + private static final Logger LOG = LoggerFactory.getLogger(CredentialProviderManager.class); + private Map credentialProviders; + + public CredentialProviderManager() { + this.credentialProviders = new ConcurrentHashMap<>(); + } + + public void registerCredentialProvider( + String catalogName, CredentialProvider credentialProvider) { + CredentialProvider current = credentialProviders.putIfAbsent(catalogName, credentialProvider); + Preconditions.checkState( + !credentialProvider.equals(current), + String.format( + "Should not register multiple times to CredentialProviderManager, catalog: %s, " + + "credential provider: %s", + catalogName, credentialProvider.credentialType())); + LOG.info( + "Register catalog:%s credential provider:%s to CredentialProviderManager", + catalogName, credentialProvider.credentialType()); + } + + public void unregisterCredentialProvider(String catalogName) { + CredentialProvider credentialProvider = credentialProviders.remove(catalogName); + // Not all catalog has credential provider + if (credentialProvider != null) { + LOG.info( + "Unregister catalog:{} credential provider:{} to CredentialProviderManager", + catalogName, + credentialProvider.credentialType()); + try { + credentialProvider.close(); + } catch (IOException e) { + LOG.warn("Close credential provider failed", e); + } + } + } + + @Nullable + public CredentialProvider getCredentialProvider(String catalogName) { + return credentialProviders.get(catalogName); + } +} diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java b/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java new file mode 100644 index 00000000000..ad81953ac61 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.google.common.collect.ImmutableSet; +import org.apache.gravitino.utils.PrincipalUtils; + +public class CredentialUtils { + public static Credential vendCredential(CredentialProvider credentialProvider, String path) { + PathBasedCredentialContext pathBasedCredentialContext = + new PathBasedCredentialContext( + PrincipalUtils.getCurrentUserName(), ImmutableSet.of(path), ImmutableSet.of()); + return credentialProvider.getCredential(pathBasedCredentialContext); + } +} diff --git a/core/src/main/java/org/apache/gravitino/hook/AccessControlHookDispatcher.java b/core/src/main/java/org/apache/gravitino/hook/AccessControlHookDispatcher.java index 125df0b2e18..f5f5a27648e 100644 --- a/core/src/main/java/org/apache/gravitino/hook/AccessControlHookDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/hook/AccessControlHookDispatcher.java @@ -33,6 +33,7 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.User; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; +import org.apache.gravitino.exceptions.IllegalRoleException; import org.apache.gravitino.exceptions.NoSuchGroupException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; @@ -111,25 +112,25 @@ public String[] listGroupNames(String metalake) throws NoSuchMetalakeException { @Override public User grantRolesToUser(String metalake, List roles, String user) - throws NoSuchUserException, NoSuchRoleException, NoSuchMetalakeException { + throws NoSuchUserException, IllegalRoleException, NoSuchMetalakeException { return dispatcher.grantRolesToUser(metalake, roles, user); } @Override public Group grantRolesToGroup(String metalake, List roles, String group) - throws NoSuchGroupException, NoSuchRoleException, NoSuchMetalakeException { + throws NoSuchGroupException, IllegalRoleException, NoSuchMetalakeException { return dispatcher.grantRolesToGroup(metalake, roles, group); } @Override public Group revokeRolesFromGroup(String metalake, List roles, String group) - throws NoSuchGroupException, NoSuchRoleException, NoSuchMetalakeException { + throws NoSuchGroupException, IllegalRoleException, NoSuchMetalakeException { return dispatcher.revokeRolesFromGroup(metalake, roles, group); } @Override public User revokeRolesFromUser(String metalake, List roles, String user) - throws NoSuchUserException, NoSuchRoleException, NoSuchMetalakeException { + throws NoSuchUserException, IllegalRoleException, NoSuchMetalakeException { return dispatcher.revokeRolesFromUser(metalake, roles, user); } diff --git a/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java b/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java index e7e792536d2..9387fef0d5f 100644 --- a/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java +++ b/core/src/test/java/org/apache/gravitino/authorization/TestAccessControlManagerForPermissions.java @@ -40,6 +40,7 @@ import org.apache.gravitino.catalog.CatalogManager; import org.apache.gravitino.connector.BaseCatalog; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; +import org.apache.gravitino.exceptions.IllegalRoleException; import org.apache.gravitino.exceptions.NoSuchGroupException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchRoleException; @@ -215,9 +216,9 @@ public void testGrantRoleToUser() { NoSuchMetalakeException.class, () -> accessControlManager.grantRolesToUser(notExist, ROLE, USER)); - // Throw NoSuchRoleException + // Throw IllegalRoleException Assertions.assertThrows( - NoSuchRoleException.class, + IllegalRoleException.class, () -> accessControlManager.grantRolesToUser(METALAKE, Lists.newArrayList(notExist), USER)); // Throw NoSuchUserException @@ -249,9 +250,9 @@ public void testRevokeRoleFromUser() { NoSuchMetalakeException.class, () -> accessControlManager.revokeRolesFromUser(notExist, ROLE, USER)); - // Throw NoSuchRoleException + // Throw IllegalRoleException Assertions.assertThrows( - NoSuchRoleException.class, + IllegalRoleException.class, () -> accessControlManager.revokeRolesFromUser(METALAKE, Lists.newArrayList(notExist), USER)); @@ -293,9 +294,9 @@ public void testGrantRoleToGroup() { NoSuchMetalakeException.class, () -> accessControlManager.grantRolesToGroup(notExist, ROLE, GROUP)); - // Throw NoSuchRoleException + // Throw IllegalRoleException Assertions.assertThrows( - NoSuchRoleException.class, + IllegalRoleException.class, () -> accessControlManager.grantRolesToGroup(METALAKE, Lists.newArrayList(notExist), GROUP)); @@ -328,9 +329,9 @@ public void testRevokeRoleFormGroup() { NoSuchMetalakeException.class, () -> accessControlManager.revokeRolesFromGroup(notExist, ROLE, GROUP)); - // Throw NoSuchRoleException + // Throw IllegalRoleException Assertions.assertThrows( - NoSuchRoleException.class, + IllegalRoleException.class, () -> accessControlManager.revokeRolesFromGroup( METALAKE, Lists.newArrayList(notExist), GROUP)); @@ -375,7 +376,7 @@ public void testGrantPrivilegeToRole() { Assertions.assertEquals(2, objects.size()); - // Throw NoSuchRoleException + // Throw IllegalRoleException Assertions.assertThrows( NoSuchRoleException.class, () -> diff --git a/docs/open-api/permissions.yaml b/docs/open-api/permissions.yaml index 1a19a9e2be3..0da45d9ca5b 100644 --- a/docs/open-api/permissions.yaml +++ b/docs/open-api/permissions.yaml @@ -49,6 +49,16 @@ paths: UserResponse: $ref: "./users.yaml#/components/examples/UserResponse" + "400": + description: Parameter is invalid - The specified role is invalid in the metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + IllegalRoleException: + $ref: "#/components/examples/IllegalRoleException" + "404": description: Not Found - The specified user or role does not exist in the specified metalake content: @@ -60,8 +70,6 @@ paths: $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" NoSuchUserException: $ref: "./users.yaml#/components/examples/NoSuchUserException" - NoSuchRoleException: - $ref: "./roles.yaml#/components/examples/NoSuchRoleException" "5xx": $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" @@ -96,6 +104,16 @@ paths: UserResponse: $ref: "./users.yaml#/components/examples/UserResponse" + "400": + description: Parameter is invalid - The specified role is invalid in the metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + IllegalRoleException: + $ref: "#/components/examples/IllegalRoleException" + "404": description: Not Found - The specified user or role does not exist in the specified metalake content: @@ -107,8 +125,6 @@ paths: $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" NoSuchUserException: $ref: "./users.yaml#/components/examples/NoSuchUserException" - NoSuchRoleException: - $ref: "./roles.yaml#/components/examples/NoSuchRoleException" "5xx": $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" @@ -143,6 +159,16 @@ paths: GroupResponse: $ref: "./groups.yaml#/components/examples/GroupResponse" + "400": + description: Parameter is invalid - The specified role is invalid in the metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + IllegalRoleException: + $ref: "#/components/examples/IllegalRoleException" + "404": description: Not Found - The specified group or role does not exist in the specified metalake content: @@ -154,8 +180,6 @@ paths: $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" NoSuchGroupException: $ref: "./groups.yaml#/components/examples/NoSuchGroupException" - NoSuchRoleException: - $ref: "./roles.yaml#/components/examples/NoSuchRoleException" "5xx": $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" @@ -190,6 +214,16 @@ paths: GroupResponse: $ref: "./groups.yaml#/components/examples/GroupResponse" + "400": + description: Parameter is invalid - The specified role is invalid in the metalake + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + IllegalRoleException: + $ref: "#/components/examples/IllegalRoleException" + "404": description: Not Found - The specified group or role does not exist in the specified metalake content: @@ -201,8 +235,6 @@ paths: $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" NoSuchGroupException: $ref: "./groups.yaml#/components/examples/NoSuchGroupException" - NoSuchRoleException: - $ref: "./roles.yaml#/components/examples/NoSuchRoleException" "5xx": $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" @@ -239,6 +271,16 @@ paths: GroupResponse: $ref: "./roles.yaml#/components/examples/RoleResponse" + "400": + description: Parameter is invalid - The specified privilege is invalid + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + IllegalPrivilegeException: + $ref: "#/components/examples/IllegalPrivilegeException" + "404": description: Not Found - The specified medata object or role does not exist in the specified metalake content: @@ -288,6 +330,16 @@ paths: GroupResponse: $ref: "./roles.yaml#/components/examples/RoleResponse" + "400": + description: Parameter is invalid - The specified privilege is invalid + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + IllegalPrivilegeException: + $ref: "#/components/examples/IllegalPrivilegeException" + "404": description: Not Found - The specified medata object or role does not exist in the specified metalake content: @@ -381,4 +433,26 @@ components: "name": "SELECT_TABLE", "condition": "ALLOW" } ] + } + + IllegalRoleException: + value: { + "code": 1001, + "type": "IllegalRoleException", + "message": "Role role1 does not exist", + "stack": [ + "org.apache.gravitino.exceptions.IllegalRoleException: Role role1 does not exist", + "..." + ] + } + + IllegalPrivilegeException: + value: { + "code": 1001, + "type": "IllegalPrivilegeException", + "message": "Doesn't support duplicated privilege name SELECT_TABLE with different condition", + "stack": [ + "org.apache.gravitino.exceptions.IllegalPrivilegeException: Doesn't support duplicated privilege name SELECT_TABLE with different condition", + "..." + ] } \ No newline at end of file diff --git a/docs/open-api/roles.yaml b/docs/open-api/roles.yaml index 8bc452a2082..986d0fdc6f1 100644 --- a/docs/open-api/roles.yaml +++ b/docs/open-api/roles.yaml @@ -75,15 +75,15 @@ paths: RoleResponse: $ref: "#/components/examples/RoleResponse" - "404": - description: Not Found - The specified securable object does not exist in the specified metalake + "400": + description: Parameter is invalid - The specified securable object is invalid the specified metalake content: application/vnd.gravitino.v1+json: schema: $ref: "./openapi.yaml#/components/schemas/ErrorModel" examples: NoSuchMetadataObjectException: - $ref: "#/components/examples/NoSuchMetadataObjectException" + $ref: "#/components/examples/IllegalMetadataObjectException" "409": description: Conflict - The target role already exists in the specified metalake @@ -360,13 +360,24 @@ components: ] } + IllegalMetadataObjectException: + value: { + "code": 1001, + "type": "IllegalMetadataObjectException", + "message": "Metadata object does not exist", + "stack": [ + "org.apache.gravitino.exceptions.IllegalMetadataObjectException: Metadata object does not exist", + "..." + ] + } + NoSuchMetadataObjectException: value: { "code": 1003, "type": "NoSuchMetadataObjectException", "message": "Metadata object does not exist", "stack": [ - "org.apache.gravitino.exceptions.NoSuchUserException: Metadata object does not exist", + "org.apache.gravitino.exceptions.NoSuchMetadataObjectException: Metadata object does not exist", "..." ] } diff --git a/iceberg/iceberg-common/build.gradle.kts b/iceberg/iceberg-common/build.gradle.kts index 23b3d30db28..abc9a05a550 100644 --- a/iceberg/iceberg-common/build.gradle.kts +++ b/iceberg/iceberg-common/build.gradle.kts @@ -25,6 +25,7 @@ plugins { } dependencies { + implementation(project(":api")) implementation(project(":catalogs:catalog-common")) implementation(project(":core")) { exclude("*") diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java index fd7b52050c3..638b4172ce4 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java @@ -32,6 +32,7 @@ import org.apache.gravitino.config.ConfigBuilder; import org.apache.gravitino.config.ConfigConstants; import org.apache.gravitino.config.ConfigEntry; +import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.storage.OSSProperties; import org.apache.gravitino.storage.S3Properties; @@ -201,13 +202,13 @@ public class IcebergConfig extends Config implements OverwriteDefaultConfig { .longConf() .createWithDefault(3600000L); - public static final ConfigEntry ICEBERG_REST_CATALOG_PROVIDER = - new ConfigBuilder(IcebergConstants.ICEBERG_REST_CATALOG_PROVIDER) + public static final ConfigEntry ICEBERG_REST_CATALOG_CONFIG_PROVIDER = + new ConfigBuilder(IcebergConstants.ICEBERG_REST_CATALOG_CONFIG_PROVIDER) .doc( - "Catalog provider class name, you can develop a class that implements `IcebergCatalogWrapperProvider` and add the corresponding jar file to the Iceberg REST service classpath directory.") + "Catalog provider class name, you can develop a class that implements `IcebergCatalogConfigProvider` and add the corresponding jar file to the Iceberg REST service classpath directory.") .version(ConfigConstants.VERSION_0_7_0) .stringConf() - .createWithDefault("config-based-provider"); + .createWithDefault(IcebergConstants.STATIC_ICEBERG_CATALOG_CONFIG_PROVIDER_NAME); public static final ConfigEntry GRAVITINO_URI = new ConfigBuilder(IcebergConstants.GRAVITINO_URI) @@ -233,6 +234,13 @@ public class IcebergConfig extends Config implements OverwriteDefaultConfig { .toSequence() .createWithDefault(Collections.emptyList()); + public static final ConfigEntry CREDENTIAL_PROVIDER_TYPE = + new ConfigBuilder(CredentialConstants.CREDENTIAL_PROVIDER_TYPE) + .doc("The credential provider type for Iceberg") + .version(ConfigConstants.VERSION_0_7_0) + .stringConf() + .create(); + public String getJdbcDriver() { return get(JDBC_DRIVER); } diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapperProvider.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogConfigProvider.java similarity index 71% rename from iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapperProvider.java rename to iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogConfigProvider.java index 758aa46aa08..fc0d488a11d 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapperProvider.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogConfigProvider.java @@ -19,12 +19,14 @@ package org.apache.gravitino.iceberg.common.ops; import java.util.Map; +import java.util.Optional; +import org.apache.gravitino.iceberg.common.IcebergConfig; /** - * IcebergCatalogWrapperProvider is an interface defining how Iceberg REST catalog server gets - * Iceberg catalogs. + * {@code IcebergCatalogConfigProvider} is an interface defining how Iceberg REST catalog server + * gets Iceberg catalog configurations. */ -public interface IcebergCatalogWrapperProvider { +public interface IcebergCatalogConfigProvider { /** * @param properties The parameters for creating Provider which from configurations whose prefix @@ -33,8 +35,8 @@ public interface IcebergCatalogWrapperProvider { void initialize(Map properties); /** - * @param catalogName a param send by clients. - * @return the instance of IcebergCatalogWrapper. + * @param catalogName Iceberg catalog name. + * @return the configuration of Iceberg catalog. */ - IcebergCatalogWrapper getIcebergTableOps(String catalogName); + Optional getIcebergCatalogConfig(String catalogName); } diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java index 6ff4bf2ce03..95e82aa2275 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java @@ -22,7 +22,6 @@ import com.google.common.collect.ImmutableSet; import java.sql.Driver; import java.sql.DriverManager; -import java.util.Collections; import java.util.Locale; import java.util.Map; import java.util.Optional; @@ -104,10 +103,6 @@ public IcebergCatalogWrapper(IcebergConfig icebergConfig) { this.catalogPropertiesMap = icebergConfig.getIcebergCatalogProperties(); } - public IcebergCatalogWrapper() { - this(new IcebergConfig(Collections.emptyMap())); - } - private void validateNamespace(Optional namespace) { namespace.ifPresent( n -> @@ -160,7 +155,7 @@ public LoadTableResponse registerTable(Namespace namespace, RegisterTableRequest /** * Reload hadoop configuration, this is useful when the hadoop configuration UserGroupInformation * is shared by multiple threads. UserGroupInformation#authenticationMethod was first initialized - * in KerberosClient, however, when switching to iceberg-rest thead, + * in KerberosClient, however, when switching to iceberg-rest thread, * UserGroupInformation#authenticationMethod will be reset to the default value; we need to * reinitialize it again. */ @@ -271,7 +266,7 @@ public void close() throws Exception { private void closeMySQLCatalogResource() { try { // Close thread AbandonedConnectionCleanupThread if we are using `com.mysql.cj.jdbc.Driver`, - // for driver `com.mysql.jdbc.Driver` (deprecated), the daemon thead maybe not this one. + // for driver `com.mysql.jdbc.Driver` (deprecated), the daemon thread maybe not this one. Class.forName("com.mysql.cj.jdbc.AbandonedConnectionCleanupThread") .getMethod("uncheckedShutdown") .invoke(null); diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/GravitinoBasedIcebergCatalogWrapperProvider.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/DynamicIcebergCatalogConfigProvider.java similarity index 81% rename from iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/GravitinoBasedIcebergCatalogWrapperProvider.java rename to iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/DynamicIcebergCatalogConfigProvider.java index a38fd9cf302..4965f4bc132 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/GravitinoBasedIcebergCatalogWrapperProvider.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/DynamicIcebergCatalogConfigProvider.java @@ -21,14 +21,15 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import java.util.Map; +import java.util.Optional; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergPropertiesUtils; import org.apache.gravitino.client.GravitinoAdminClient; +import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.iceberg.common.IcebergConfig; -import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; -import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapperProvider; +import org.apache.gravitino.iceberg.common.ops.IcebergCatalogConfigProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -39,13 +40,10 @@ * *

The catalogName is iceberg_catalog */ -public class GravitinoBasedIcebergCatalogWrapperProvider - implements IcebergCatalogWrapperProvider, AutoCloseable { +public class DynamicIcebergCatalogConfigProvider + implements IcebergCatalogConfigProvider, AutoCloseable { public static final Logger LOG = - LoggerFactory.getLogger(GravitinoBasedIcebergCatalogWrapperProvider.class); - - public static final String GRAVITINO_BASE_ICEBERG_TABLE_OPS_PROVIDER_NAME = - "gravitino-based-provider"; + LoggerFactory.getLogger(DynamicIcebergCatalogConfigProvider.class); private String gravitinoMetalake; @@ -66,14 +64,19 @@ public void initialize(Map properties) { } @Override - public IcebergCatalogWrapper getIcebergTableOps(String catalogName) { + public Optional getIcebergCatalogConfig(String catalogName) { Preconditions.checkArgument( StringUtils.isNotBlank(catalogName), "blank catalogName is illegal"); Preconditions.checkArgument( !IcebergConstants.GRAVITINO_DEFAULT_CATALOG.equals(catalogName), IcebergConstants.GRAVITINO_DEFAULT_CATALOG + " is illegal in gravitino-based-provider"); - Catalog catalog = client.loadMetalake(gravitinoMetalake).loadCatalog(catalogName); + Catalog catalog; + try { + catalog = client.loadMetalake(gravitinoMetalake).loadCatalog(catalogName); + } catch (NoSuchCatalogException e) { + return Optional.empty(); + } Preconditions.checkArgument( "lakehouse-iceberg".equals(catalog.provider()), @@ -81,7 +84,7 @@ public IcebergCatalogWrapper getIcebergTableOps(String catalogName) { Map properties = IcebergPropertiesUtils.toIcebergCatalogProperties(catalog.properties()); - return new IcebergCatalogWrapper(new IcebergConfig(properties)); + return Optional.of(new IcebergConfig(properties)); } @VisibleForTesting diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/ConfigBasedIcebergCatalogWrapperProvider.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/StaticIcebergCatalogConfigProvider.java similarity index 78% rename from iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/ConfigBasedIcebergCatalogWrapperProvider.java rename to iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/StaticIcebergCatalogConfigProvider.java index 522bca39fe3..aa7f1032134 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/ConfigBasedIcebergCatalogWrapperProvider.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/provider/StaticIcebergCatalogConfigProvider.java @@ -24,8 +24,7 @@ import java.util.stream.Collectors; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.iceberg.common.IcebergConfig; -import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; -import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapperProvider; +import org.apache.gravitino.iceberg.common.ops.IcebergCatalogConfigProvider; import org.apache.gravitino.utils.MapUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,11 +39,9 @@ * gravitino.iceberg-rest.catalog.hive_proxy.catalog-backend = hive * gravitino.iceberg-rest.catalog.hive_proxy.uri = thrift://{host}:{port} ... */ -public class ConfigBasedIcebergCatalogWrapperProvider implements IcebergCatalogWrapperProvider { +public class StaticIcebergCatalogConfigProvider implements IcebergCatalogConfigProvider { public static final Logger LOG = - LoggerFactory.getLogger(ConfigBasedIcebergCatalogWrapperProvider.class); - - public static final String CONFIG_BASE_ICEBERG_TABLE_OPS_PROVIDER_NAME = "config-based-provider"; + LoggerFactory.getLogger(StaticIcebergCatalogConfigProvider.class); @VisibleForTesting Map catalogConfigs; @@ -68,14 +65,8 @@ public void initialize(Map properties) { } @Override - public IcebergCatalogWrapper getIcebergTableOps(String catalogName) { - IcebergConfig icebergConfig = this.catalogConfigs.get(catalogName); - if (icebergConfig == null) { - String errorMsg = String.format("%s can not match any catalog", catalogName); - LOG.warn(errorMsg); - throw new RuntimeException(errorMsg); - } - return new IcebergCatalogWrapper(icebergConfig); + public Optional getIcebergCatalogConfig(String catalogName) { + return Optional.ofNullable(catalogConfigs.get(catalogName)); } private Optional getCatalogName(String catalogConfigKey) { diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java index 17342acf71f..823f42ddb16 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java @@ -21,41 +21,48 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Scheduler; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.credential.CredentialProvider; +import org.apache.gravitino.credential.CredentialProviderFactory; +import org.apache.gravitino.credential.CredentialProviderManager; import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.iceberg.common.ops.IcebergCatalogConfigProvider; import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; -import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapperProvider; -import org.apache.gravitino.iceberg.provider.ConfigBasedIcebergCatalogWrapperProvider; -import org.apache.gravitino.iceberg.provider.GravitinoBasedIcebergCatalogWrapperProvider; +import org.apache.gravitino.iceberg.provider.DynamicIcebergCatalogConfigProvider; +import org.apache.gravitino.iceberg.provider.StaticIcebergCatalogConfigProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class IcebergCatalogWrapperManager implements AutoCloseable { public static final Logger LOG = LoggerFactory.getLogger(IcebergCatalogWrapperManager.class); - private static final ImmutableMap ICEBERG_TABLE_OPS_PROVIDER_NAMES = + private static final ImmutableMap ICEBERG_CATALOG_CONFIG_PROVIDER_NAMES = ImmutableMap.of( - ConfigBasedIcebergCatalogWrapperProvider.CONFIG_BASE_ICEBERG_TABLE_OPS_PROVIDER_NAME, - ConfigBasedIcebergCatalogWrapperProvider.class.getCanonicalName(), - GravitinoBasedIcebergCatalogWrapperProvider - .GRAVITINO_BASE_ICEBERG_TABLE_OPS_PROVIDER_NAME, - GravitinoBasedIcebergCatalogWrapperProvider.class.getCanonicalName()); + IcebergConstants.STATIC_ICEBERG_CATALOG_CONFIG_PROVIDER_NAME, + StaticIcebergCatalogConfigProvider.class.getCanonicalName(), + IcebergConstants.DYNAMIC_ICEBERG_CATALOG_CONFIG_PROVIDER_NAME, + DynamicIcebergCatalogConfigProvider.class.getCanonicalName()); - private final Cache icebergTableOpsCache; + private final Cache icebergCatalogWrapperCache; - private final IcebergCatalogWrapperProvider provider; + private final IcebergCatalogConfigProvider provider; + + private CredentialProviderManager credentialProviderManager; public IcebergCatalogWrapperManager(Map properties) { - this.provider = createProvider(properties); + this.credentialProviderManager = new CredentialProviderManager(); + this.provider = createIcebergCatalogConfigProvider(properties); this.provider.initialize(properties); - this.icebergTableOpsCache = + this.icebergCatalogWrapperCache = Caffeine.newBuilder() .expireAfterWrite( (new IcebergConfig(properties)) @@ -63,8 +70,10 @@ public IcebergCatalogWrapperManager(Map properties) { TimeUnit.MILLISECONDS) .removalListener( (k, v, c) -> { - LOG.info("Remove IcebergCatalogWrapper cache {}.", k); - closeIcebergTableOps((IcebergCatalogWrapper) v); + String catalogName = (String) k; + LOG.info("Remove IcebergCatalogWrapper cache {}.", catalogName); + closeIcebergCatalogWrapper((IcebergCatalogWrapper) v); + credentialProviderManager.unregisterCredentialProvider(catalogName); }) .scheduler( Scheduler.forScheduledExecutorService( @@ -72,7 +81,7 @@ public IcebergCatalogWrapperManager(Map properties) { 1, new ThreadFactoryBuilder() .setDaemon(true) - .setNameFormat("table-ops-cleaner-%d") + .setNameFormat("iceberg-catalog-wrapper-cleaner-%d") .build()))) .build(); } @@ -85,13 +94,40 @@ public IcebergCatalogWrapperManager(Map properties) { public IcebergCatalogWrapper getOps(String rawPrefix) { String catalogName = getCatalogName(rawPrefix); IcebergCatalogWrapper tableOps = - icebergTableOpsCache.get(catalogName, k -> provider.getIcebergTableOps(catalogName)); + icebergCatalogWrapperCache.get(catalogName, k -> createCatalogWrapper(catalogName)); // Reload conf to reset UserGroupInformation or icebergTableOps will always use // Simple auth. tableOps.reloadHadoopConf(); return tableOps; } + public CredentialProvider getCredentialProvider(String prefix) { + String catalogName = getCatalogName(prefix); + return credentialProviderManager.getCredentialProvider(catalogName); + } + + @VisibleForTesting + protected IcebergCatalogWrapper createIcebergCatalogWrapper(IcebergConfig icebergConfig) { + return new IcebergCatalogWrapper(icebergConfig); + } + + private IcebergCatalogWrapper createCatalogWrapper(String catalogName) { + Optional icebergConfig = provider.getIcebergCatalogConfig(catalogName); + if (!icebergConfig.isPresent()) { + throw new RuntimeException("Couldn't find Iceberg configuration for " + catalogName); + } + + IcebergConfig config = icebergConfig.get(); + String credentialProviderType = config.get(IcebergConfig.CREDENTIAL_PROVIDER_TYPE); + if (StringUtils.isNotBlank(credentialProviderType)) { + CredentialProvider credentialProvider = + CredentialProviderFactory.create(credentialProviderType, config.getAllConfig()); + credentialProviderManager.registerCredentialProvider(catalogName, credentialProvider); + } + + return createIcebergCatalogWrapper(icebergConfig.get()); + } + private String getCatalogName(String rawPrefix) { String prefix = shelling(rawPrefix); Preconditions.checkArgument( @@ -103,14 +139,16 @@ private String getCatalogName(String rawPrefix) { return prefix; } - private IcebergCatalogWrapperProvider createProvider(Map properties) { + private IcebergCatalogConfigProvider createIcebergCatalogConfigProvider( + Map properties) { String providerName = - (new IcebergConfig(properties)).get(IcebergConfig.ICEBERG_REST_CATALOG_PROVIDER); - String className = ICEBERG_TABLE_OPS_PROVIDER_NAMES.getOrDefault(providerName, providerName); + (new IcebergConfig(properties)).get(IcebergConfig.ICEBERG_REST_CATALOG_CONFIG_PROVIDER); + String className = + ICEBERG_CATALOG_CONFIG_PROVIDER_NAMES.getOrDefault(providerName, providerName); LOG.info("Load Iceberg catalog provider: {}.", className); try { Class providerClz = Class.forName(className); - return (IcebergCatalogWrapperProvider) providerClz.getDeclaredConstructor().newInstance(); + return (IcebergCatalogConfigProvider) providerClz.getDeclaredConstructor().newInstance(); } catch (Exception e) { throw new RuntimeException(e); } @@ -127,17 +165,17 @@ private String shelling(String rawPrefix) { } } - private void closeIcebergTableOps(IcebergCatalogWrapper ops) { + private void closeIcebergCatalogWrapper(IcebergCatalogWrapper catalogWrapper) { try { - ops.close(); + catalogWrapper.close(); } catch (Exception ex) { - LOG.warn("Close Iceberg table ops fail: {}, {}", ops, ex); + LOG.warn("Close Iceberg table catalog wrapper fail: {}, {}", catalogWrapper, ex); } } @Override public void close() throws Exception { - icebergTableOpsCache.invalidateAll(); + icebergCatalogWrapperCache.invalidateAll(); if (provider instanceof AutoCloseable) { ((AutoCloseable) provider).close(); } diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java index 0c383e52063..33023343ef3 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java @@ -22,6 +22,8 @@ import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import java.util.Map; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; @@ -29,6 +31,8 @@ import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.HEAD; +import javax.ws.rs.HeaderParam; +import javax.ws.rs.NotSupportedException; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -37,16 +41,24 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.credential.CredentialPropertyUtils; +import org.apache.gravitino.credential.CredentialProvider; +import org.apache.gravitino.credential.CredentialUtils; import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager; import org.apache.gravitino.iceberg.service.IcebergObjectMapper; import org.apache.gravitino.iceberg.service.IcebergRestUtils; import org.apache.gravitino.iceberg.service.metrics.IcebergMetricsManager; import org.apache.gravitino.metrics.MetricNames; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.ServiceUnavailableException; import org.apache.iceberg.rest.RESTUtil; import org.apache.iceberg.rest.requests.CreateTableRequest; import org.apache.iceberg.rest.requests.ReportMetricsRequest; import org.apache.iceberg.rest.requests.UpdateTableRequest; +import org.apache.iceberg.rest.responses.LoadTableResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,6 +69,9 @@ public class IcebergTableOperations { private static final Logger LOG = LoggerFactory.getLogger(IcebergTableOperations.class); + @VisibleForTesting + public static final String X_ICEBERG_ACCESS_DELEGATION = "X-Iceberg-Access-Delegation"; + private IcebergCatalogWrapperManager icebergCatalogWrapperManager; private IcebergMetricsManager icebergMetricsManager; @@ -92,15 +107,24 @@ public Response listTable( public Response createTable( @PathParam("prefix") String prefix, @PathParam("namespace") String namespace, - CreateTableRequest createTableRequest) { + CreateTableRequest createTableRequest, + @HeaderParam(X_ICEBERG_ACCESS_DELEGATION) String accessDelegation) { + boolean isCredentialVending = isCredentialVending(accessDelegation); LOG.info( - "Create Iceberg table, namespace: {}, create table request: {}", + "Create Iceberg table, namespace: {}, create table request: {}, accessDelegation: {}, isCredentialVending: {}", namespace, - createTableRequest); - return IcebergRestUtils.ok( + createTableRequest, + accessDelegation, + isCredentialVending); + LoadTableResponse loadTableResponse = icebergCatalogWrapperManager .getOps(prefix) - .createTable(RESTUtil.decodeNamespace(namespace), createTableRequest)); + .createTable(RESTUtil.decodeNamespace(namespace), createTableRequest); + if (isCredentialVending) { + return IcebergRestUtils.ok(injectCredentialConfig(prefix, loadTableResponse)); + } else { + return IcebergRestUtils.ok(loadTableResponse); + } } @POST @@ -162,12 +186,26 @@ public Response loadTable( @PathParam("prefix") String prefix, @PathParam("namespace") String namespace, @PathParam("table") String table, - @DefaultValue("all") @QueryParam("snapshots") String snapshots) { + @DefaultValue("all") @QueryParam("snapshots") String snapshots, + @HeaderParam(X_ICEBERG_ACCESS_DELEGATION) String accessDelegation) { + boolean isCredentialVending = isCredentialVending(accessDelegation); + LOG.info( + "Load iceberg table, namespace: {}, table: {}, access delegation: {}, " + + "credential vending: {}", + namespace, + table, + accessDelegation, + isCredentialVending); // todo support snapshots TableIdentifier tableIdentifier = TableIdentifier.of(RESTUtil.decodeNamespace(namespace), table); - return IcebergRestUtils.ok( - icebergCatalogWrapperManager.getOps(prefix).loadTable(tableIdentifier)); + LoadTableResponse loadTableResponse = + icebergCatalogWrapperManager.getOps(prefix).loadTable(tableIdentifier); + if (isCredentialVending) { + return IcebergRestUtils.ok(injectCredentialConfig(prefix, loadTableResponse)); + } else { + return IcebergRestUtils.ok(loadTableResponse); + } } @HEAD @@ -210,4 +248,49 @@ private String SerializeUpdateTableRequest(UpdateTableRequest updateTableRequest return updateTableRequest.toString(); } } + + private LoadTableResponse injectCredentialConfig( + String prefix, LoadTableResponse loadTableResponse) { + CredentialProvider credentialProvider = + icebergCatalogWrapperManager.getCredentialProvider(prefix); + if (credentialProvider == null) { + throw new NotSupportedException( + "Doesn't support credential vending, please add " + + CredentialConstants.CREDENTIAL_PROVIDER_TYPE + + " to the catalog configurations"); + } + Credential credential = + CredentialUtils.vendCredential( + credentialProvider, loadTableResponse.tableMetadata().location()); + if (credential == null) { + throw new ServiceUnavailableException( + "Couldn't generate credential for %s", credentialProvider.credentialType()); + } + Map credentialConfig = CredentialPropertyUtils.toIcebergProperties(credential); + return LoadTableResponse.builder() + .withTableMetadata(loadTableResponse.tableMetadata()) + .addAllConfig(loadTableResponse.config()) + .addAllConfig(credentialConfig) + .build(); + } + + private boolean isCredentialVending(String accessDelegation) { + if (StringUtils.isBlank(accessDelegation)) { + return false; + } + if ("vended-credentials".equalsIgnoreCase(accessDelegation)) { + return true; + } + if ("remote-signing".equalsIgnoreCase(accessDelegation)) { + throw new UnsupportedOperationException( + "Gravitino IcebergRESTServer doesn't support remote signing"); + } else { + throw new IllegalArgumentException( + X_ICEBERG_ACCESS_DELEGATION + + ": " + + accessDelegation + + " is illegal, Iceberg REST spec supports:[vended-credentials,remote-signing], " + + "Gravitino Iceberg REST server supports: vended-credentials"); + } + } } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestGravitinoBasedIcebergCatalogWrapperProvider.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestDynamicIcebergCatalogWrapperProvider.java similarity index 87% rename from iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestGravitinoBasedIcebergCatalogWrapperProvider.java rename to iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestDynamicIcebergCatalogWrapperProvider.java index 8acac4ffd6b..f9ffbb42747 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestGravitinoBasedIcebergCatalogWrapperProvider.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestDynamicIcebergCatalogWrapperProvider.java @@ -30,7 +30,7 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; -public class TestGravitinoBasedIcebergCatalogWrapperProvider { +public class TestDynamicIcebergCatalogWrapperProvider { @Test public void testValidIcebergTableOps() { String hiveCatalogName = "hive_backend"; @@ -71,14 +71,15 @@ public void testValidIcebergTableOps() { } }); - GravitinoBasedIcebergCatalogWrapperProvider provider = - new GravitinoBasedIcebergCatalogWrapperProvider(); + DynamicIcebergCatalogConfigProvider provider = new DynamicIcebergCatalogConfigProvider(); GravitinoAdminClient client = Mockito.mock(GravitinoAdminClient.class); Mockito.when(client.loadMetalake(Mockito.any())).thenReturn(gravitinoMetalake); provider.setClient(client); - IcebergCatalogWrapper hiveOps = provider.getIcebergTableOps(hiveCatalogName); - IcebergCatalogWrapper jdbcOps = provider.getIcebergTableOps(jdbcCatalogName); + IcebergCatalogWrapper hiveOps = + new IcebergCatalogWrapper(provider.getIcebergCatalogConfig(hiveCatalogName).get()); + IcebergCatalogWrapper jdbcOps = + new IcebergCatalogWrapper(provider.getIcebergCatalogConfig(jdbcCatalogName).get()); Assertions.assertEquals(hiveCatalogName, hiveOps.getCatalog().name()); Assertions.assertEquals(jdbcCatalogName, jdbcOps.getCatalog().name()); @@ -101,16 +102,15 @@ public void testInvalidIcebergTableOps() { GravitinoAdminClient client = Mockito.mock(GravitinoAdminClient.class); Mockito.when(client.loadMetalake(Mockito.any())).thenReturn(gravitinoMetalake); - GravitinoBasedIcebergCatalogWrapperProvider provider = - new GravitinoBasedIcebergCatalogWrapperProvider(); + DynamicIcebergCatalogConfigProvider provider = new DynamicIcebergCatalogConfigProvider(); provider.setClient(client); Assertions.assertThrowsExactly( - IllegalArgumentException.class, () -> provider.getIcebergTableOps(invalidCatalogName)); + IllegalArgumentException.class, () -> provider.getIcebergCatalogConfig(invalidCatalogName)); Assertions.assertThrowsExactly( - IllegalArgumentException.class, () -> provider.getIcebergTableOps("")); + IllegalArgumentException.class, () -> provider.getIcebergCatalogConfig("")); Assertions.assertThrowsExactly( IllegalArgumentException.class, - () -> provider.getIcebergTableOps(IcebergConstants.GRAVITINO_DEFAULT_CATALOG)); + () -> provider.getIcebergCatalogConfig(IcebergConstants.GRAVITINO_DEFAULT_CATALOG)); } } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestConfigBasedIcebergCatalogWrapperProvider.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestStaticIcebergCatalogWrapperProvider.java similarity index 86% rename from iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestConfigBasedIcebergCatalogWrapperProvider.java rename to iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestStaticIcebergCatalogWrapperProvider.java index 99e83f2e41d..69f5b5ad257 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestConfigBasedIcebergCatalogWrapperProvider.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/provider/TestStaticIcebergCatalogWrapperProvider.java @@ -20,6 +20,7 @@ import com.google.common.collect.Maps; import java.util.Map; +import java.util.Optional; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; @@ -31,7 +32,8 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -public class TestConfigBasedIcebergCatalogWrapperProvider { +public class TestStaticIcebergCatalogWrapperProvider { + @Test public void testValidIcebergTableOps() { String hiveCatalogName = "hive_backend"; @@ -58,16 +60,18 @@ public void testValidIcebergTableOps() { config.put("catalog-backend", "memory"); config.put("warehouse", "/tmp/"); - ConfigBasedIcebergCatalogWrapperProvider provider = - new ConfigBasedIcebergCatalogWrapperProvider(); + StaticIcebergCatalogConfigProvider provider = new StaticIcebergCatalogConfigProvider(); provider.initialize(config); IcebergConfig hiveIcebergConfig = provider.catalogConfigs.get(hiveCatalogName); IcebergConfig jdbcIcebergConfig = provider.catalogConfigs.get(jdbcCatalogName); IcebergConfig defaultIcebergConfig = provider.catalogConfigs.get(defaultCatalogName); - IcebergCatalogWrapper hiveOps = provider.getIcebergTableOps(hiveCatalogName); - IcebergCatalogWrapper jdbcOps = provider.getIcebergTableOps(jdbcCatalogName); - IcebergCatalogWrapper defaultOps = provider.getIcebergTableOps(defaultCatalogName); + IcebergCatalogWrapper hiveOps = + new IcebergCatalogWrapper(provider.getIcebergCatalogConfig(hiveCatalogName).get()); + IcebergCatalogWrapper jdbcOps = + new IcebergCatalogWrapper(provider.getIcebergCatalogConfig(jdbcCatalogName).get()); + IcebergCatalogWrapper defaultOps = + new IcebergCatalogWrapper(provider.getIcebergCatalogConfig(defaultCatalogName).get()); Assertions.assertEquals( hiveCatalogName, hiveIcebergConfig.get(IcebergConfig.CATALOG_BACKEND_NAME)); @@ -102,11 +106,10 @@ public void testValidIcebergTableOps() { @ParameterizedTest @ValueSource(strings = {"", "not_match"}) public void testInvalidIcebergTableOps(String catalogName) { - ConfigBasedIcebergCatalogWrapperProvider provider = - new ConfigBasedIcebergCatalogWrapperProvider(); + StaticIcebergCatalogConfigProvider provider = new StaticIcebergCatalogConfigProvider(); provider.initialize(Maps.newHashMap()); - Assertions.assertThrowsExactly( - RuntimeException.class, () -> provider.getIcebergTableOps(catalogName)); + Optional config = provider.getIcebergCatalogConfig(catalogName); + Assertions.assertEquals(Optional.empty(), config); } } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/extension/DummyCredentialProvider.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/extension/DummyCredentialProvider.java new file mode 100644 index 00000000000..6b1e4c08710 --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/extension/DummyCredentialProvider.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.service.extension; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import javax.annotation.Nullable; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialContext; +import org.apache.gravitino.credential.CredentialProvider; + +public class DummyCredentialProvider implements CredentialProvider { + public static final String DUMMY_CREDENTIAL_TYPE = "iceberg-rest-dummy-test"; + + public static class SimpleCredential implements Credential { + @Override + public String credentialType() { + return DUMMY_CREDENTIAL_TYPE; + } + + @Override + public long expireTimeInMs() { + return 0; + } + + @Override + public Map credentialInfo() { + return new HashMap<>(); + } + } + + @Override + public void initialize(Map properties) {} + + @Override + public String credentialType() { + return DUMMY_CREDENTIAL_TYPE; + } + + @Nullable + @Override + public Credential getCredential(CredentialContext context) { + return new SimpleCredential(); + } + + @Override + public void close() throws IOException {} +} diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/ConfigBasedIcebergCatalogWrapperProviderForTest.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/ConfigBasedIcebergCatalogWrapperProviderForTest.java deleted file mode 100644 index 222391bcc04..00000000000 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/ConfigBasedIcebergCatalogWrapperProviderForTest.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.gravitino.iceberg.service.rest; - -import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; -import org.apache.gravitino.iceberg.provider.ConfigBasedIcebergCatalogWrapperProvider; - -public class ConfigBasedIcebergCatalogWrapperProviderForTest - extends ConfigBasedIcebergCatalogWrapperProvider { - @Override - public IcebergCatalogWrapper getIcebergTableOps(String prefix) { - return new IcebergCatalogWrapperForTest(); - } -} diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperForTest.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperForTest.java index 69c0a50e409..f6326dd229e 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperForTest.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperForTest.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.iceberg.service.rest; +import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; @@ -30,7 +31,12 @@ import org.apache.iceberg.types.Types.StringType; import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; +// Used to override registerTable public class IcebergCatalogWrapperForTest extends IcebergCatalogWrapper { + public IcebergCatalogWrapperForTest(IcebergConfig icebergConfig) { + super(icebergConfig); + } + @Override public LoadTableResponse registerTable(Namespace namespace, RegisterTableRequest request) { if (request.name().contains("fail")) { diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java new file mode 100644 index 00000000000..7d359926a85 --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.service.rest; + +import java.util.Map; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; +import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager; + +// Provide a custom catalogWrapper to do test like `registerTable` +public class IcebergCatalogWrapperManagerForTest extends IcebergCatalogWrapperManager { + public IcebergCatalogWrapperManagerForTest(Map properties) { + super(properties); + } + + @Override + public IcebergCatalogWrapper createIcebergCatalogWrapper(IcebergConfig icebergConfig) { + return new IcebergCatalogWrapperForTest(icebergConfig); + } +} diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java index 4fc645132e1..1a085a251d9 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java @@ -24,10 +24,13 @@ import java.util.logging.Level; import java.util.logging.Logger; import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.credential.CredentialConstants; import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.iceberg.provider.StaticIcebergCatalogConfigProvider; import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager; import org.apache.gravitino.iceberg.service.IcebergExceptionMapper; import org.apache.gravitino.iceberg.service.IcebergObjectMapperProvider; +import org.apache.gravitino.iceberg.service.extension.DummyCredentialProvider; import org.apache.gravitino.iceberg.service.metrics.IcebergMetricsManager; import org.glassfish.hk2.utilities.binding.AbstractBinder; import org.glassfish.jersey.jackson.JacksonFeature; @@ -74,12 +77,17 @@ public static ResourceConfig getIcebergResourceConfig(Class c, boolean bindIcebe if (bindIcebergTableOps) { Map catalogConf = Maps.newHashMap(); - catalogConf.put(String.format("catalog.%s.catalog-backend-name", PREFIX), PREFIX); + String catalogConfigPrefix = "catalog." + PREFIX; catalogConf.put( - IcebergConstants.ICEBERG_REST_CATALOG_PROVIDER, - ConfigBasedIcebergCatalogWrapperProviderForTest.class.getName()); + IcebergConstants.ICEBERG_REST_CATALOG_CONFIG_PROVIDER, + StaticIcebergCatalogConfigProvider.class.getName()); + catalogConf.put(String.format("%s.catalog-backend-name", catalogConfigPrefix), PREFIX); + catalogConf.put( + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + DummyCredentialProvider.DUMMY_CREDENTIAL_TYPE); + // used to override register table interface IcebergCatalogWrapperManager icebergCatalogWrapperManager = - new IcebergCatalogWrapperManager(catalogConf); + new IcebergCatalogWrapperManagerForTest(catalogConf); IcebergMetricsManager icebergMetricsManager = new IcebergMetricsManager(new IcebergConfig()); resourceConfig.register( diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java index 6037302b8b2..809a4ff2cd5 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/TestIcebergTableOperations.java @@ -29,6 +29,8 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.iceberg.service.extension.DummyCredentialProvider; import org.apache.iceberg.MetadataUpdate; import org.apache.iceberg.Schema; import org.apache.iceberg.TableMetadata; @@ -55,6 +57,12 @@ public class TestIcebergTableOperations extends TestIcebergNamespaceOperations { + private static final Schema tableSchema = + new Schema(NestedField.of(1, false, "foo_string", StringType.get())); + + private static final Schema newTableSchema = + new Schema(NestedField.of(2, false, "foo_string1", StringType.get())); + @Override protected Application configure() { ResourceConfig resourceConfig = @@ -66,11 +74,163 @@ protected Application configure() { return resourceConfig; } - private static final Schema tableSchema = - new Schema(NestedField.of(1, false, "foo_string", StringType.get())); + @Test + void testCreateTable() { + verifyCreateTableFail("create_foo1", 404); - private static final Schema newTableSchema = - new Schema(NestedField.of(2, false, "foo_string1", StringType.get())); + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + + verifyCreateTableSucc("create_foo1"); + + verifyCreateTableFail("create_foo1", 409); + verifyCreateTableFail("", 400); + } + + @Test + void testLoadTable() { + verifyLoadTableFail("load_foo1", 404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateTableSucc("load_foo1"); + verifyLoadTableSucc("load_foo1"); + + verifyLoadTableFail("load_foo2", 404); + } + + @Test + void testDropTable() { + verifyDropTableFail("drop_foo1", 404); + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyDropTableFail("drop_foo1", 404); + + verifyCreateTableSucc("drop_foo1"); + verifyDropTableSucc("drop_foo1"); + verifyLoadTableFail("drop_foo1", 404); + } + + @Test + void testUpdateTable() { + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateTableSucc("update_foo1"); + TableMetadata metadata = getTableMeta("update_foo1"); + verifyUpdateSucc("update_foo1", metadata); + + verifyDropTableSucc("update_foo1"); + verifyUpdateTableFail("update_foo1", 404, metadata); + + verifyDropNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyUpdateTableFail("update_foo1", 404, metadata); + } + + @ParameterizedTest + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + void testListTables(String prefix) { + setUrlPathWithPrefix(prefix); + verifyListTableFail(404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateTableSucc("list_foo1"); + verifyCreateTableSucc("list_foo2"); + verifyListTableSucc(ImmutableSet.of("list_foo1", "list_foo2")); + } + + @Test + void testTableExits() { + verifyTableExistsStatusCode("exists_foo2", 404); + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyTableExistsStatusCode("exists_foo2", 404); + + verifyCreateTableSucc("exists_foo1"); + verifyTableExistsStatusCode("exists_foo1", 200); + verifyLoadTableSucc("exists_foo1"); + } + + @ParameterizedTest + @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) + void testRenameTable(String prefix) { + setUrlPathWithPrefix(prefix); + // namespace not exits + verifyRenameTableFail("rename_foo1", "rename_foo3", 404); + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateTableSucc("rename_foo1"); + // rename + verifyRenameTableSucc("rename_foo1", "rename_foo2"); + verifyLoadTableFail("rename_foo1", 404); + verifyLoadTableSucc("rename_foo2"); + + // source table not exists + verifyRenameTableFail("rename_foo1", "rename_foo3", 404); + + // dest table exists + verifyCreateTableSucc("rename_foo3"); + verifyRenameTableFail("rename_foo2", "rename_foo3", 409); + } + + @Test + void testReportTableMetrics() { + + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + verifyCreateTableSucc("metrics_foo1"); + + ImmutableCommitMetricsResult commitMetrics = ImmutableCommitMetricsResult.builder().build(); + CommitReport commitReport = + ImmutableCommitReport.builder() + .tableName("metrics_foo1") + .snapshotId(-1) + .sequenceNumber(-1) + .operation("append") + .commitMetrics(commitMetrics) + .build(); + ReportMetricsRequest request = ReportMetricsRequest.of(commitReport); + Response response = + getReportMetricsClientBuilder("metrics_foo1") + .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Status.NO_CONTENT.getStatusCode(), response.getStatus()); + } + + @Test + void testCreateTableWithCredentialVending() { + verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); + + // create the table without credential vending + Response response = doCreateTable("create_without_credential_vending"); + Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus()); + LoadTableResponse loadTableResponse = response.readEntity(LoadTableResponse.class); + Assertions.assertTrue(!loadTableResponse.config().containsKey(Credential.CREDENTIAL_TYPE)); + + // create the table with credential vending + String tableName = "create_with_credential_vending"; + response = doCreateTableWithCredentialVending(tableName); + Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus()); + loadTableResponse = response.readEntity(LoadTableResponse.class); + Assertions.assertEquals( + DummyCredentialProvider.DUMMY_CREDENTIAL_TYPE, + loadTableResponse.config().get(Credential.CREDENTIAL_TYPE)); + + // load the table without credential vending + response = doLoadTable(tableName); + Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus()); + loadTableResponse = response.readEntity(LoadTableResponse.class); + Assertions.assertTrue(!loadTableResponse.config().containsKey(Credential.CREDENTIAL_TYPE)); + + // load the table with credential vending + response = doLoadTableWithCredentialVending(tableName); + Assertions.assertEquals(Status.OK.getStatusCode(), response.getStatus()); + loadTableResponse = response.readEntity(LoadTableResponse.class); + Assertions.assertEquals( + DummyCredentialProvider.DUMMY_CREDENTIAL_TYPE, + loadTableResponse.config().get(Credential.CREDENTIAL_TYPE)); + } + + private Response doCreateTableWithCredentialVending(String name) { + CreateTableRequest createTableRequest = + CreateTableRequest.builder().withName(name).withSchema(tableSchema).build(); + return getTableClientBuilder() + .header(IcebergTableOperations.X_ICEBERG_ACCESS_DELEGATION, "vended-credentials") + .post(Entity.entity(createTableRequest, MediaType.APPLICATION_JSON_TYPE)); + } private Response doCreateTable(String name) { CreateTableRequest createTableRequest = @@ -103,6 +263,12 @@ private Response doTableExists(String name) { return getTableClientBuilder(Optional.of(name)).head(); } + private Response doLoadTableWithCredentialVending(String name) { + return getTableClientBuilder(Optional.of(name)) + .header(IcebergTableOperations.X_ICEBERG_ACCESS_DELEGATION, "vended-credentials") + .get(); + } + private Response doLoadTable(String name) { return getTableClientBuilder(Optional.of(name)).get(); } @@ -116,6 +282,12 @@ private Response doUpdateTable(String name, TableMetadata base) { .post(Entity.entity(updateTableRequest, MediaType.APPLICATION_JSON_TYPE)); } + private TableMetadata getTableMeta(String tableName) { + Response response = doLoadTable(tableName); + LoadTableResponse loadTableResponse = response.readEntity(LoadTableResponse.class); + return loadTableResponse.tableMetadata(); + } + private void verifyUpdateTableFail(String name, int status, TableMetadata base) { Response response = doUpdateTable(name, base); Assertions.assertEquals(status, response.getStatus()); @@ -204,126 +376,4 @@ private void verifyCreateTableFail(String name, int status) { Response response = doCreateTable(name); Assertions.assertEquals(status, response.getStatus()); } - - @Test - void testCreateTable() { - verifyCreateTableFail("create_foo1", 404); - - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - - verifyCreateTableSucc("create_foo1"); - - verifyCreateTableFail("create_foo1", 409); - verifyCreateTableFail("", 400); - } - - @Test - void testLoadTable() { - verifyLoadTableFail("load_foo1", 404); - - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyCreateTableSucc("load_foo1"); - verifyLoadTableSucc("load_foo1"); - - verifyLoadTableFail("load_foo2", 404); - } - - @Test - void testDropTable() { - verifyDropTableFail("drop_foo1", 404); - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyDropTableFail("drop_foo1", 404); - - verifyCreateTableSucc("drop_foo1"); - verifyDropTableSucc("drop_foo1"); - verifyLoadTableFail("drop_foo1", 404); - } - - private TableMetadata getTableMeta(String tableName) { - Response response = doLoadTable(tableName); - LoadTableResponse loadTableResponse = response.readEntity(LoadTableResponse.class); - return loadTableResponse.tableMetadata(); - } - - @Test - void testUpdateTable() { - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyCreateTableSucc("update_foo1"); - TableMetadata metadata = getTableMeta("update_foo1"); - verifyUpdateSucc("update_foo1", metadata); - - verifyDropTableSucc("update_foo1"); - verifyUpdateTableFail("update_foo1", 404, metadata); - - verifyDropNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyUpdateTableFail("update_foo1", 404, metadata); - } - - @ParameterizedTest - @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) - void testListTables(String prefix) { - setUrlPathWithPrefix(prefix); - verifyListTableFail(404); - - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyCreateTableSucc("list_foo1"); - verifyCreateTableSucc("list_foo2"); - verifyListTableSucc(ImmutableSet.of("list_foo1", "list_foo2")); - } - - @Test - void testTableExits() { - verifyTableExistsStatusCode("exists_foo2", 404); - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyTableExistsStatusCode("exists_foo2", 404); - - verifyCreateTableSucc("exists_foo1"); - verifyTableExistsStatusCode("exists_foo1", 200); - verifyLoadTableSucc("exists_foo1"); - } - - @ParameterizedTest - @ValueSource(strings = {"", IcebergRestTestUtil.PREFIX}) - void testRenameTable(String prefix) { - setUrlPathWithPrefix(prefix); - // namespace not exits - verifyRenameTableFail("rename_foo1", "rename_foo3", 404); - - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyCreateTableSucc("rename_foo1"); - // rename - verifyRenameTableSucc("rename_foo1", "rename_foo2"); - verifyLoadTableFail("rename_foo1", 404); - verifyLoadTableSucc("rename_foo2"); - - // source table not exists - verifyRenameTableFail("rename_foo1", "rename_foo3", 404); - - // dest table exists - verifyCreateTableSucc("rename_foo3"); - verifyRenameTableFail("rename_foo2", "rename_foo3", 409); - } - - @Test - void testReportTableMetrics() { - - verifyCreateNamespaceSucc(IcebergRestTestUtil.TEST_NAMESPACE_NAME); - verifyCreateTableSucc("metrics_foo1"); - - ImmutableCommitMetricsResult commitMetrics = ImmutableCommitMetricsResult.builder().build(); - CommitReport commitReport = - ImmutableCommitReport.builder() - .tableName("metrics_foo1") - .snapshotId(-1) - .sequenceNumber(-1) - .operation("append") - .commitMetrics(commitMetrics) - .build(); - ReportMetricsRequest request = ReportMetricsRequest.of(commitReport); - Response response = - getReportMetricsClientBuilder("metrics_foo1") - .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); - - Assertions.assertEquals(Status.NO_CONTENT.getStatusCode(), response.getStatus()); - } } diff --git a/iceberg/iceberg-rest-server/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/iceberg/iceberg-rest-server/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider new file mode 100644 index 00000000000..25a4f2d46b1 --- /dev/null +++ b/iceberg/iceberg-rest-server/src/test/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.gravitino.iceberg.service.extension.DummyCredentialProvider diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/RoleOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/RoleOperations.java index 91ebaf5b464..e986753d0ce 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/RoleOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/RoleOperations.java @@ -50,6 +50,8 @@ import org.apache.gravitino.dto.responses.NameListResponse; import org.apache.gravitino.dto.responses.RoleResponse; import org.apache.gravitino.dto.util.DTOConverters; +import org.apache.gravitino.exceptions.IllegalMetadataObjectException; +import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; import org.apache.gravitino.lock.LockType; import org.apache.gravitino.lock.TreeLockUtils; import org.apache.gravitino.metrics.MetricNames; @@ -143,7 +145,11 @@ public Response createRole(@PathParam("metalake") String metalake, RoleCreateReq for (Privilege privilege : object.privileges()) { AuthorizationUtils.checkPrivilege((PrivilegeDTO) privilege, object, metalake); } - MetadataObjectUtil.checkMetadataObject(metalake, object); + try { + MetadataObjectUtil.checkMetadataObject(metalake, object); + } catch (NoSuchMetadataObjectException nsm) { + throw new IllegalMetadataObjectException(nsm); + } } List securableObjects = diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestPermissionOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestPermissionOperations.java index e927a0a4e96..8876e9035f4 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestPermissionOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestPermissionOperations.java @@ -55,8 +55,8 @@ import org.apache.gravitino.dto.responses.RoleResponse; import org.apache.gravitino.dto.responses.UserResponse; import org.apache.gravitino.exceptions.IllegalPrivilegeException; +import org.apache.gravitino.exceptions.IllegalRoleException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; -import org.apache.gravitino.exceptions.NoSuchRoleException; import org.apache.gravitino.exceptions.NoSuchUserException; import org.apache.gravitino.lock.LockManager; import org.apache.gravitino.meta.AuditInfo; @@ -186,8 +186,8 @@ public void testGrantRolesToUser() { Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse.getCode()); Assertions.assertEquals(NoSuchUserException.class.getSimpleName(), errorResponse.getType()); - // Test to throw NoSuchRoleException - doThrow(new NoSuchRoleException("mock error")) + // Test to throw IllegalRoleException + doThrow(new IllegalRoleException("mock error")) .when(manager) .grantRolesToUser(any(), any(), any()); resp1 = @@ -196,12 +196,12 @@ public void testGrantRolesToUser() { .accept("application/vnd.gravitino.v1+json") .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); - Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp1.getStatus()); Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); errorResponse = resp1.readEntity(ErrorResponse.class); - Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse.getCode()); - Assertions.assertEquals(NoSuchRoleException.class.getSimpleName(), errorResponse.getType()); + Assertions.assertEquals(ErrorConstants.ILLEGAL_ARGUMENTS_CODE, errorResponse.getCode()); + Assertions.assertEquals(IllegalRoleException.class.getSimpleName(), errorResponse.getType()); // Test to throw internal RuntimeException doThrow(new RuntimeException("mock error")).when(manager).grantRolesToUser(any(), any(), any()); @@ -284,8 +284,8 @@ public void testGrantRolesToGroup() { Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse.getCode()); Assertions.assertEquals(NoSuchUserException.class.getSimpleName(), errorResponse.getType()); - // Test to throw NoSuchRoleException - doThrow(new NoSuchRoleException("mock error")) + // Test to throw IllegalRoleException + doThrow(new IllegalRoleException("mock error")) .when(manager) .grantRolesToGroup(any(), any(), any()); resp1 = @@ -294,12 +294,12 @@ public void testGrantRolesToGroup() { .accept("application/vnd.gravitino.v1+json") .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); - Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), resp1.getStatus()); Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); errorResponse = resp1.readEntity(ErrorResponse.class); - Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResponse.getCode()); - Assertions.assertEquals(NoSuchRoleException.class.getSimpleName(), errorResponse.getType()); + Assertions.assertEquals(ErrorConstants.ILLEGAL_ARGUMENTS_CODE, errorResponse.getCode()); + Assertions.assertEquals(IllegalRoleException.class.getSimpleName(), errorResponse.getType()); // Test to throw internal RuntimeException doThrow(new RuntimeException("mock error")) @@ -362,6 +362,23 @@ public void testRevokeRolesFromUser() { ErrorResponse errorResponse = resp3.readEntity(ErrorResponse.class); Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResponse.getCode()); Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse.getType()); + + // Test to throw IllegalRoleException + doThrow(new IllegalRoleException("mock error")) + .when(manager) + .revokeRolesFromUser(any(), any(), any()); + Response nsrResponse = + target("/metalakes/metalake1/permissions/users/user/revoke") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), nsrResponse.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, nsrResponse.getMediaType()); + + errorResponse = nsrResponse.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.ILLEGAL_ARGUMENTS_CODE, errorResponse.getCode()); + Assertions.assertEquals(IllegalRoleException.class.getSimpleName(), errorResponse.getType()); } @Test @@ -407,6 +424,23 @@ public void testRevokeRolesFromGroup() { ErrorResponse errorResponse = resp3.readEntity(ErrorResponse.class); Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResponse.getCode()); Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResponse.getType()); + + // Test to throw IllegalRoleException + doThrow(new IllegalRoleException("mock error")) + .when(manager) + .revokeRolesFromGroup(any(), any(), any()); + Response nsrResponse = + target("/metalakes/metalake1/permissions/groups/group/revoke") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .put(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), nsrResponse.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, nsrResponse.getMediaType()); + + errorResponse = nsrResponse.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.ILLEGAL_ARGUMENTS_CODE, errorResponse.getCode()); + Assertions.assertEquals(IllegalRoleException.class.getSimpleName(), errorResponse.getType()); } @Test diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestRoleOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestRoleOperations.java index 55fa7dd3aea..5a53ec5f9f0 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestRoleOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestRoleOperations.java @@ -206,10 +206,10 @@ public void testCreateRole() { .request(MediaType.APPLICATION_JSON_TYPE) .accept("application/vnd.gravitino.v1+json") .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); - Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), respNotExist.getStatus()); + Assertions.assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), respNotExist.getStatus()); Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, respNotExist.getMediaType()); ErrorResponse notExistResponse = respNotExist.readEntity(ErrorResponse.class); - Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, notExistResponse.getCode()); + Assertions.assertEquals(ErrorConstants.ILLEGAL_ARGUMENTS_CODE, notExistResponse.getCode()); // Test to throw NoSuchMetalakeException when(catalogDispatcher.catalogExists(any())).thenReturn(true);