Skip to content

Commit

Permalink
Merge branch 'main' of github.com:datastrato/graviton into issue_5139
Browse files Browse the repository at this point in the history
  • Loading branch information
yuqi1129 committed Oct 18, 2024
2 parents 4478673 + 437f367 commit 5a194df
Show file tree
Hide file tree
Showing 94 changed files with 1,779 additions and 199 deletions.
2 changes: 2 additions & 0 deletions LICENSE.bin
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,8 @@
ApacheDS I18n
ApacheDS Protocol Kerberos Codec
Apache Hadoop
Apache Hadoop Aliyun connector
Apache Hadoop GCS connector
Apache Hadoop Annotatations
Apache Hadoop Auth
Apache Hadoop Client Aggregator
Expand Down
3 changes: 3 additions & 0 deletions api/src/main/java/org/apache/gravitino/Catalog.java
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ enum CloudName {
*/
String PROPERTY_PACKAGE = "package";

/** The property indicates the catalog is in use. */
String PROPERTY_IN_USE = "in-use";

/**
* The property to specify the cloud that the catalog is running on. The value should be one of
* the {@link CloudName}.
Expand Down
80 changes: 71 additions & 9 deletions api/src/main/java/org/apache/gravitino/SupportsCatalogs.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@
import java.util.Map;
import org.apache.gravitino.annotation.Evolving;
import org.apache.gravitino.exceptions.CatalogAlreadyExistsException;
import org.apache.gravitino.exceptions.CatalogInUseException;
import org.apache.gravitino.exceptions.CatalogNotInUseException;
import org.apache.gravitino.exceptions.NoSuchCatalogException;
import org.apache.gravitino.exceptions.NoSuchMetalakeException;
import org.apache.gravitino.exceptions.NonEmptyEntityException;

/**
* Client interface for supporting catalogs. It includes methods for listing, loading, creating,
Expand All @@ -48,9 +51,9 @@ public interface SupportsCatalogs {
Catalog[] listCatalogsInfo() throws NoSuchMetalakeException;

/**
* Load a catalog by its identifier.
* Load a catalog by its name.
*
* @param catalogName the identifier of the catalog.
* @param catalogName the name of the catalog.
* @return The catalog.
* @throws NoSuchCatalogException If the catalog does not exist.
*/
Expand All @@ -59,7 +62,7 @@ public interface SupportsCatalogs {
/**
* Check if a catalog exists.
*
* @param catalogName The identifier of the catalog.
* @param catalogName The name of the catalog.
* @return True if the catalog exists, false otherwise.
*/
default boolean catalogExists(String catalogName) {
Expand All @@ -72,7 +75,7 @@ default boolean catalogExists(String catalogName) {
}

/**
* Create a catalog with specified identifier.
* Create a catalog with specified catalog name, type, provider, comment, and properties.
*
* <p>The parameter "provider" is a short name of the catalog, used to tell Gravitino which
* catalog should be created. The short name should be the same as the {@link CatalogProvider}
Expand All @@ -96,9 +99,9 @@ Catalog createCatalog(
throws NoSuchMetalakeException, CatalogAlreadyExistsException;

/**
* Alter a catalog with specified identifier.
* Alter a catalog with specified catalog name and changes.
*
* @param catalogName the identifier of the catalog.
* @param catalogName the name of the catalog.
* @param changes the changes to apply to the catalog.
* @return The altered catalog.
* @throws NoSuchCatalogException If the catalog does not exist.
Expand All @@ -108,12 +111,71 @@ Catalog alterCatalog(String catalogName, CatalogChange... changes)
throws NoSuchCatalogException, IllegalArgumentException;

/**
* Drop a catalog with specified identifier.
* Drop a catalog with specified name. Please make sure:
*
* <ul>
* <li>There is no schema in the catalog. Otherwise, a {@link NonEmptyEntityException} will be
* thrown.
* <li>The method {@link #disableCatalog(String)} has been called before dropping the catalog.
* Otherwise, a {@link CatalogInUseException} will be thrown.
* </ul>
*
* It is equivalent to calling {@code dropCatalog(ident, false)}.
*
* @param catalogName the name of the catalog.
* @return True if the catalog was dropped, false otherwise.
* @return True if the catalog was dropped, false if the catalog does not exist.
* @throws NonEmptyEntityException If the catalog is not empty.
* @throws CatalogInUseException If the catalog is in use.
*/
default boolean dropCatalog(String catalogName)
throws NonEmptyEntityException, CatalogInUseException {
return dropCatalog(catalogName, false);
}

/**
* Drop a catalog with specified name. If the force flag is true, it will:
*
* <ul>
* <li>Cascade drop all sub-entities (schemas, tables, etc.) of the catalog in Gravitino store.
* <li>Drop the catalog even if it is in use.
* <li>External resources (e.g. database, table, etc.) associated with sub-entities will not be
* dropped unless it is managed (such as managed fileset).
* </ul>
*
* If the force flag is false, it is equivalent to calling {@link #dropCatalog(String)}.
*
* @param catalogName The identifier of the catalog.
* @param force Whether to force the drop.
* @return True if the catalog was dropped, false if the catalog does not exist.
* @throws NonEmptyEntityException If the catalog is not empty and force is false.
* @throws CatalogInUseException If the catalog is in use and force is false.
*/
boolean dropCatalog(String catalogName, boolean force)
throws NonEmptyEntityException, CatalogInUseException;

/**
* Enable a catalog. If the catalog is already enabled, this method does nothing.
*
* @param catalogName The identifier of the catalog.
* @throws NoSuchCatalogException If the catalog does not exist.
*/
void enableCatalog(String catalogName) throws NoSuchCatalogException;

/**
* Disable a catalog. If the catalog is already disabled, this method does nothing. Once a catalog
* is disabled:
*
* <ul>
* <li>It can only be listed, loaded, dropped, or disable.
* <li>Any other operations on the catalog will throw an {@link CatalogNotInUseException}.
* <li>Any operation on the sub-entities (schemas, tables, etc.) will throw an {@link
* CatalogNotInUseException}.
* </ul>
*
* @param catalogName The identifier of the catalog.
* @throws NoSuchCatalogException If the catalog does not exist.
*/
boolean dropCatalog(String catalogName);
void disableCatalog(String catalogName) throws NoSuchCatalogException;

/**
* Test whether the catalog with specified parameters can be connected to before creating it.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.gravitino.exceptions;

import com.google.errorprone.annotations.FormatMethod;
import com.google.errorprone.annotations.FormatString;

/** Exception thrown when a catalog is in use and cannot be deleted. */
public class CatalogInUseException extends InUseException {
/**
* Constructs a new exception with the specified detail message.
*
* @param message the detail message.
* @param args the arguments to the message.
*/
@FormatMethod
public CatalogInUseException(@FormatString String message, Object... args) {
super(message, args);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.gravitino.exceptions;

import com.google.errorprone.annotations.FormatMethod;
import com.google.errorprone.annotations.FormatString;

/** An exception thrown when operating on a catalog that is not in use. */
public class CatalogNotInUseException extends NotInUseException {
/**
* Constructs a new exception with the specified detail message.
*
* @param message the detail message.
* @param args the arguments to the message.
*/
@FormatMethod
public CatalogNotInUseException(@FormatString String message, Object... args) {
super(message, args);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.gravitino.exceptions;

import com.google.errorprone.annotations.FormatMethod;
import com.google.errorprone.annotations.FormatString;

/** Exception thrown when a resource is in use and cannot be deleted. */
public class InUseException extends GravitinoRuntimeException {
/**
* Constructs a new exception with the specified detail message.
*
* @param message the detail message.
* @param args the arguments to the message.
*/
@FormatMethod
public InUseException(@FormatString String message, Object... args) {
super(message, args);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.gravitino.exceptions;

import com.google.errorprone.annotations.FormatMethod;
import com.google.errorprone.annotations.FormatString;

/** An exception thrown when operating on a resource that is not in use. */
public class NotInUseException extends GravitinoRuntimeException {
/**
* Constructs a new exception with the specified detail message.
*
* @param message the detail message.
* @param args the arguments to the message.
*/
@FormatMethod
public NotInUseException(@FormatString String message, Object... args) {
super(message, args);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -190,10 +190,7 @@ public void stop() throws IOException {
catalog.asSchemas().dropSchema(schema, true);
}));
Arrays.stream(metalake.listCatalogs())
.forEach(
(catalogName -> {
metalake.dropCatalog(catalogName);
}));
.forEach((catalogName -> metalake.dropCatalog(catalogName, true)));
client.dropMetalake(metalakeName);
}
if (sparkSession != null) {
Expand Down
5 changes: 3 additions & 2 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -745,7 +745,7 @@ tasks {
if (!it.name.startsWith("catalog") &&
!it.name.startsWith("authorization") &&
!it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" &&
it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") && it.name != "gcp-bundle"
it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") && it.name != "gcp-bundle" && it.name != "aliyun-bundle"
) {
from(it.configurations.runtimeClasspath)
into("distribution/package/libs")
Expand All @@ -764,7 +764,8 @@ tasks {
!it.name.startsWith("integration-test") &&
!it.name.startsWith("flink") &&
!it.name.startsWith("trino-connector") &&
it.name != "hive-metastore-common" && it.name != "gcp-bundle"
it.name != "hive-metastore-common" && it.name != "gcp-bundle" &&
it.name != "aliyun-bundle"
) {
dependsOn("${it.name}:build")
from("${it.name}/build/libs")
Expand Down
58 changes: 58 additions & 0 deletions bundles/aliyun-bundle/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar

plugins {
`maven-publish`
id("java")
alias(libs.plugins.shadow)
}

dependencies {
compileOnly(project(":catalogs:catalog-hadoop"))
compileOnly(libs.hadoop3.common)
implementation(libs.hadoop3.oss)

// oss needs StringUtils from commons-lang or the following error will occur in 3.1.0
// java.lang.NoClassDefFoundError: org/apache/commons/lang/StringUtils
// org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore.initialize(AliyunOSSFileSystemStore.java:111)
// org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem.initialize(AliyunOSSFileSystem.java:323)
// org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3611)
implementation(libs.commons.lang)
}

tasks.withType(ShadowJar::class.java) {
isZip64 = true
configurations = listOf(project.configurations.runtimeClasspath.get())
archiveClassifier.set("")
mergeServiceFiles()

// Relocate dependencies to avoid conflicts
relocate("org.jdom", "org.apache.gravitino.shaded.org.jdom")
relocate("org.apache.commons.lang", "org.apache.gravitino.shaded.org.apache.commons.lang")
}

tasks.jar {
dependsOn(tasks.named("shadowJar"))
archiveClassifier.set("empty")
}

tasks.compileJava {
dependsOn(":catalogs:catalog-hadoop:runtimeJars")
}
Loading

0 comments on commit 5a194df

Please sign in to comment.