Skip to content

Commit

Permalink
Allow broker to use catalog for datasource schemas
Browse files Browse the repository at this point in the history
  • Loading branch information
@jon-wei
jon-wei committed Dec 1, 2023
1 parent 9f3b266 commit 98912aa
Show file tree
Hide file tree
Showing 57 changed files with 2,705 additions and 350 deletions.
112 changes: 30 additions & 82 deletions extensions-core/druid-catalog/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,44 +48,22 @@
<version>${project.parent.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-indexing-service</artifactId>
<version>${project.parent.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-sql</artifactId>
<version>${project.parent.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-services</artifactId>
<version>${project.parent.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-multibindings</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.opencsv</groupId>
<artifactId>opencsv</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
Expand Down Expand Up @@ -142,11 +120,6 @@
<artifactId>curator-client</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-smile</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.jdbi</groupId>
<artifactId>jdbi</artifactId>
Expand All @@ -162,11 +135,6 @@
<artifactId>jsr311-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
Expand All @@ -177,68 +145,18 @@
<artifactId>jersey-server</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.lz4</groupId>
<artifactId>lz4-java</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.datasketches</groupId>
<artifactId>datasketches-java</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.datasketches</groupId>
<artifactId>datasketches-memory</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil-core</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<scope>provided</scope>
</dependency>

<!-- Tests -->
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>nl.jqno.equalsverifier</groupId>
<artifactId>equalsverifier</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-processing</artifactId>
Expand All @@ -262,4 +180,34 @@
</dependency>

</dependencies>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<configuration>
<!-- tool gets confused between these two items. -->
<usedDependencies>
<dependency>javax.inject:javax.inject</dependency>
</usedDependencies>
<ignoredUsedUndeclaredDependencies>
<ignoredUsedUndeclaredDependency>javax.inject:javax.inject</ignoredUsedUndeclaredDependency>
<ignoredUsedUndeclaredDependency>jakarta.inject:jakarta.inject-api</ignoredUsedUndeclaredDependency>
</ignoredUsedUndeclaredDependencies>
</configuration>
</plugin>
</plugins>
</build>
</project>
100 changes: 100 additions & 0 deletions ...-core/druid-catalog/src/main/java/org/apache/druid/catalog/guice/CatalogBrokerModule.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.druid.catalog.guice;

import com.google.inject.Binder;
import org.apache.druid.catalog.http.CatalogListenerResource;
import org.apache.druid.catalog.model.SchemaRegistry;
import org.apache.druid.catalog.model.SchemaRegistryImpl;
import org.apache.druid.catalog.sql.LiveCatalogResolver;
import org.apache.druid.catalog.sync.CachedMetadataCatalog;
import org.apache.druid.catalog.sync.CatalogClient;
import org.apache.druid.catalog.sync.CatalogUpdateListener;
import org.apache.druid.catalog.sync.CatalogUpdateReceiver;
import org.apache.druid.catalog.sync.MetadataCatalog;
import org.apache.druid.catalog.sync.MetadataCatalog.CatalogSource;
import org.apache.druid.discovery.NodeRole;
import org.apache.druid.guice.Jerseys;
import org.apache.druid.guice.LazySingleton;
import org.apache.druid.guice.LifecycleModule;
import org.apache.druid.guice.ManageLifecycle;
import org.apache.druid.guice.annotations.LoadScope;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.sql.calcite.planner.CatalogResolver;

/**
* Configures the metadata catalog on the Broker to use a cache
* and network communications for pull and push updates.
*/
@LoadScope(roles = NodeRole.BROKER_JSON_NAME)
public class CatalogBrokerModule implements DruidModule
{
@Override
public void configure(Binder binder)
{
// The Broker (catalog client) uses a cached metadata catalog.
binder
.bind(CachedMetadataCatalog.class)
.in(LazySingleton.class);

// Broker code accesses he catalog through the
// MetadataCatalog interface.
binder
.bind(MetadataCatalog.class)
.to(CachedMetadataCatalog.class)
.in(LazySingleton.class);

// The cached metadata catalog needs a "pull" source,
// which is the network client.
binder
.bind(CatalogSource.class)
.to(CatalogClient.class)
.in(LazySingleton.class);

// The cached metadata catalog is the listener for"push" events.
binder
.bind(CatalogUpdateListener.class)
.to(CachedMetadataCatalog.class)
.in(LazySingleton.class);

// At present, the set of schemas is fixed.
binder
.bind(SchemaRegistry.class)
.to(SchemaRegistryImpl.class)
.in(LazySingleton.class);

// Lifecycle-managed class to prime the metadata cache
binder
.bind(CatalogUpdateReceiver.class)
.in(ManageLifecycle.class);
LifecycleModule.register(binder, CatalogUpdateReceiver.class);

// Catalog resolver for the planner. This will override the
// base binding.
binder
.bind(CatalogResolver.class)
.to(LiveCatalogResolver.class)
.in(LazySingleton.class);

// The listener resource sends to the catalog
// listener (the cached catalog.)
Jerseys.addResource(binder, CatalogListenerResource.class);
}
}
18 changes: 18 additions & 0 deletions ...re/druid-catalog/src/main/java/org/apache/druid/catalog/http/CatalogListenerResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,4 +67,22 @@ public Response syncTable(final UpdateEvent event)
listener.updated(event);
return Response.status(Response.Status.ACCEPTED).build();
}

@POST
@Path("flush")
@ResourceFilters(ConfigResourceFilter.class)
public Response flush()
{
listener.flush();
return Response.status(Response.Status.ACCEPTED).build();
}

@POST
@Path("resync")
@ResourceFilters(ConfigResourceFilter.class)
public Response resync()
{
listener.resync();
return Response.status(Response.Status.ACCEPTED).build();
}
}
3 changes: 1 addition & 2 deletions ...sions-core/druid-catalog/src/main/java/org/apache/druid/catalog/http/CatalogResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,8 +278,7 @@ public Response editTable(
// Retrieval

/**
* Retrieves the list of all Druid schema names, all table names, or
* all table metadata.
* Retrieves the list of all Druid schema names.
*
* @param format the format of the response. See the code for the
* available formats
Expand Down
Loading

0 comments on commit 98912aa

Please sign in to comment.