Skip to content

Commit

Permalink
Introduce Glue protocol module
Browse files Browse the repository at this point in the history 
New module that implements a Glue REST API endpoint allowing
Glue proxying/emulation.

- Adds serializers/deserializers for all Glue models
- Glue REST endpoint that marshals request payloads
- Injectable request handler
- Could be the basis of a Glue proxy, though this isn't implemented currently
- Can be run as a standalone Glue endpoint handler
- Can be add Glue endpoint to AWS S3 proxy as standard plugin
  • Loading branch information
@Randgalt
Randgalt committed Jan 30, 2025
1 parent 2a8e05b commit 3814f31
Show file tree
Hide file tree
Showing 23 changed files with 1,317 additions and 0 deletions.
26 changes: 26 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

<modules>
<module>trino-aws-proxy</module>
<module>trino-aws-proxy-glue</module>
<module>trino-aws-proxy-spark3</module>
<module>trino-aws-proxy-spark4</module>
<module>trino-aws-proxy-spi</module>
Expand Down Expand Up @@ -84,6 +85,19 @@
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>trino-aws-proxy</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
</dependency>

<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>trino-aws-proxy-glue</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>trino-aws-proxy-spark3</artifactId>
Expand Down Expand Up @@ -186,6 +200,18 @@
</exclusions>
</dependency>

<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>glue</artifactId>
<version>${dep.aws-sdk.version}</version>
<exclusions>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
Expand Down
69 changes: 69 additions & 0 deletions trino-aws-proxy-glue/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
# AWS Glue Emulation

Implementation of the AWS Glue endpoint and model serialization. Can be used as part of the
Trino AWS S3 Proxy or as part of a separate/standalone project.

## As part of Trino AWS S3 Proxy

Including the `trino-aws-proxy-glue` dependency will automatically add the AWS Glue plugin (via the
JDK Service Loader). See [configration and binding](#configuration-and-binding) below for more details.

## As part of a separate/standalone project

The AWS Glue endpoint implementation can be added to any Airlift application by installing
the `TrinoStandaloneGlueModule`.

## Configuration and binding

### Endpoint

The path to the Glue endpoint can be configured via the `aws.proxy.glue.path` configuration property.

### Handler

Bind an instance of `GlueRequestHandler` to handle Glue requests. Use the
`GlueRequestHandlerBinding` utility to do the binding. Your `GlueRequestHandler`
should examine the `ParsedGlueRequest` and handle any Glue requests and
return a Glue response.

E.g.

```java
import jakarta.ws.rs.WebApplicationException;
import jakarta.ws.rs.core.Response;
import software.amazon.awssdk.services.glue.model.GetDatabasesRequest;
import software.amazon.awssdk.services.glue.model.GetDatabasesResponse;

public class MyGlueRequestHandler
implements GlueRequestHandler
{
@Override
public GlueResponse handle(ParsedGlueRequest request)
{
// validate credentials, etc. via: request.requestAuthorization()

return switch (request.glueRequest()) {
case GetDatabasesRequest getDatabasesRequest -> {
// handle a GetDatabases request

yield new GetDatabasesResponse.builder()
// ...
.build();
}

// ...

default -> throw new WebApplicationException(NOT_FOUND);
};
}
}
```

E.g bind your handler

```java
Module module = binder -> {
glueRequestHandlerBinding(binder)
.bind(binding -> binding.to(MyGlueRequestHandler.class));
};
```
173 changes: 173 additions & 0 deletions trino-aws-proxy-glue/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>io.trino</groupId>
<artifactId>trino-aws-proxy-root</artifactId>
<version>4-SNAPSHOT</version>
</parent>

<artifactId>trino-aws-proxy-glue</artifactId>

<properties>
<air.main.basedir>${project.parent.basedir}</air.main.basedir>
</properties>

<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>trino-aws-proxy</artifactId>
</dependency>

<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>trino-aws-proxy-spi</artifactId>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>

<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>

<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>

<dependency>
<groupId>io.airlift</groupId>
<artifactId>configuration</artifactId>
</dependency>

<dependency>
<groupId>io.airlift</groupId>
<artifactId>jaxrs</artifactId>
</dependency>

<dependency>
<groupId>jakarta.validation</groupId>
<artifactId>jakarta.validation-api</artifactId>
</dependency>

<dependency>
<groupId>jakarta.ws.rs</groupId>
<artifactId>jakarta.ws.rs-api</artifactId>
</dependency>

<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>glue</artifactId>
</dependency>

<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>sdk-core</artifactId>
</dependency>

<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>utils</artifactId>
</dependency>

<dependency>
<groupId>io.airlift</groupId>
<artifactId>bootstrap</artifactId>
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>io.airlift</groupId>
<artifactId>http-server</artifactId>
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>io.airlift</groupId>
<artifactId>json</artifactId>
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>io.airlift</groupId>
<artifactId>node</artifactId>
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>jakarta.annotation</groupId>
<artifactId>jakarta.annotation-api</artifactId>
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>auth</artifactId>
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>aws-core</artifactId>
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>regions</artifactId>
<scope>runtime</scope>
</dependency>

<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>trino-aws-proxy</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>minio</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>postgresql</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>
37 changes: 37 additions & 0 deletions trino-aws-proxy-glue/src/main/java/io/trino/aws/proxy/glue/TrinoGlueConfig.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.aws.proxy.glue;

import io.airlift.configuration.Config;
import io.airlift.configuration.ConfigDescription;
import jakarta.validation.constraints.NotNull;

public class TrinoGlueConfig
{
private String gluePath = "/api/v1/glue";

@NotNull
public String getGluePath()
{
return gluePath;
}

@Config("aws.proxy.glue.path")
@ConfigDescription("URL Path for Glue operations, optional")
public TrinoGlueConfig setGluePath(String gluePath)
{
this.gluePath = gluePath;
return this;
}
}
46 changes: 46 additions & 0 deletions trino-aws-proxy-glue/src/main/java/io/trino/aws/proxy/glue/TrinoGlueModule.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.trino.aws.proxy.glue;

import com.google.inject.Binder;
import io.airlift.configuration.AbstractConfigurationAwareModule;
import io.trino.aws.proxy.glue.handler.GlueRequestHandler;
import io.trino.aws.proxy.glue.rest.ModelLoader;
import io.trino.aws.proxy.glue.rest.TrinoGlueResource;
import jakarta.ws.rs.WebApplicationException;

import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder;
import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder;
import static io.trino.aws.proxy.server.TrinoAwsProxyServerModule.bindResourceAtPath;
import static jakarta.ws.rs.core.Response.Status.NOT_FOUND;

public class TrinoGlueModule
extends AbstractConfigurationAwareModule
{
@Override
protected void setup(Binder binder)
{
ModelLoader modelLoader = new ModelLoader();
modelLoader.bindSerializers(binder);
binder.bind(ModelLoader.class).toInstance(modelLoader);

TrinoGlueConfig trinoGlueConfig = buildConfigObject(TrinoGlueConfig.class);
bindResourceAtPath(jaxrsBinder(binder), TrinoGlueResource.class, trinoGlueConfig.getGluePath());

newOptionalBinder(binder, GlueRequestHandler.class)
.setDefault().toInstance((_, _, _) -> {
throw new WebApplicationException(NOT_FOUND);
});
}
}
Loading

0 comments on commit 3814f31

Please sign in to comment.