From 367da6b7fcd16fbe5bfc9109feec15a576a6ec70 Mon Sep 17 00:00:00 2001 From: Jordan Zimmerman Date: Thu, 16 May 2024 11:00:53 +0100 Subject: [PATCH] Create initial S3 endpoint resource/handler Closes #8 --- README.md | 26 +++- pom.xml | 7 + trino-s3-proxy/pom.xml | 15 ++ .../server/TrinoS3ProxyServerModule.java | 19 ++- .../credentials/CredentialsController.java | 2 + .../server/credentials/SigningController.java | 59 ++++++- .../server/credentials/SigningMetadata.java | 25 +++ .../server/rest/StreamingResponseHandler.java | 92 +++++++++++ .../proxy/server/rest/TrinoS3ProxyClient.java | 147 ++++++++++++++++++ .../server/rest/TrinoS3ProxyResource.java | 75 ++++++++- .../rest/TrinoS3ProxyRestConstants.java | 1 + .../server/TestingCredentialsController.java | 39 +++++ .../server/TestingTrinoS3ProxyServer.java | 26 +++- .../TestingTrinoS3ProxyServerModule.java | 29 ++++ .../credentials/TestSigningController.java | 29 +++- 15 files changed, 563 insertions(+), 28 deletions(-) create mode 100644 trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SigningMetadata.java create mode 100644 trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/StreamingResponseHandler.java create mode 100644 trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyClient.java create mode 100644 trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingCredentialsController.java create mode 100644 trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServerModule.java diff --git a/README.md b/README.md index acf5be91..04d51e54 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,28 @@ Proxy for S3 To run testing server: ```shell -./mvnw package -./mvnw -pl trino-s3-proxy exec:java -Dexec.classpathScope=test -Dexec.mainClass=io.trino.s3.proxy.server.TestingTrinoS3ProxyServer +./mvnw clean package +./mvnw -pl trino-s3-proxy exec:java -Dexec.classpathScope=test -Dexec.mainClass=io.trino.s3.proxy.server.TestingTrinoS3ProxyServer +``` + +Make note of the logged `Endpoint` + +To test out proxied APIs: + +First, set up AWS CLI: + +```shell +> aws configure +AWS Access Key ID [None]: enter the fake access key +AWS Secret Access Key [None]: enter the fake secret key +Default region name [None]: +Default output format [None]: +``` + +Now, make calls. E.g. + +```shell +aws --endpoint-url http:///api/v1/s3Proxy/s3 s3 ls s3:/// + +aws --endpoint-url http:///api/v1/s3Proxy/s3 s3 cp s3://// . ``` diff --git a/pom.xml b/pom.xml index 7d4c70f3..e7d51591 100644 --- a/pom.xml +++ b/pom.xml @@ -47,6 +47,7 @@ 245 2.25.32 + 3.1.6 @@ -66,6 +67,12 @@ pom import + + + org.glassfish.jersey.core + jersey-server + ${dep.jersey.version} + diff --git a/trino-s3-proxy/pom.xml b/trino-s3-proxy/pom.xml index 35c460dd..994d7262 100644 --- a/trino-s3-proxy/pom.xml +++ b/trino-s3-proxy/pom.xml @@ -34,6 +34,11 @@ event + + io.airlift + http-client + + io.airlift http-server @@ -59,11 +64,21 @@ node + + jakarta.annotation + jakarta.annotation-api + + jakarta.ws.rs jakarta.ws.rs-api + + org.glassfish.jersey.core + jersey-server + + software.amazon.awssdk auth diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServerModule.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServerModule.java index 8ce01249..f7312fe7 100644 --- a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServerModule.java +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServerModule.java @@ -15,16 +15,33 @@ import com.google.inject.Binder; import com.google.inject.Module; +import com.google.inject.Scopes; +import io.trino.s3.proxy.server.credentials.SigningController; +import io.trino.s3.proxy.server.rest.TrinoS3ProxyClient; +import io.trino.s3.proxy.server.rest.TrinoS3ProxyClient.ForProxyClient; import io.trino.s3.proxy.server.rest.TrinoS3ProxyResource; +import static io.airlift.http.client.HttpClientBinder.httpClientBinder; import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; public class TrinoS3ProxyServerModule implements Module { @Override - public void configure(Binder binder) + public final void configure(Binder binder) { jaxrsBinder(binder).bind(TrinoS3ProxyResource.class); + binder.bind(SigningController.class).in(Scopes.SINGLETON); + + // TODO config, etc. + httpClientBinder(binder).bindHttpClient("ProxyClient", ForProxyClient.class); + binder.bind(TrinoS3ProxyClient.class).in(Scopes.SINGLETON); + + moduleSpecificBinding(binder); + } + + protected void moduleSpecificBinding(Binder binder) + { + // default does nothing } } diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/CredentialsController.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/CredentialsController.java index 96f7852b..1930fd5f 100644 --- a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/CredentialsController.java +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/CredentialsController.java @@ -18,4 +18,6 @@ public interface CredentialsController { Optional credentials(String emulatedAccessKey); + + void upsertCredentials(Credentials credentials); } diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SigningController.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SigningController.java index 499cc253..3d5d79b2 100644 --- a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SigningController.java +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SigningController.java @@ -13,11 +13,14 @@ */ package io.trino.s3.proxy.server.credentials; +import com.google.common.base.Splitter; import com.google.inject.Inject; import jakarta.ws.rs.core.MultivaluedMap; import java.net.URI; +import java.util.List; import java.util.Optional; +import java.util.function.Function; import static java.util.Objects.requireNonNull; @@ -31,17 +34,57 @@ public SigningController(CredentialsController credentialsController) this.credentialsController = requireNonNull(credentialsController, "credentialsController is null"); } + public Optional signingMetadataFromRequest( + Function credentialsSupplier, + URI requestURI, + MultivaluedMap requestHeaders, + MultivaluedMap queryParameters, + String httpMethod, + String encodedPath) + { + String authorization = requestHeaders.getFirst("Authorization"); + if (authorization == null) { + return Optional.empty(); + } + + List authorizationParts = Splitter.on(",").trimResults().splitToList(authorization); + if (authorizationParts.isEmpty()) { + return Optional.empty(); + } + + String credential = authorizationParts.getFirst(); + List credentialParts = Splitter.on("=").splitToList(credential); + if (credentialParts.size() < 2) { + return Optional.empty(); + } + + String credentialValue = credentialParts.get(1); + List credentialValueParts = Splitter.on("/").splitToList(credentialValue); + if (credentialValueParts.size() < 3) { + return Optional.empty(); + } + + String emulatedAccessKey = credentialValueParts.getFirst(); + String region = credentialValueParts.get(2); + + return credentialsController.credentials(emulatedAccessKey) + .map(credentials -> new SigningMetadata(credentials, region)) + .filter(metadata -> { + String expectedAuthorization = signRequest(metadata, Credentials::emulated, requestURI, requestHeaders, queryParameters, httpMethod, encodedPath); + return authorization.equals(expectedAuthorization); + }); + } + public String signRequest( + SigningMetadata metadata, + Function credentialsSupplier, URI requestURI, MultivaluedMap requestHeaders, MultivaluedMap queryParameters, String httpMethod, - String encodedPath, - String region, - String accessKey) + String encodedPath) { - // TODO - Credentials credentials = credentialsController.credentials(accessKey).orElseThrow(); + Credentials.Credential credential = credentialsSupplier.apply(metadata.credentials()); return Signer.sign( "s3", @@ -50,9 +93,9 @@ public String signRequest( queryParameters, httpMethod, encodedPath, - region, - accessKey, - credentials.emulated().secretKey(), + metadata.region(), + credential.accessKey(), + credential.secretKey(), Optional.empty()); } } diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SigningMetadata.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SigningMetadata.java new file mode 100644 index 00000000..2c49a579 --- /dev/null +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/credentials/SigningMetadata.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.s3.proxy.server.credentials; + +import static java.util.Objects.requireNonNull; + +public record SigningMetadata(Credentials credentials, String region) +{ + public SigningMetadata + { + requireNonNull(credentials, "credentials is null"); + requireNonNull(region, "region is null"); + } +} diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/StreamingResponseHandler.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/StreamingResponseHandler.java new file mode 100644 index 00000000..6c8defaf --- /dev/null +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/StreamingResponseHandler.java @@ -0,0 +1,92 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.s3.proxy.server.rest; + +import com.google.common.io.ByteStreams; +import io.airlift.http.client.HeaderName; +import io.airlift.http.client.Request; +import io.airlift.http.client.Response; +import io.airlift.http.client.ResponseHandler; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.container.AsyncResponse; +import jakarta.ws.rs.core.StreamingOutput; + +import java.io.InputStream; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import static io.airlift.http.client.ResponseHandlerUtils.propagate; +import static java.util.Objects.requireNonNull; + +class StreamingResponseHandler + implements ResponseHandler +{ + private final AsyncResponse asyncResponse; + private final Duration maxWaitForResponse; + + StreamingResponseHandler(AsyncResponse asyncResponse, Duration maxWaitForResponse) + { + this.asyncResponse = requireNonNull(asyncResponse, "asyncResponse is null"); + this.maxWaitForResponse = requireNonNull(maxWaitForResponse, "maxWaitForResponse is null"); + } + + @Override + public Void handleException(Request request, Exception exception) + throws RuntimeException + { + throw propagate(request, exception); + } + + @Override + public Void handle(Request request, Response response) + throws RuntimeException + { + CountDownLatch latch = new CountDownLatch(1); + StreamingOutput streamingOutput = output -> { + try { + InputStream inputStream = response.getInputStream(); + ByteStreams.copy(inputStream, output); + } + finally { + latch.countDown(); + } + }; + + jakarta.ws.rs.core.Response.ResponseBuilder responseBuilder = jakarta.ws.rs.core.Response.status(response.getStatusCode()).entity(streamingOutput); + response.getHeaders() + .keySet() + .stream() + .map(HeaderName::toString) + .forEach(name -> response.getHeaders(name).forEach(value -> responseBuilder.header(name, value))); + + try { + asyncResponse.resume(responseBuilder.build()); + } + catch (Exception e) { + throw new WebApplicationException(e, jakarta.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR); + } + try { + if (!latch.await(maxWaitForResponse.toMillis(), TimeUnit.MILLISECONDS)) { + throw new WebApplicationException(jakarta.ws.rs.core.Response.Status.REQUEST_TIMEOUT); + } + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new WebApplicationException(jakarta.ws.rs.core.Response.Status.SERVICE_UNAVAILABLE); + } + + return null; + } +} diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyClient.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyClient.java new file mode 100644 index 00000000..0fc46d2b --- /dev/null +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyClient.java @@ -0,0 +1,147 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.s3.proxy.server.rest; + +import com.google.inject.BindingAnnotation; +import com.google.inject.Inject; +import io.airlift.http.client.HttpClient; +import io.airlift.http.client.Request; +import io.trino.s3.proxy.server.credentials.Credentials; +import io.trino.s3.proxy.server.credentials.SigningController; +import io.trino.s3.proxy.server.credentials.SigningMetadata; +import jakarta.annotation.PreDestroy; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.container.AsyncResponse; +import jakarta.ws.rs.core.MultivaluedHashMap; +import jakarta.ws.rs.core.MultivaluedMap; +import jakarta.ws.rs.core.Response; +import org.glassfish.jersey.server.ContainerRequest; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; +import java.net.URI; +import java.time.Duration; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.util.concurrent.MoreExecutors.shutdownAndAwaitTermination; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; +import static java.util.Objects.requireNonNull; + +public class TrinoS3ProxyClient +{ + private final HttpClient httpClient; + private final SigningController signingController; + private final ExecutorService executorService = Executors.newVirtualThreadPerTaskExecutor(); + + @Retention(RUNTIME) + @Target({FIELD, PARAMETER, METHOD}) + @BindingAnnotation + public @interface ForProxyClient {} + + @Inject + public TrinoS3ProxyClient(@ForProxyClient HttpClient httpClient, SigningController signingController) + { + this.httpClient = requireNonNull(httpClient, "httpClient is null"); + this.signingController = requireNonNull(signingController, "signingController is null"); + } + + @PreDestroy + public void shutDown() + { + if (!shutdownAndAwaitTermination(executorService, Duration.ofSeconds(30))) { + // TODO add logging - check for false result + } + } + + public void proxyRequest(SigningMetadata signingMetadata, ContainerRequest request, AsyncResponse asyncResponse, String bucket) + { + String realPath = rewriteRequestPath(request, bucket); + + String realHost = buildRealHost(bucket, signingMetadata.region()); + URI realUri = request.getUriInfo().getRequestUriBuilder() + .host(realHost) + .port(-1) + .scheme("https") + .replacePath(realPath) + .build(); + + Request.Builder realRequestBuilder = new Request.Builder() + .setMethod(request.getMethod()) + .setUri(realUri) + .setFollowRedirects(true); + + if (realUri.getHost() == null) { + throw new WebApplicationException(Response.Status.BAD_REQUEST); + } + + // modify the source request headers for real values needed for the real AWS request + MultivaluedMap realRequestHeaders = new MultivaluedHashMap<>(request.getRequestHeaders()); + // we don't use sessions when making the real AWS call + realRequestHeaders.remove("X-Amz-Security-Token"); + // replace source host with the real AWS host + realRequestHeaders.putSingle("Host", realHost); + + // set the new signed request auth header + String encodedPath = firstNonNull(realUri.getRawPath(), ""); + String signature = signingController.signRequest( + signingMetadata, + Credentials::real, + realUri, + realRequestHeaders, + request.getUriInfo().getQueryParameters(), + request.getMethod(), + encodedPath); + realRequestHeaders.putSingle("Authorization", signature); + + // requestHeaders now has correct values, copy to the real request + realRequestHeaders.forEach((name, values) -> values.forEach(value -> realRequestBuilder.addHeader(name, value))); + + Request realRequest = realRequestBuilder.build(); + + // TODO config a max time to wait + executorService.submit(() -> httpClient.execute(realRequest, new StreamingResponseHandler(asyncResponse, Duration.ofHours(1)))); + } + + private static String rewriteRequestPath(ContainerRequest request, String bucket) + { + String path = "/" + request.getPath(false); + if (!path.startsWith(TrinoS3ProxyRestConstants.S3_PATH)) { + throw new WebApplicationException(Response.Status.BAD_REQUEST); + } + + path = path.substring(TrinoS3ProxyRestConstants.S3_PATH.length()); + if (path.startsWith("/" + bucket)) { + path = path.substring(("/" + bucket).length()); + } + + if (path.isEmpty() && bucket.isEmpty()) { + path = "/"; + } + + return path; + } + + private String buildRealHost(String bucket, String region) + { + if (bucket.isEmpty()) { + return "s3.%s.amazonaws.com".formatted(region); + } + return "%s.s3.%s.amazonaws.com".formatted(bucket, region); + } +} diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyResource.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyResource.java index ea81082a..40acdf49 100644 --- a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyResource.java +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyResource.java @@ -13,19 +13,80 @@ */ package io.trino.s3.proxy.server.rest; +import com.google.common.base.Splitter; +import com.google.inject.Inject; +import io.trino.s3.proxy.server.credentials.Credentials; +import io.trino.s3.proxy.server.credentials.SigningController; +import io.trino.s3.proxy.server.credentials.SigningMetadata; import jakarta.ws.rs.GET; +import jakarta.ws.rs.HEAD; import jakarta.ws.rs.Path; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.container.AsyncResponse; +import jakarta.ws.rs.container.Suspended; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; +import org.glassfish.jersey.server.ContainerRequest; -@Path(TrinoS3ProxyRestConstants.BASE_PATH) +import java.util.List; + +import static java.util.Objects.requireNonNull; + +@Path(TrinoS3ProxyRestConstants.S3_PATH) public class TrinoS3ProxyResource { + private final SigningController signingController; + private final TrinoS3ProxyClient proxyClient; + + @Inject + public TrinoS3ProxyResource(SigningController signingController, TrinoS3ProxyClient proxyClient) + { + this.signingController = requireNonNull(signingController, "signingController is null"); + this.proxyClient = requireNonNull(proxyClient, "proxyClient is null"); + } + @GET - @Path("hello") - @Produces(MediaType.APPLICATION_JSON) - public String hello() + public void s3Get(@Context ContainerRequest request, @Suspended AsyncResponse asyncResponse) + { + s3Get(request, asyncResponse, "/"); + } + + @GET + @Path("{path:.*}") + public void s3Get(@Context ContainerRequest request, @Suspended AsyncResponse asyncResponse, @PathParam("path") String path) + { + proxyClient.proxyRequest(validateRequest(request), request, asyncResponse, getBucket(path)); + } + + @HEAD + public void s3Head(@Context ContainerRequest request, @Suspended AsyncResponse asyncResponse) + { + s3Head(request, asyncResponse, "/"); + } + + @HEAD + @Path("{path:.*}") + public void s3Head(@Context ContainerRequest request, @Suspended AsyncResponse asyncResponse, @PathParam("path") String path) + { + proxyClient.proxyRequest(validateRequest(request), request, asyncResponse, getBucket(path)); + } + + private String getBucket(String path) + { + List parts = Splitter.on("/").splitToList(path); + return parts.isEmpty() ? "" : parts.getFirst(); + } + + private SigningMetadata validateRequest(ContainerRequest request) { - return "hello"; + return signingController.signingMetadataFromRequest( + Credentials::emulated, + request.getRequestUri(), + request.getRequestHeaders(), + request.getUriInfo().getQueryParameters(), + request.getMethod(), + request.getPath(false)) + .orElseThrow(() -> new WebApplicationException(Response.Status.UNAUTHORIZED)); } } diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyRestConstants.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyRestConstants.java index de304d0f..c00ade22 100644 --- a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyRestConstants.java +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyRestConstants.java @@ -18,4 +18,5 @@ public final class TrinoS3ProxyRestConstants private TrinoS3ProxyRestConstants() {} public static final String BASE_PATH = "/api/v1/s3Proxy/"; + public static final String S3_PATH = BASE_PATH + "s3"; } diff --git a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingCredentialsController.java b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingCredentialsController.java new file mode 100644 index 00000000..ed592705 --- /dev/null +++ b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingCredentialsController.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.s3.proxy.server; + +import io.trino.s3.proxy.server.credentials.Credentials; +import io.trino.s3.proxy.server.credentials.CredentialsController; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +public class TestingCredentialsController + implements CredentialsController +{ + private final Map credentials = new ConcurrentHashMap<>(); + + @Override + public Optional credentials(String emulatedAccessKey) + { + return Optional.ofNullable(credentials.get(emulatedAccessKey)); + } + + @Override + public void upsertCredentials(Credentials credentials) + { + this.credentials.put(credentials.emulated().accessKey(), credentials); + } +} diff --git a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServer.java b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServer.java index 069e8282..0a344740 100644 --- a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServer.java +++ b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServer.java @@ -14,14 +14,18 @@ package io.trino.s3.proxy.server; import com.google.common.collect.ImmutableList; +import com.google.inject.Injector; import com.google.inject.Module; import io.airlift.bootstrap.Bootstrap; import io.airlift.event.client.EventModule; +import io.airlift.http.server.testing.TestingHttpServer; import io.airlift.http.server.testing.TestingHttpServerModule; import io.airlift.jaxrs.JaxrsModule; import io.airlift.json.JsonModule; import io.airlift.log.Logger; import io.airlift.node.testing.TestingNodeModule; +import io.trino.s3.proxy.server.credentials.Credentials; +import io.trino.s3.proxy.server.credentials.Credentials.Credential; public final class TestingTrinoS3ProxyServer { @@ -31,8 +35,18 @@ private TestingTrinoS3ProxyServer() {} public static void main(String[] args) { + if (args.length != 4) { + System.err.println("Usage: TestingTrinoS3ProxyServer "); + System.exit(1); + } + + String emulatedAccessKey = args[0]; + String emulatedSecretKey = args[1]; + String realAccessKey = args[2]; + String realSecretKey = args[3]; + ImmutableList.Builder modules = ImmutableList.builder() - .add(new TrinoS3ProxyServerModule()) + .add(new TestingTrinoS3ProxyServerModule()) .add(new TestingNodeModule()) .add(new EventModule()) .add(new TestingHttpServerModule()) @@ -40,8 +54,16 @@ public static void main(String[] args) .add(new JaxrsModule()); Bootstrap app = new Bootstrap(modules.build()); - app.initialize(); + Injector injector = app.initialize(); + + TestingCredentialsController credentialsController = injector.getInstance(TestingCredentialsController.class); + credentialsController.upsertCredentials(new Credentials(new Credential(emulatedAccessKey, emulatedSecretKey), new Credential(realAccessKey, realSecretKey))); log.info("======== TESTING SERVER STARTED ========"); + + TestingHttpServer httpServer = injector.getInstance(TestingHttpServer.class); + log.info(""); + log.info("Endpoint: %s", httpServer.getBaseUrl()); + log.info(""); } } diff --git a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServerModule.java b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServerModule.java new file mode 100644 index 00000000..ea9f3c86 --- /dev/null +++ b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServerModule.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.s3.proxy.server; + +import com.google.inject.Binder; +import com.google.inject.Scopes; +import io.trino.s3.proxy.server.credentials.CredentialsController; + +public class TestingTrinoS3ProxyServerModule + extends TrinoS3ProxyServerModule +{ + @Override + protected void moduleSpecificBinding(Binder binder) + { + binder.bind(CredentialsController.class).to(TestingCredentialsController.class).in(Scopes.SINGLETON); + binder.bind(TestingCredentialsController.class).in(Scopes.SINGLETON); + } +} diff --git a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/credentials/TestSigningController.java b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/credentials/TestSigningController.java index feeb9ee5..0e95a3d1 100644 --- a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/credentials/TestSigningController.java +++ b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/credentials/TestSigningController.java @@ -27,10 +27,24 @@ public class TestSigningController { private static final Credentials CREDENTIALS = new Credentials(new Credential("THIS_IS_AN_ACCESS_KEY", "THIS_IS_A_SECRET_KEY"), new Credential("dummy", "dummy")); + private final CredentialsController credentialsController = new CredentialsController() + { + @Override + public Optional credentials(String emulatedAccessKey) + { + return Optional.of(CREDENTIALS); + } + + @Override + public void upsertCredentials(Credentials credentials) + { + throw new UnsupportedOperationException(); + } + }; + @Test public void testRootLs() { - CredentialsController credentialsController = accessKey -> Optional.of(CREDENTIALS); SigningController signingController = new SigningController(credentialsController); // values discovered from an AWS CLI request sent to a dummy local HTTP server @@ -43,13 +57,13 @@ public void testRootLs() requestHeaders.putSingle("Accept-Encoding", "identity"); String signature = signingController.signRequest( + new SigningMetadata(CREDENTIALS, "us-east-1"), + Credentials::emulated, URI.create("http://localhost:10064"), requestHeaders, new MultivaluedHashMap<>(), "GET", - "/", - "us-east-1", - "THIS_IS_AN_ACCESS_KEY"); + "/"); assertThat(signature).isEqualTo("AWS4-HMAC-SHA256 Credential=THIS_IS_AN_ACCESS_KEY/20240516/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=9a19c251bf4e1533174e80da59fa57c65b3149b611ec9a4104f6944767c25704"); } @@ -57,7 +71,6 @@ public void testRootLs() @Test public void testBucketLs() { - CredentialsController credentialsController = accessKey -> Optional.of(CREDENTIALS); SigningController signingController = new SigningController(credentialsController); // values discovered from an AWS CLI request sent to a dummy local HTTP server @@ -76,13 +89,13 @@ public void testBucketLs() queryParameters.putSingle("encoding-type", "url"); String signature = signingController.signRequest( + new SigningMetadata(CREDENTIALS, "us-east-1"), + Credentials::emulated, URI.create("http://localhost:10064"), requestHeaders, queryParameters, "GET", - "/mybucket", - "us-east-1", - "THIS_IS_AN_ACCESS_KEY"); + "/mybucket"); assertThat(signature).isEqualTo("AWS4-HMAC-SHA256 Credential=THIS_IS_AN_ACCESS_KEY/20240516/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=222d7b7fcd4d5560c944e8fecd9424ee3915d131c3ad9e000d65db93e87946c4"); }