diff --git a/README.md b/README.md index acf5be91..baef632e 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,26 @@ Proxy for S3 To run testing server: ```shell -./mvnw package -./mvnw -pl trino-s3-proxy exec:java -Dexec.classpathScope=test -Dexec.mainClass=io.trino.s3.proxy.server.TestingTrinoS3ProxyServer +./mvnw clean package +./mvnw -pl trino-s3-proxy exec:java -Dexec.classpathScope=test -Dexec.mainClass=io.trino.s3.proxy.server.TestingTrinoS3ProxyServer +``` + +Make note of the logged `Endpoint` + +To test out proxied APIs: + +First, set up AWS CLI: + +```shell +> aws configure +AWS Access Key ID [None]: enter the fake access key +AWS Secret Access Key [None]: enter the fake secret key +Default region name [None]: +Default output format [None]: +``` + +Now, make calls. E.g. + +```shell +aws --endpoint-url http:///api/v1/s3Proxy/s3 s3 ls s3:/// ``` diff --git a/trino-s3-proxy/pom.xml b/trino-s3-proxy/pom.xml index de17d856..1c911046 100644 --- a/trino-s3-proxy/pom.xml +++ b/trino-s3-proxy/pom.xml @@ -34,6 +34,11 @@ event + + io.airlift + http-client + + io.airlift http-server diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/BytesResponseHandler.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/BytesResponseHandler.java new file mode 100644 index 00000000..f03d425f --- /dev/null +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/BytesResponseHandler.java @@ -0,0 +1,58 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.s3.proxy.server; + +import io.airlift.http.client.HttpStatus; +import io.airlift.http.client.Request; +import io.airlift.http.client.Response; +import io.airlift.http.client.ResponseHandler; +import jakarta.ws.rs.WebApplicationException; + +import static io.airlift.http.client.ResponseHandlerUtils.propagate; +import static io.airlift.http.client.ResponseHandlerUtils.readResponseBytes; +import static java.util.Objects.requireNonNull; + +public class BytesResponseHandler + implements ResponseHandler +{ + @Override + public BytesResponse handleException(Request request, Exception exception) + throws RuntimeException + { + throw propagate(request, exception); + } + + public record BytesResponse(Response response, byte[] entity) + { + public BytesResponse + { + requireNonNull(response, "response is null"); + requireNonNull(entity, "entity is null"); + } + } + + @Override + public BytesResponse handle(Request request, Response response) + throws RuntimeException + { + if (HttpStatus.familyForStatusCode(response.getStatusCode()) != HttpStatus.Family.SUCCESSFUL) { + // TODO + throw new WebApplicationException(response.getStatusCode()); + } + + byte[] bytes = readResponseBytes(request, response); + + return new BytesResponse(response, bytes); + } +} diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/Credentials.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/Credentials.java index 81a69028..718d3d2f 100644 --- a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/Credentials.java +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/Credentials.java @@ -15,11 +15,12 @@ import static java.util.Objects.requireNonNull; -public record Credentials(CredentialsEntry emulated) +public record Credentials(CredentialsEntry emulated, CredentialsEntry real) { public Credentials { requireNonNull(emulated, "emulated is null"); + requireNonNull(real, "real is null"); } public record CredentialsEntry(String accessKey, String secretKey) diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/ProxyClient.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/ProxyClient.java new file mode 100644 index 00000000..eed6a291 --- /dev/null +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/ProxyClient.java @@ -0,0 +1,117 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.s3.proxy.server; + +import com.google.inject.BindingAnnotation; +import com.google.inject.Inject; +import io.airlift.http.client.HeaderName; +import io.airlift.http.client.HttpClient; +import io.airlift.http.client.Request; +import io.trino.s3.proxy.server.BytesResponseHandler.BytesResponse; +import io.trino.s3.proxy.server.Credentials.CredentialsEntry; +import io.trino.s3.proxy.server.SigningController.Scope; +import io.trino.s3.proxy.server.rest.TrinoS3ProxyRestConstants; +import jakarta.ws.rs.WebApplicationException; +import jakarta.ws.rs.core.MultivaluedHashMap; +import jakarta.ws.rs.core.MultivaluedMap; +import jakarta.ws.rs.core.Response; +import org.glassfish.jersey.server.ContainerRequest; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; +import java.net.URI; +import java.util.Map; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; +import static java.util.Objects.requireNonNull; + +public class ProxyClient +{ + private final HttpClient httpClient; + private final SigningController signingController; + private final CredentialsController credentialsController; + + @Retention(RUNTIME) + @Target({FIELD, PARAMETER, METHOD}) + @BindingAnnotation + public @interface ForProxyClient {} + + @Inject + public ProxyClient(@ForProxyClient HttpClient httpClient, SigningController signingController, CredentialsController credentialsController) + { + this.httpClient = requireNonNull(httpClient, "httpClient is null"); + this.signingController = requireNonNull(signingController, "signingController is null"); + this.credentialsController = requireNonNull(credentialsController, "credentialsController is null"); + } + + public Response proxyRequest(ContainerRequest request, String bucket) + { + String fullPath = "/" + request.getPath(false); + if (!fullPath.startsWith(TrinoS3ProxyRestConstants.S3_PATH)) { + throw new WebApplicationException(Response.Status.BAD_REQUEST); + } + fullPath = fullPath.substring(TrinoS3ProxyRestConstants.S3_PATH.length()); + if (fullPath.startsWith("/" + bucket)) { + fullPath = fullPath.substring(("/" + bucket).length()); + } + + Scope scope = Scope.fromHeaders(request.getHeaders()).orElseThrow(() -> new WebApplicationException(Response.Status.BAD_REQUEST)); + + String host = buildHost(bucket, scope.region()); + URI uri = request.getUriInfo().getRequestUriBuilder() + .host(host) + .port(-1) + .scheme("https") + .replacePath(fullPath) + .build(); + + Request.Builder requestBuilder = new Request.Builder() + .setMethod(request.getMethod()) + .setUri(uri) + .setFollowRedirects(true); + + // TODO + CredentialsEntry credentials = credentialsController.credentials(scope.accessKey()).orElseThrow().real(); + + MultivaluedMap requestHeaders = new MultivaluedHashMap<>(request.getRequestHeaders()); + requestHeaders.remove("X-Amz-Security-Token"); + requestHeaders.putSingle("Host", host); + + Map signedRequestHeaders = signingController.signedRequestHeaders(scope, credentials, request.getMethod(), requestHeaders, uri.getRawPath(), uri.getRawQuery()); + signedRequestHeaders.forEach(requestBuilder::addHeader); + + requestHeaders.forEach((name, values) -> { + if (!signedRequestHeaders.containsKey(name)) { + values.forEach(value -> requestBuilder.addHeader(name, value)); + } + }); + + BytesResponse bytesResponse = httpClient.execute(requestBuilder.build(), new BytesResponseHandler()); + Response.ResponseBuilder responseBuilder = Response.status(bytesResponse.response().getStatusCode()).entity(bytesResponse.entity()); + bytesResponse.response().getHeaders() + .keySet() + .stream() + .map(HeaderName::toString) + .forEach(name -> bytesResponse.response().getHeaders(name).forEach(value -> responseBuilder.header(name, value))); + return responseBuilder.build(); + } + + private String buildHost(String bucket, String region) + { + return "%s.s3.%s.amazonaws.com".formatted(bucket, region); + } +} diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/SigningController.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/SigningController.java index dab76e4c..dd5e2e7d 100644 --- a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/SigningController.java +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/SigningController.java @@ -15,6 +15,7 @@ import com.google.common.base.Splitter; import com.google.inject.Inject; +import io.trino.s3.proxy.server.Credentials.CredentialsEntry; import io.trino.s3.proxy.server.minio.Signer; import io.trino.s3.proxy.server.minio.emulation.MinioRequest; import io.trino.s3.proxy.server.minio.emulation.MinioUrl; @@ -80,17 +81,16 @@ public static Optional fromHeaders(MultivaluedMap request public boolean validateRequest(String method, MultivaluedMap requestHeaders, String encodedPath, String encodedQuery) { return Scope.fromHeaders(requestHeaders).map(scope -> { - Map signedRequestHeaders = signedRequestHeaders(scope, method, requestHeaders, encodedPath, encodedQuery); + // TODO + CredentialsEntry credentials = credentialsController.credentials(scope.accessKey).orElseThrow().emulated(); + Map signedRequestHeaders = signedRequestHeaders(scope, credentials, method, requestHeaders, encodedPath, encodedQuery); String requestAuthorization = signedRequestHeaders.get("Authorization"); return scope.authorization.equals(requestAuthorization); }).orElse(false); } - public Map signedRequestHeaders(Scope scope, String method, MultivaluedMap requestHeaders, String encodedPath, String encodedQuery) + public Map signedRequestHeaders(Scope scope, CredentialsEntry credentials, String method, MultivaluedMap requestHeaders, String encodedPath, String encodedQuery) { - // TODO - Credentials credentials = credentialsController.credentials(scope.accessKey).orElseThrow(); - MinioUrl minioUrl = MinioUrl.build(encodedPath, encodedQuery); MinioRequest minioRequest = MinioRequest.build(requestHeaders, method, minioUrl); @@ -98,7 +98,7 @@ public Map signedRequestHeaders(Scope scope, String method, Mult String sha256 = minioRequest.headerValue("x-amz-content-sha256").orElseThrow(); try { - return Signer.signV4S3(minioRequest, scope.region, scope.accessKey, credentials.emulated().secretKey(), sha256).headers(); + return Signer.signV4S3(minioRequest, scope.region, credentials.accessKey(), credentials.secretKey(), sha256).headers(); } catch (NoSuchAlgorithmException | InvalidKeyException e) { // TODO diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServerModule.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServerModule.java index d32a9b97..bef9d477 100644 --- a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServerModule.java +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServerModule.java @@ -18,16 +18,28 @@ import com.google.inject.Scopes; import io.trino.s3.proxy.server.rest.TrinoS3ProxyResource; +import static io.airlift.http.client.HttpClientBinder.httpClientBinder; import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; public class TrinoS3ProxyServerModule implements Module { @Override - public void configure(Binder binder) + public final void configure(Binder binder) { jaxrsBinder(binder).bind(TrinoS3ProxyResource.class); binder.bind(SigningController.class).in(Scopes.SINGLETON); + + // TODO config, etc. + httpClientBinder(binder).bindHttpClient("ProxyClient", ProxyClient.ForProxyClient.class); + binder.bind(ProxyClient.class).in(Scopes.SINGLETON); + + moduleSpecificBinding(binder); + } + + protected void moduleSpecificBinding(Binder binder) + { + // default does nothing } } diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyResource.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyResource.java index 27208247..4371321c 100644 --- a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyResource.java +++ b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyResource.java @@ -14,6 +14,7 @@ package io.trino.s3.proxy.server.rest; import com.google.inject.Inject; +import io.trino.s3.proxy.server.ProxyClient; import io.trino.s3.proxy.server.SigningController; import jakarta.ws.rs.Consumes; import jakarta.ws.rs.GET; @@ -34,11 +35,13 @@ public class TrinoS3ProxyResource { private final SigningController signingController; + private final ProxyClient proxyClient; @Inject - public TrinoS3ProxyResource(SigningController signingController) + public TrinoS3ProxyResource(SigningController signingController, ProxyClient proxyClient) { this.signingController = requireNonNull(signingController, "signingController is null"); + this.proxyClient = requireNonNull(proxyClient, "proxyClient is null"); } @GET @@ -56,7 +59,7 @@ public Response s3Get(@Context ContainerRequest request) public Response s3Get(@Context ContainerRequest request, @PathParam("bucket") String bucket) { validateRequest(request); - return Response.ok().build(); + return proxyClient.proxyRequest(request, bucket); } @HEAD diff --git a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestSigningController.java b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestSigningController.java index 7954b5ce..7b0f50f0 100644 --- a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestSigningController.java +++ b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestSigningController.java @@ -26,7 +26,7 @@ public class TestSigningController { - private static final Credentials CREDENTIALS = new Credentials(new CredentialsEntry("THIS_IS_AN_ACCESS_KEY", "THIS_IS_A_SECRET_KEY")); + private static final Credentials CREDENTIALS = new Credentials(new CredentialsEntry("THIS_IS_AN_ACCESS_KEY", "THIS_IS_A_SECRET_KEY"), new CredentialsEntry("dummy", "dummy")); private final CredentialsController credentialsController = new CredentialsController() { @@ -57,7 +57,7 @@ public void testRootLs() requestHeaders.putSingle("User-Agent", "aws-cli/2.15.16 Python/3.11.7 Darwin/22.6.0 source/x86_64 prompt/off command/s3.ls"); requestHeaders.putSingle("Accept-Encoding", "identity"); - Map signedHeaders = signingController.signedRequestHeaders(new Scope("dummy", "THIS_IS_AN_ACCESS_KEY", "us-east-1"), "GET", requestHeaders, "/", ""); + Map signedHeaders = signingController.signedRequestHeaders(new Scope("dummy", "THIS_IS_AN_ACCESS_KEY", "us-east-1"), CREDENTIALS.emulated(), "GET", requestHeaders, "/", ""); assertThat(signedHeaders).contains(Map.entry("Authorization", "AWS4-HMAC-SHA256 Credential=THIS_IS_AN_ACCESS_KEY/20240516/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=9a19c251bf4e1533174e80da59fa57c65b3149b611ec9a4104f6944767c25704")); } @@ -76,7 +76,7 @@ public void testBucketLs() requestHeaders.putSingle("User-Agent", "aws-cli/2.15.16 Python/3.11.7 Darwin/22.6.0 source/x86_64 prompt/off command/s3.ls"); requestHeaders.putSingle("Accept-Encoding", "identity"); - Map signedHeaders = signingController.signedRequestHeaders(new Scope("dummy", "THIS_IS_AN_ACCESS_KEY", "us-east-1"), "GET", requestHeaders, "/mybucket", "list-type=2&prefix=foo%2Fbar&delimiter=%2F&encoding-type=url"); + Map signedHeaders = signingController.signedRequestHeaders(new Scope("dummy", "THIS_IS_AN_ACCESS_KEY", "us-east-1"), CREDENTIALS.emulated(), "GET", requestHeaders, "/mybucket", "list-type=2&prefix=foo%2Fbar&delimiter=%2F&encoding-type=url"); assertThat(signedHeaders).contains(Map.entry("Authorization", "AWS4-HMAC-SHA256 Credential=THIS_IS_AN_ACCESS_KEY/20240516/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=222d7b7fcd4d5560c944e8fecd9424ee3915d131c3ad9e000d65db93e87946c4")); } diff --git a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServer.java b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServer.java index 5712ba92..b8500838 100644 --- a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServer.java +++ b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServer.java @@ -16,9 +16,9 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Injector; import com.google.inject.Module; -import com.google.inject.Scopes; import io.airlift.bootstrap.Bootstrap; import io.airlift.event.client.EventModule; +import io.airlift.http.server.testing.TestingHttpServer; import io.airlift.http.server.testing.TestingHttpServerModule; import io.airlift.jaxrs.JaxrsModule; import io.airlift.json.JsonModule; @@ -34,24 +34,35 @@ private TestingTrinoS3ProxyServer() {} public static void main(String[] args) { + if (args.length != 4) { + System.err.println("Usage: TestingTrinoS3ProxyServer "); + System.exit(1); + } + + String emulatedAccessKey = args[0]; + String emulatedSecretKey = args[1]; + String realAccessKey = args[2]; + String realSecretKey = args[3]; + ImmutableList.Builder modules = ImmutableList.builder() - .add(new TrinoS3ProxyServerModule()) + .add(new TestingTrinoS3ProxyServerModule()) .add(new TestingNodeModule()) .add(new EventModule()) .add(new TestingHttpServerModule()) .add(new JsonModule()) - .add(new JaxrsModule()) - .add(binder -> { - binder.bind(CredentialsController.class).to(TestingCredentialsController.class).in(Scopes.SINGLETON); - binder.bind(TestingCredentialsController.class).in(Scopes.SINGLETON); - }); + .add(new JaxrsModule()); Bootstrap app = new Bootstrap(modules.build()); Injector injector = app.initialize(); TestingCredentialsController credentialsController = injector.getInstance(TestingCredentialsController.class); - credentialsController.upsertCredentials(new Credentials(new CredentialsEntry("THIS_IS_AN_ACCESS_KEY", "THIS_IS_A_SECRET_KEY"))); + credentialsController.upsertCredentials(new Credentials(new CredentialsEntry(emulatedAccessKey, emulatedSecretKey), new CredentialsEntry(realAccessKey, realSecretKey))); log.info("======== TESTING SERVER STARTED ========"); + + TestingHttpServer httpServer = injector.getInstance(TestingHttpServer.class); + log.info(""); + log.info("Endpoint: %s", httpServer.getBaseUrl()); + log.info(""); } } diff --git a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServerModule.java b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServerModule.java new file mode 100644 index 00000000..0c4c4b0a --- /dev/null +++ b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestingTrinoS3ProxyServerModule.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.s3.proxy.server; + +import com.google.inject.Binder; +import com.google.inject.Scopes; + +public class TestingTrinoS3ProxyServerModule + extends TrinoS3ProxyServerModule +{ + @Override + protected void moduleSpecificBinding(Binder binder) + { + binder.bind(CredentialsController.class).to(TestingCredentialsController.class).in(Scopes.SINGLETON); + binder.bind(TestingCredentialsController.class).in(Scopes.SINGLETON); + } +}