Support custom prefixes for S3 and STS

Closes trinodb#57
vagaerg · Jun 20, 2024 · ffe546e · ffe546e
1 parent 6cf3a7a
commit ffe546e
Show file tree

Hide file tree

Showing 21 changed files with 261 additions and 115 deletions.
diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServerModule.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/TrinoS3ProxyServerModule.java
@@ -35,6 +35,7 @@
 import io.trino.s3.proxy.server.security.SecurityResponse;
 import io.trino.s3.proxy.server.signing.SigningController;
 import io.trino.s3.proxy.server.signing.SigningControllerConfig;
+import org.glassfish.jersey.server.model.Resource;
 
 import java.util.Optional;
 import java.util.ServiceLoader;
@@ -52,11 +53,14 @@ public class TrinoS3ProxyServerModule
     @Override
     protected void setup(Binder binder)
     {
+        configBinder(binder).bindConfig(SigningControllerConfig.class);
+        TrinoS3ProxyConfig builtConfig = buildConfigObject(TrinoS3ProxyConfig.class);
+
         jaxrsBinder(binder).bind(TrinoS3ProxyResource.class);
+        jaxrsBinder(binder).bindInstance(buildResourceAtPath(TrinoS3ProxyResource.class, builtConfig.getS3Path()));
         jaxrsBinder(binder).bind(TrinoStsResource.class);
+        jaxrsBinder(binder).bindInstance(buildResourceAtPath(TrinoStsResource.class, builtConfig.getStsPath()));
 
-        configBinder(binder).bindConfig(SigningControllerConfig.class);
-        configBinder(binder).bindConfig(TrinoS3ProxyConfig.class);
         binder.bind(SigningController.class).in(Scopes.SINGLETON);
         binder.bind(CredentialsController.class).in(Scopes.SINGLETON);
         binder.bind(SecurityController.class).in(Scopes.SINGLETON);
@@ -96,4 +100,9 @@ private void installPlugins()
                     install(plugin.module());
                 });
     }
+
+    private static Resource buildResourceAtPath(Class<?> resourceClass, String resourcePathPrefix)
+    {
+        return Resource.builder(resourceClass).path(resourcePathPrefix).build();
+    }
 }
diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyConfig.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyConfig.java
@@ -21,19 +21,49 @@
 
 public class TrinoS3ProxyConfig
 {
-    private Optional<String> hostName = Optional.empty();
+    private Optional<String> s3HostName = Optional.empty();
+    private String s3Path = "/api/v1/s3Proxy/s3";
+    private String stsPath = "/api/v1/s3Proxy/sts";
 
-    @Config("s3proxy.hostname")
-    @ConfigDescription("Hostname to use for REST operations, virtual-host style addressing is only supported if this is set")
-    public TrinoS3ProxyConfig setHostName(String hostName)
+    @Config("s3proxy.s3.hostname")
+    @ConfigDescription("Hostname to use for S3 REST operations, virtual-host style addressing is only supported if this is set")
+    public TrinoS3ProxyConfig setS3HostName(String s3HostName)
     {
-        this.hostName = Optional.ofNullable(hostName);
+        this.s3HostName = Optional.ofNullable(s3HostName);
         return this;
     }
 
     @NotNull
-    public Optional<String> getHostName()
+    public Optional<String> getS3HostName()
     {
-        return hostName;
+        return s3HostName;
+    }
+
+    @Config("s3proxy.s3.path")
+    @ConfigDescription("URL Path for S3 operations, optional")
+    public TrinoS3ProxyConfig setS3Path(String s3Path)
+    {
+        this.s3Path = s3Path;
+        return this;
+    }
+
+    @NotNull
+    public String getS3Path()
+    {
+        return s3Path;
+    }
+
+    @Config("s3proxy.sts.path")
+    @ConfigDescription("URL Path for STS operations, optional")
+    public TrinoS3ProxyConfig setStsPath(String stsPath)
+    {
+        this.stsPath = stsPath;
+        return this;
+    }
+
+    @NotNull
+    public String getStsPath()
+    {
+        return stsPath;
     }
 }
diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyResource.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyResource.java
@@ -33,7 +33,6 @@
 import static io.trino.s3.proxy.server.rest.RequestBuilder.fromRequest;
 import static java.util.Objects.requireNonNull;
 
-@Path(TrinoS3ProxyRestConstants.S3_PATH)
 public class TrinoS3ProxyResource
 {
     private final SigningController signingController;
@@ -45,7 +44,7 @@ public TrinoS3ProxyResource(SigningController signingController, TrinoS3ProxyCli
     {
         this.signingController = requireNonNull(signingController, "signingController is null");
         this.proxyClient = requireNonNull(proxyClient, "proxyClient is null");
-        this.serverHostName = requireNonNull(trinoS3ProxyConfig, "restConfig is null").getHostName();
+        this.serverHostName = trinoS3ProxyConfig.getS3HostName();
     }
 
     @GET

diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyRestConstants.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoS3ProxyRestConstants.java
diff --git a/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoStsResource.java b/trino-s3-proxy/src/main/java/io/trino/s3/proxy/server/rest/TrinoStsResource.java
@@ -25,7 +25,6 @@
 import io.trino.s3.proxy.server.signing.SigningMetadata;
 import io.trino.s3.proxy.server.signing.SigningServiceType;
 import jakarta.ws.rs.POST;
-import jakarta.ws.rs.Path;
 import jakarta.ws.rs.WebApplicationException;
 import jakarta.ws.rs.core.Context;
 import jakarta.ws.rs.core.MediaType;
@@ -43,7 +42,6 @@
 import static io.trino.s3.proxy.server.signing.SigningController.formatResponseInstant;
 import static java.util.Objects.requireNonNull;
 
-@Path(TrinoS3ProxyRestConstants.STS_PATH)
 public class TrinoStsResource
 {
     private static final Logger log = Logger.get(TrinoStsResource.class);

diff --git a/...rino/s3/proxy/server/TestStsRequests.java → ...proxy/server/AbstractTestStsRequests.java b/...rino/s3/proxy/server/TestStsRequests.java → ...proxy/server/AbstractTestStsRequests.java
@@ -13,14 +13,11 @@
  */
 package io.trino.s3.proxy.server;
 
-import com.google.inject.Inject;
 import io.airlift.http.server.testing.TestingHttpServer;
 import io.trino.s3.proxy.server.credentials.Credential;
 import io.trino.s3.proxy.server.credentials.Credentials;
 import io.trino.s3.proxy.server.credentials.CredentialsProvider;
-import io.trino.s3.proxy.server.rest.TrinoS3ProxyRestConstants;
-import io.trino.s3.proxy.server.testing.TestingUtil.ForTesting;
-import io.trino.s3.proxy.server.testing.harness.TrinoS3ProxyTest;
+import io.trino.s3.proxy.server.rest.TrinoS3ProxyConfig;
 import org.junit.jupiter.api.Test;
 import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
 import software.amazon.awssdk.endpoints.Endpoint;
@@ -36,18 +33,16 @@
 import static java.util.concurrent.CompletableFuture.completedFuture;
 import static org.assertj.core.api.Assertions.assertThat;
 
-@TrinoS3ProxyTest
-public class TestStsRequests
+public abstract class AbstractTestStsRequests
 {
     private final StsClient stsClient;
     private final CredentialsProvider credentialsProvider;
 
-    @Inject
-    public TestStsRequests(@ForTesting Credentials testingCredentials, TestingHttpServer httpServer, CredentialsProvider credentialsProvider)
+    public AbstractTestStsRequests(Credentials testingCredentials, TestingHttpServer httpServer, CredentialsProvider credentialsProvider, TrinoS3ProxyConfig s3ProxyConfig)
     {
         this.credentialsProvider = requireNonNull(credentialsProvider, "credentialsProvider is null");
 
-        URI localProxyServerUri = httpServer.getBaseUrl().resolve(TrinoS3ProxyRestConstants.STS_PATH);
+        URI localProxyServerUri = httpServer.getBaseUrl().resolve(s3ProxyConfig.getStsPath());
         Endpoint endpoint = Endpoint.builder().url(localProxyServerUri).build();
 
         stsClient = StsClient.builder()

diff --git a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/LocalServer.java b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/LocalServer.java
@@ -17,7 +17,7 @@
 import io.airlift.http.server.testing.TestingHttpServer;
 import io.airlift.log.Logger;
 import io.trino.s3.proxy.server.credentials.Credentials;
-import io.trino.s3.proxy.server.rest.TrinoS3ProxyRestConstants;
+import io.trino.s3.proxy.server.rest.TrinoS3ProxyConfig;
 import io.trino.s3.proxy.server.testing.TestingTrinoS3ProxyServer;
 import io.trino.s3.proxy.server.testing.TestingUtil.ForTesting;
 
@@ -38,9 +38,10 @@ public static void main(String[] args)
 
         TestingHttpServer httpServer = trinoS3ProxyServer.getInjector().getInstance(TestingHttpServer.class);
         Credentials testingCredentials = trinoS3ProxyServer.getInjector().getInstance(Key.get(Credentials.class, ForTesting.class));
+        TrinoS3ProxyConfig s3ProxyConfig = trinoS3ProxyServer.getInjector().getInstance(TrinoS3ProxyConfig.class);
 
         log.info("");
-        log.info("Endpoint:   %s", httpServer.getBaseUrl().resolve(TrinoS3ProxyRestConstants.S3_PATH));
+        log.info("Endpoint:   %s", httpServer.getBaseUrl().resolve(s3ProxyConfig.getS3Path()));
         log.info("Access Key: %s", testingCredentials.emulated().accessKey());
         log.info("Secret Key: %s", testingCredentials.emulated().secretKey());
         log.info("");

diff --git a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestGenericRestRequests.java b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestGenericRestRequests.java
@@ -21,7 +21,7 @@
 import io.airlift.units.Duration;
 import io.trino.s3.proxy.server.credentials.Credential;
 import io.trino.s3.proxy.server.credentials.Credentials;
-import io.trino.s3.proxy.server.rest.TrinoS3ProxyRestConstants;
+import io.trino.s3.proxy.server.rest.TrinoS3ProxyConfig;
 import io.trino.s3.proxy.server.testing.ManagedS3MockContainer.ForS3MockContainer;
 import io.trino.s3.proxy.server.testing.TestingCredentialsRolesProvider;
 import io.trino.s3.proxy.server.testing.TestingTrinoS3ProxyServer;
@@ -100,9 +100,10 @@ public TestGenericRestRequests(
             TestingCredentialsRolesProvider credentialsRolesProvider,
             @ForTesting HttpClient httpClient,
             @ForTesting Credentials testingCredentials,
-            @ForS3MockContainer S3Client storageClient)
+            @ForS3MockContainer S3Client storageClient,
+            TrinoS3ProxyConfig trinoS3ProxyConfig)
     {
-        baseUri = httpServer.getBaseUrl();
+        baseUri = httpServer.getBaseUrl().resolve(trinoS3ProxyConfig.getS3Path());
         this.credentialsRolesProvider = requireNonNull(credentialsRolesProvider, "credentialsRolesProvider is null");
         this.httpClient = requireNonNull(httpClient, "httpClient is null");
         this.testingCredentials = requireNonNull(testingCredentials, "testingCredentials is null");
@@ -141,7 +142,6 @@ public void testPutObject()
     private StatusResponse doPutObject(String content, String sha256)
     {
         URI uri = UriBuilder.fromUri(baseUri)
-                .replacePath(TrinoS3ProxyRestConstants.S3_PATH)
                 .path("foo")
                 .path("bar")
                 .build();
@@ -167,7 +167,6 @@ private StatusResponse doPutObject(String content, String sha256)
     private StatusResponse doAwsChunkedUpload(String content)
     {
         URI uri = UriBuilder.fromUri(baseUri)
-                .replacePath(TrinoS3ProxyRestConstants.S3_PATH)
                 .path("two")
                 .path("test")
                 .build();

diff --git a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestProxiedAssumedRoleRequests.java b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestProxiedAssumedRoleRequests.java
@@ -16,7 +16,7 @@
 import com.google.inject.Inject;
 import io.airlift.http.server.testing.TestingHttpServer;
 import io.trino.s3.proxy.server.credentials.Credentials;
-import io.trino.s3.proxy.server.rest.TrinoS3ProxyRestConstants;
+import io.trino.s3.proxy.server.rest.TrinoS3ProxyConfig;
 import io.trino.s3.proxy.server.testing.ManagedS3MockContainer.ForS3MockContainer;
 import io.trino.s3.proxy.server.testing.TestingCredentialsRolesProvider;
 import io.trino.s3.proxy.server.testing.TestingUtil.ForTesting;
@@ -51,9 +51,10 @@ public TestProxiedAssumedRoleRequests(
             @ForTesting Credentials testingCredentials,
             TestingCredentialsRolesProvider credentialsController,
             @ForS3MockContainer S3Client storageClient,
-            @ForS3MockContainer List<String> configuredBuckets)
+            @ForS3MockContainer List<String> configuredBuckets,
+            TrinoS3ProxyConfig trinoS3ProxyConfig)
     {
-        this(buildClient(httpServer, testingCredentials), testingCredentials, credentialsController, storageClient, configuredBuckets);
+        this(buildClient(httpServer, testingCredentials, trinoS3ProxyConfig.getS3Path(), trinoS3ProxyConfig.getStsPath()), testingCredentials, credentialsController, storageClient, configuredBuckets);
     }
 
     protected TestProxiedAssumedRoleRequests(
@@ -75,11 +76,11 @@ public void validateCount()
         credentialsController.resetAssumedRoles();
     }
 
-    protected static S3Client buildClient(TestingHttpServer httpServer, Credentials credentials)
+    protected static S3Client buildClient(TestingHttpServer httpServer, Credentials credentials, String s3Path, String stsPath)
     {
         URI baseUrl = httpServer.getBaseUrl();
-        URI localProxyServerUri = baseUrl.resolve(TrinoS3ProxyRestConstants.S3_PATH);
-        URI localStsServerUri = baseUrl.resolve(TrinoS3ProxyRestConstants.STS_PATH);
+        URI localProxyServerUri = baseUrl.resolve(s3Path);
+        URI localStsServerUri = baseUrl.resolve(stsPath);
 
         AwsBasicCredentials awsBasicCredentials = AwsBasicCredentials.create(credentials.emulated().accessKey(), credentials.emulated().secretKey());
 

diff --git a/...c/test/java/io/trino/s3/proxy/server/TestProxiedEmulatedAndRemoteAssumedRoleRequests.java b/...c/test/java/io/trino/s3/proxy/server/TestProxiedEmulatedAndRemoteAssumedRoleRequests.java
@@ -16,6 +16,7 @@
 import com.google.inject.Inject;
 import io.airlift.http.server.testing.TestingHttpServer;
 import io.trino.s3.proxy.server.credentials.Credentials;
+import io.trino.s3.proxy.server.rest.TrinoS3ProxyConfig;
 import io.trino.s3.proxy.server.testing.ManagedS3MockContainer.ForS3MockContainer;
 import io.trino.s3.proxy.server.testing.TestingCredentialsRolesProvider;
 import io.trino.s3.proxy.server.testing.TestingTrinoS3ProxyServerModule.ForTestingRemoteCredentials;
@@ -34,8 +35,9 @@ public TestProxiedEmulatedAndRemoteAssumedRoleRequests(
             TestingCredentialsRolesProvider credentialsController,
             @ForS3MockContainer S3Client storageClient,
             @ForS3MockContainer List<String> configuredBuckets,
-            @ForTestingRemoteCredentials Credentials remoteCredentials)
+            @ForTestingRemoteCredentials Credentials remoteCredentials,
+            TrinoS3ProxyConfig trinoS3ProxyConfig)
     {
-        super(buildClient(httpServer, remoteCredentials), testingCredentials, credentialsController, storageClient, configuredBuckets);
+        super(buildClient(httpServer, remoteCredentials, trinoS3ProxyConfig.getS3Path(), trinoS3ProxyConfig.getStsPath()), testingCredentials, credentialsController, storageClient, configuredBuckets);
     }
 }
diff --git a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestProxiedRequests.java b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestProxiedRequests.java
@@ -15,16 +15,28 @@
 
 import com.google.inject.Inject;
 import io.trino.s3.proxy.server.testing.ManagedS3MockContainer.ForS3MockContainer;
+import io.trino.s3.proxy.server.testing.TestingTrinoS3ProxyServer;
+import io.trino.s3.proxy.server.testing.harness.BuilderFilter;
 import io.trino.s3.proxy.server.testing.harness.TrinoS3ProxyTest;
 import io.trino.s3.proxy.server.testing.harness.TrinoS3ProxyTestCommonModules.WithConfiguredBuckets;
 import software.amazon.awssdk.services.s3.S3Client;
 
 import java.util.List;
 
-@TrinoS3ProxyTest(filters = WithConfiguredBuckets.class)
+@TrinoS3ProxyTest(filters = {WithConfiguredBuckets.class, TestProxiedRequests.Filter.class})
 public class TestProxiedRequests
         extends AbstractTestProxiedRequests
 {
+    public static class Filter
+            implements BuilderFilter
+    {
+        @Override
+        public TestingTrinoS3ProxyServer.Builder filter(TestingTrinoS3ProxyServer.Builder builder)
+        {
+            return builder.withProperty("s3proxy.s3.path", "/api/some/s3/path");
+        }
+    }
+
     @Inject
     public TestProxiedRequests(S3Client s3Client, @ForS3MockContainer S3Client storageClient, @ForS3MockContainer List<String> configuredBuckets)
     {

diff --git a/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestProxiedRequestsWithEmptyPath.java b/trino-s3-proxy/src/test/java/io/trino/s3/proxy/server/TestProxiedRequestsWithEmptyPath.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.s3.proxy.server;
+
+import com.google.inject.Inject;
+import io.trino.s3.proxy.server.testing.ManagedS3MockContainer.ForS3MockContainer;
+import io.trino.s3.proxy.server.testing.TestingTrinoS3ProxyServer;
+import io.trino.s3.proxy.server.testing.harness.BuilderFilter;
+import io.trino.s3.proxy.server.testing.harness.TrinoS3ProxyTest;
+import io.trino.s3.proxy.server.testing.harness.TrinoS3ProxyTestCommonModules.WithConfiguredBuckets;
+import software.amazon.awssdk.services.s3.S3Client;
+
+import java.util.List;
+
+@TrinoS3ProxyTest(filters = {WithConfiguredBuckets.class, TestProxiedRequestsWithEmptyPath.Filter.class})
+public class TestProxiedRequestsWithEmptyPath
+        extends AbstractTestProxiedRequests
+{
+    public static class Filter
+            implements BuilderFilter
+    {
+        @Override
+        public TestingTrinoS3ProxyServer.Builder filter(TestingTrinoS3ProxyServer.Builder builder)
+        {
+            return builder.withProperty("s3proxy.s3.path", "");
+        }
+    }
+
+    @Inject
+    public TestProxiedRequestsWithEmptyPath(S3Client s3Client, @ForS3MockContainer S3Client storageClient, @ForS3MockContainer List<String> configuredBuckets)
+    {
+        super(s3Client, storageClient, configuredBuckets);
+    }
+}