storage: add list GCS objects in Java S3 SDK (GoogleCloudPlatform#1351)

* Add list GCS objects in Java with workaround * Add a comment on "encoding-type" * Fix lint issues * Remove workaround from sample * added new sample format test expectation and lint * Update commented variables
mtabasko · Apr 4, 2019 · d42ffac · d42ffac
1 parent 848fad5
commit d42ffac
Show file tree

Hide file tree

Showing 4 changed files with 182 additions and 9 deletions.
diff --git a/storage/s3-sdk/src/main/java/ListGcsBuckets.java b/storage/s3-sdk/src/main/java/ListGcsBuckets.java
@@ -25,17 +25,20 @@
 import java.util.List;
 
 public class ListGcsBuckets {
-  public static List<Bucket> listGcsBuckets(String googleAccessKeyId, 
+  public static void listGcsBuckets(String googleAccessKeyId,
       String googleAccessKeySecret) {
+
+    // String googleAccessKeyId = "your-google-access-key-id";
+    // String googleAccessKeySecret = "your-google-access-key-secret";
+
     // Create a BasicAWSCredentials using Cloud Storage HMAC credentials.
-    BasicAWSCredentials googleCreds = new BasicAWSCredentials(googleAccessKeyId, 
+    BasicAWSCredentials googleCreds = new BasicAWSCredentials(googleAccessKeyId,
         googleAccessKeySecret);
 
     // Create a new client and do the following:
     // 1. Change the endpoint URL to use the Google Cloud Storage XML API endpoint.
     // 2. Use Cloud Storage HMAC Credentials.
-    AmazonS3 interopClient =
-        AmazonS3ClientBuilder.standard()
+    AmazonS3 interopClient = AmazonS3ClientBuilder.standard()
             .withEndpointConfiguration(
                 new AwsClientBuilder.EndpointConfiguration(
                     "https://storage.googleapis.com", "auto"))
@@ -53,8 +56,6 @@ public static List<Bucket> listGcsBuckets(String googleAccessKeyId,
 
     // Explicitly clean up client resources.
     interopClient.shutdown();
-
-    return buckets;
   }
   // [END storage_s3_sdk_list_buckets]
 }

diff --git a/storage/s3-sdk/src/main/java/ListGcsObjects.java b/storage/s3-sdk/src/main/java/ListGcsObjects.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+// [START storage_s3_sdk_list_objects]
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+import java.util.List;
+
+public class ListGcsObjects {
+  public static void listGcsObjects(String googleAccessKeyId,
+      String googleAccessKeySecret, String bucketName) {
+
+    // String googleAccessKeyId = "your-google-access-key-id";
+    // String googleAccessKeySecret = "your-google-access-key-secret";
+    // String bucketName = "bucket-name";
+
+    // Create a BasicAWSCredentials using Cloud Storage HMAC credentials.
+    BasicAWSCredentials googleCreds = new BasicAWSCredentials(googleAccessKeyId,
+        googleAccessKeySecret);
+
+    // Create a new client and do the following:
+    // 1. Change the endpoint URL to use the Google Cloud Storage XML API endpoint.
+    // 2. Use Cloud Storage HMAC Credentials.
+    AmazonS3 interopClient = AmazonS3ClientBuilder.standard()
+            .withEndpointConfiguration(
+                new AwsClientBuilder.EndpointConfiguration(
+                    "https://storage.googleapis.com", "auto"))
+            .withCredentials(new AWSStaticCredentialsProvider(googleCreds))
+            .build();
+
+    // Call GCS to list current objects
+    ObjectListing objects = interopClient.listObjects(bucketName);
+
+    // Print objects names
+    System.out.println("Objects:");
+    for (S3ObjectSummary object : objects.getObjectSummaries()) {
+      System.out.println(object.getKey());
+    }
+
+    // Explicitly clean up client resources.
+    interopClient.shutdown();
+  }
+}
+// [END storage_s3_sdk_list_objects]
diff --git a/storage/s3-sdk/src/test/java/ListGcsBucketsTest.java b/storage/s3-sdk/src/test/java/ListGcsBucketsTest.java
@@ -14,20 +14,56 @@
  * limitations under the License.
  */
 
-import static com.google.common.truth.Truth.assertThat;
+import static junit.framework.TestCase.assertNotNull;
+import static org.junit.Assert.assertThat;
 
 import com.amazonaws.services.s3.model.Bucket;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
 import java.util.List;
+import org.hamcrest.CoreMatchers;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Test;
 
 public class ListGcsBucketsTest {
   private static final String BUCKET = System.getenv("GOOGLE_CLOUD_PROJECT_S3_SDK");
   private static final String KEY_ID = System.getenv("STORAGE_HMAC_ACCESS_KEY_ID");
   private static final String SECRET_KEY = System.getenv("STORAGE_HMAC_ACCESS_SECRET_KEY");
+  private ByteArrayOutputStream bout;
+
+  private static void requireEnvVar(String varName) {
+    assertNotNull(
+        System.getenv(varName),
+        "Environment variable '%s' is required to perform these tests.".format(varName)
+    );
+  }
+
+  @BeforeClass
+  public static void checkRequirements() {
+    requireEnvVar("GOOGLE_CLOUD_PROJECT_S3_SDK");
+    requireEnvVar("STORAGE_HMAC_ACCESS_KEY_ID");
+    requireEnvVar("STORAGE_HMAC_ACCESS_SECRET_KEY");
+  }
+
+  @Before
+  public void beforeTest() {
+    bout = new ByteArrayOutputStream();
+    System.setOut(new PrintStream(bout));
+  }
+
+  @After
+  public void tearDown() {
+    System.setOut(null);
+    bout.reset();
+  }
 
   @Test
   public void testListBucket() throws Exception {
-    List<Bucket> buckets = ListGcsBuckets.listGcsBuckets(KEY_ID, SECRET_KEY);
-    assertThat(buckets.toString()).contains(BUCKET);
+    ListGcsBuckets.listGcsBuckets(KEY_ID, SECRET_KEY);
+    String output = bout.toString();
+    assertThat(output, CoreMatchers.containsString("Buckets:"));
   }
 }
diff --git a/storage/s3-sdk/src/test/java/ListGcsObjectsTest.java b/storage/s3-sdk/src/test/java/ListGcsObjectsTest.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static junit.framework.TestCase.assertNotNull;
+import static org.junit.Assert.assertThat;
+
+import com.amazonaws.services.s3.model.Bucket;
+import com.amazonaws.services.s3.model.ObjectListing;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.List;
+import org.hamcrest.CoreMatchers;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+
+public class ListGcsObjectsTest {
+  private static final String BUCKET = System.getenv("GOOGLE_CLOUD_PROJECT_S3_SDK");
+  private static final String KEY_ID = System.getenv("STORAGE_HMAC_ACCESS_KEY_ID");
+  private static final String SECRET_KEY = System.getenv("STORAGE_HMAC_ACCESS_SECRET_KEY");
+  private ByteArrayOutputStream bout;
+
+  private static void requireEnvVar(String varName) {
+    assertNotNull(
+        System.getenv(varName),
+        "Environment variable '%s' is required to perform these tests.".format(varName)
+    );
+  }
+
+  @BeforeClass
+  public static void checkRequirements() {
+    requireEnvVar("GOOGLE_CLOUD_PROJECT_S3_SDK");
+    requireEnvVar("STORAGE_HMAC_ACCESS_KEY_ID");
+    requireEnvVar("STORAGE_HMAC_ACCESS_SECRET_KEY");
+  }
+
+  @Before
+  public void beforeTest() {
+    bout = new ByteArrayOutputStream();
+    System.setOut(new PrintStream(bout));
+  }
+
+  @After
+  public void tearDown() {
+    System.setOut(null);
+    bout.reset();
+  }
+
+  @Test
+  public void testListObjects() throws Exception {
+    ListGcsObjects.listGcsObjects(KEY_ID, SECRET_KEY, BUCKET);
+    String output = bout.toString();
+    assertThat(output, CoreMatchers.containsString("Objects:"));
+  }
+}