Backblaze refactor (#1011)

* In this commit I refactor BackblazePhotoImporter to enable unit testing (see next commit) * Add a test file for BackblazePhotosImporter and update the gradle file with new dependencies In this commit I added a test file for BackblazePhotosImporter and updated the gradle file with new dependencies required to run it. * Refactor BackblazeVideosImporter to enable unit testing Commit is corresponding to the one for the photos importer but this time we're changing the constructor of the video importer. The unit tests are added in the next commit * Add a test file for BackblazeVideosImporter In this commit I'm adding a file with unit tests for the video importer * Adding BaseBackblazeS3ClientFactory to enable unit testing on the client The following two commits refacor BackblazeDataTransferClient for facilitating BackblazeDataTransferClient unit testing. In this one I'm adding interface for client factory and adding a base class * Refactor BackblazeDataTransferClient and BackblazeDataTransferClientFactory to make use of BaseBackblazeS3ClientFactory In this commit I refactor BackblazeDataTransferClient and BackblazeDataTransferClientFactory to make use of BaseBackblazeS3ClientFactory * Add BackblazeDataTransferClientTest In the last commit I implement BackblazeDataTransferClientTest * Remove unused variable from BackblazeDataTransferClient Co-authored-by: William Morland <[email protected]>
dtinit · Oct 6, 2021 · 5e3cfd6 · 5e3cfd6
1 parent 525cf50
commit 5e3cfd6
Show file tree

Hide file tree

Showing 12 changed files with 669 additions and 40 deletions.
diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/build.gradle b/extensions/data-transfer/portability-data-transfer-backblaze/build.gradle
@@ -27,6 +27,9 @@ dependencies {
 
     compile('software.amazon.awssdk:s3:2.15.24')
     compile('org.apache.commons:commons-lang3:3.11')
+    compile("commons-io:commons-io:2.6")
+
+    testCompile("org.mockito:mockito-core:${mockitoVersion}")
 }
 
 configurePublication(project)
diff --git a/.../main/java/org/datatransferproject/datatransfer/backblaze/BackblazeTransferExtension.java b/.../main/java/org/datatransferproject/datatransfer/backblaze/BackblazeTransferExtension.java
@@ -22,12 +22,14 @@
 import java.util.List;
 import org.datatransferproject.api.launcher.ExtensionContext;
 import org.datatransferproject.api.launcher.Monitor;
+import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClientFactory;
 import org.datatransferproject.datatransfer.backblaze.photos.BackblazePhotosImporter;
 import org.datatransferproject.datatransfer.backblaze.videos.BackblazeVideosImporter;
 import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore;
 import org.datatransferproject.spi.transfer.extension.TransferExtension;
 import org.datatransferproject.spi.transfer.provider.Exporter;
 import org.datatransferproject.spi.transfer.provider.Importer;
+import org.datatransferproject.transfer.ImageStreamProvider;
 
 public class BackblazeTransferExtension implements TransferExtension {
   public static final String SERVICE_ID = "Backblaze";
@@ -69,8 +71,18 @@ public void initialize(ExtensionContext context) {
     TemporaryPerJobDataStore jobStore = context.getService(TemporaryPerJobDataStore.class);
 
     ImmutableMap.Builder<String, Importer> importerBuilder = ImmutableMap.builder();
-    importerBuilder.put("PHOTOS", new BackblazePhotosImporter(monitor, jobStore));
-    importerBuilder.put("VIDEOS", new BackblazeVideosImporter(monitor, jobStore));
+    BackblazeDataTransferClientFactory backblazeDataTransferClientFactory =
+            new BackblazeDataTransferClientFactory();
+    ImageStreamProvider isProvider = new ImageStreamProvider();
+
+    importerBuilder.put(
+            "PHOTOS",
+            new BackblazePhotosImporter(
+                    monitor, jobStore, isProvider, backblazeDataTransferClientFactory));
+    importerBuilder.put(
+            "VIDEOS",
+            new BackblazeVideosImporter(
+                    monitor, jobStore, isProvider, backblazeDataTransferClientFactory));
     importerMap = importerBuilder.build();
     initialized = true;
   }

diff --git a/...va/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java b/...va/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java
@@ -56,23 +56,32 @@
 
 public class BackblazeDataTransferClient {
   private static final String DATA_TRANSFER_BUCKET_PREFIX_FORMAT_STRING = "%s-data-transfer";
-  private static final String S3_ENDPOINT_FORMAT_STRING = "https://s3.%s.backblazeb2.com";
   private static final int MAX_BUCKET_CREATION_ATTEMPTS = 10;
   private final List<String> BACKBLAZE_REGIONS =
       Arrays.asList("us-west-000", "us-west-001", "us-west-002", "eu-central-003");
 
-  private static final long SIZE_THRESHOLD_FOR_MULTIPART_UPLOAD = 20 * 1024 * 1024; // 20 MB.
-  private static final long PART_SIZE_FOR_MULTIPART_UPLOAD = 5 * 1024 * 1024; // 5 MB.
-
+  private final long sizeThresholdForMultipartUpload;
+  private final long partSizeForMultiPartUpload;
+  private final BackblazeS3ClientFactory backblazeS3ClientFactory;
   private final Monitor monitor;
   private S3Client s3Client;
   private String bucketName;
 
-  public BackblazeDataTransferClient(Monitor monitor) {
+  public BackblazeDataTransferClient(
+          Monitor monitor,
+          BackblazeS3ClientFactory backblazeS3ClientFactory,
+          long sizeThresholdForMultipartUpload,
+          long partSizeForMultiPartUpload) {
     this.monitor = monitor;
+    this.backblazeS3ClientFactory = backblazeS3ClientFactory;
+    // Avoid infinite loops
+    if (partSizeForMultiPartUpload <= 0)
+      throw new IllegalArgumentException("Part size for multipart upload must be positive.");
+    this.sizeThresholdForMultipartUpload = sizeThresholdForMultipartUpload;
+    this.partSizeForMultiPartUpload = partSizeForMultiPartUpload;
   }
 
-  public void init(String keyId, String applicationKey)
+  public void init(String keyId, String applicationKey, String exportService)
       throws BackblazeCredentialsException, IOException {
     // Fetch all the available buckets and use that to find which region the user is in
     ListBucketsResponse listBucketsResponse = null;
@@ -92,7 +101,7 @@ public void init(String keyId, String applicationKey)
     Throwable s3Exception = null;
     for (String region : BACKBLAZE_REGIONS) {
       try {
-        s3Client = getOrCreateS3Client(keyId, applicationKey, region);
+        s3Client = backblazeS3ClientFactory.createS3Client(keyId, applicationKey, region);
 
         listBucketsResponse = s3Client.listBuckets();
         userRegion = region;
@@ -113,7 +122,7 @@ public void init(String keyId, String applicationKey)
           "User's credentials or permissions are not valid for any regions available", s3Exception);
     }
 
-    bucketName = getOrCreateBucket(s3Client, listBucketsResponse, userRegion);
+    bucketName = getOrCreateBucket(s3Client, listBucketsResponse, userRegion, exportService);
   }
 
   public String uploadFile(String fileKey, File file) throws IOException {
@@ -126,12 +135,12 @@ public String uploadFile(String fileKey, File file) throws IOException {
       monitor.debug(
           () -> String.format("Uploading '%s' with file size %d bytes", fileKey, contentLength));
 
-      if (contentLength >= SIZE_THRESHOLD_FOR_MULTIPART_UPLOAD) {
+      if (contentLength >= sizeThresholdForMultipartUpload) {
         monitor.debug(
             () ->
                 String.format(
                     "File size is larger than %d bytes, so using multipart upload",
-                    SIZE_THRESHOLD_FOR_MULTIPART_UPLOAD));
+                        sizeThresholdForMultipartUpload));
         return uploadFileUsingMultipartUpload(fileKey, file, contentLength);
       }
 
@@ -160,7 +169,7 @@ private String uploadFileUsingMultipartUpload(String fileKey, File file, long co
     try (InputStream fileInputStream = new FileInputStream(file)) {
       for (int i = 1; filePosition < contentLength; i++) {
         // Because the last part could be smaller than others, adjust the part size as needed
-        long partSize = Math.min(PART_SIZE_FOR_MULTIPART_UPLOAD, (contentLength - filePosition));
+        long partSize = Math.min(partSizeForMultiPartUpload, (contentLength - filePosition));
 
         UploadPartRequest uploadRequest =
             UploadPartRequest.builder()
@@ -194,13 +203,16 @@ private String uploadFileUsingMultipartUpload(String fileKey, File file, long co
   }
 
   private String getOrCreateBucket(
-      S3Client s3Client, ListBucketsResponse listBucketsResponse, String region)
+      S3Client s3Client,
+      ListBucketsResponse listBucketsResponse,
+      String region,
+      String exportService)
       throws IOException {
 
     String fullPrefix =
         String.format(
             DATA_TRANSFER_BUCKET_PREFIX_FORMAT_STRING,
-            JobMetadata.getExportService().toLowerCase());
+            exportService.toLowerCase());
     try {
       for (Bucket bucket : listBucketsResponse.buckets()) {
         if (bucket.name().startsWith(fullPrefix)) {
@@ -234,21 +246,4 @@ private String getOrCreateBucket(
       throw new IOException("Error while creating bucket", e);
     }
   }
-
-  private S3Client getOrCreateS3Client(String accessKey, String secretKey, String region) {
-    AwsSessionCredentials awsCreds = AwsSessionCredentials.create(accessKey, secretKey, "");
-
-    ClientOverrideConfiguration clientOverrideConfiguration =
-        ClientOverrideConfiguration.builder().putHeader("User-Agent", "Facebook-DTP").build();
-
-    // Use any AWS region for the client, the Backblaze API does not care about it
-    Region awsRegion = Region.US_EAST_1;
-
-    return S3Client.builder()
-        .credentialsProvider(StaticCredentialsProvider.create(awsCreds))
-        .overrideConfiguration(clientOverrideConfiguration)
-        .endpointOverride(URI.create(String.format(S3_ENDPOINT_FORMAT_STRING, region)))
-        .region(awsRegion)
-        .build();
-  }
 }
diff --git a/...datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientFactory.java b/...datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientFactory.java
@@ -19,18 +19,26 @@
 import java.io.IOException;
 import org.datatransferproject.api.launcher.Monitor;
 import org.datatransferproject.datatransfer.backblaze.exception.BackblazeCredentialsException;
+import org.datatransferproject.transfer.JobMetadata;
 import org.datatransferproject.types.transfer.auth.TokenSecretAuthData;
 
 public class BackblazeDataTransferClientFactory {
   private BackblazeDataTransferClient b2Client;
+  private static final long SIZE_THRESHOLD_FOR_MULTIPART_UPLOAD = 20 * 1024 * 1024; // 20 MB.
+  private static final long PART_SIZE_FOR_MULTIPART_UPLOAD = 5 * 1024 * 1024; // 5 MB.
 
   public BackblazeDataTransferClient getOrCreateB2Client(
       Monitor monitor, TokenSecretAuthData authData)
       throws BackblazeCredentialsException, IOException {
     if (b2Client == null) {
       BackblazeDataTransferClient backblazeDataTransferClient =
-          new BackblazeDataTransferClient(monitor);
-      backblazeDataTransferClient.init(authData.getToken(), authData.getSecret());
+              new BackblazeDataTransferClient(
+                      monitor,
+                      new BaseBackblazeS3ClientFactory(),
+                      SIZE_THRESHOLD_FOR_MULTIPART_UPLOAD,
+                      PART_SIZE_FOR_MULTIPART_UPLOAD);
+      String exportService = JobMetadata.getExportService();
+      backblazeDataTransferClient.init(authData.getToken(), authData.getSecret(), exportService);
       b2Client = backblazeDataTransferClient;
     }
     return b2Client;

diff --git a/.../java/org/datatransferproject/datatransfer/backblaze/common/BackblazeS3ClientFactory.java b/.../java/org/datatransferproject/datatransfer/backblaze/common/BackblazeS3ClientFactory.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2021 The Data Transfer Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.datatransferproject.datatransfer.backblaze.common;
+
+import software.amazon.awssdk.services.s3.S3Client;
+
+public interface BackblazeS3ClientFactory {
+    S3Client createS3Client(String accessKey, String secretKey, String region);
+}
diff --git a/...a/org/datatransferproject/datatransfer/backblaze/common/BaseBackblazeS3ClientFactory.java b/...a/org/datatransferproject/datatransfer/backblaze/common/BaseBackblazeS3ClientFactory.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2021 The Data Transfer Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.datatransferproject.datatransfer.backblaze.common;
+
+import java.net.URI;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+
+public class BaseBackblazeS3ClientFactory implements BackblazeS3ClientFactory {
+    private static final String S3_ENDPOINT_FORMAT_STRING = "https://s3.%s.backblazeb2.com";
+
+    public S3Client createS3Client(String accessKey, String secretKey, String region) {
+        AwsSessionCredentials awsCreds = AwsSessionCredentials.create(accessKey, secretKey, "");
+
+        ClientOverrideConfiguration clientOverrideConfiguration =
+                ClientOverrideConfiguration.builder().putHeader("User-Agent", "Facebook-DTP").build();
+
+        // Use any AWS region for the client, the Backblaze API does not care about it
+        Region awsRegion = Region.US_EAST_1;
+
+        return S3Client.builder()
+                .credentialsProvider(StaticCredentialsProvider.create(awsCreds))
+                .overrideConfiguration(clientOverrideConfiguration)
+                .endpointOverride(URI.create(String.format(S3_ENDPOINT_FORMAT_STRING, region)))
+                .region(awsRegion)
+                .build();
+    }
+}
diff --git a/...n/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java b/...n/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java
@@ -41,14 +41,19 @@ public class BackblazePhotosImporter
   private static final String PHOTO_TRANSFER_MAIN_FOLDER = "Photo Transfer";
 
   private final TemporaryPerJobDataStore jobStore;
-  private final ImageStreamProvider imageStreamProvider = new ImageStreamProvider();
+  private final ImageStreamProvider imageStreamProvider;
   private final Monitor monitor;
   private final BackblazeDataTransferClientFactory b2ClientFactory;
 
-  public BackblazePhotosImporter(Monitor monitor, TemporaryPerJobDataStore jobStore) {
+  public BackblazePhotosImporter(
+          Monitor monitor,
+          TemporaryPerJobDataStore jobStore,
+          ImageStreamProvider imageStreamProvider,
+          BackblazeDataTransferClientFactory b2ClientFactory) {
     this.monitor = monitor;
     this.jobStore = jobStore;
-    this.b2ClientFactory = new BackblazeDataTransferClientFactory();
+    this.imageStreamProvider = imageStreamProvider;
+    this.b2ClientFactory = b2ClientFactory;
   }
 
   @Override

diff --git a/...n/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java b/...n/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java
@@ -37,14 +37,19 @@ public class BackblazeVideosImporter
   private static final String VIDEO_TRANSFER_MAIN_FOLDER = "Video Transfer";
 
   private final TemporaryPerJobDataStore jobStore;
-  private final ImageStreamProvider imageStreamProvider = new ImageStreamProvider();
+  private final ImageStreamProvider imageStreamProvider;
   private final Monitor monitor;
   private final BackblazeDataTransferClientFactory b2ClientFactory;
 
-  public BackblazeVideosImporter(Monitor monitor, TemporaryPerJobDataStore jobStore) {
+  public BackblazeVideosImporter(
+          Monitor monitor,
+          TemporaryPerJobDataStore jobStore,
+          ImageStreamProvider imageStreamProvider,
+          BackblazeDataTransferClientFactory b2ClientFactory) {
     this.monitor = monitor;
     this.jobStore = jobStore;
-    this.b2ClientFactory = new BackblazeDataTransferClientFactory();
+    this.imageStreamProvider = imageStreamProvider;
+    this.b2ClientFactory = b2ClientFactory;
   }
 
   @Override