From 5e3cfd6ba57459b152e9996052db06f7884c1095 Mon Sep 17 00:00:00 2001 From: pziebal <51990839+pziebal@users.noreply.github.com> Date: Wed, 6 Oct 2021 10:19:51 +0100 Subject: [PATCH] Backblaze refactor (#1011) * In this commit I refactor BackblazePhotoImporter to enable unit testing (see next commit) * Add a test file for BackblazePhotosImporter and update the gradle file with new dependencies In this commit I added a test file for BackblazePhotosImporter and updated the gradle file with new dependencies required to run it. * Refactor BackblazeVideosImporter to enable unit testing Commit is corresponding to the one for the photos importer but this time we're changing the constructor of the video importer. The unit tests are added in the next commit * Add a test file for BackblazeVideosImporter In this commit I'm adding a file with unit tests for the video importer * Adding BaseBackblazeS3ClientFactory to enable unit testing on the client The following two commits refacor BackblazeDataTransferClient for facilitating BackblazeDataTransferClient unit testing. In this one I'm adding interface for client factory and adding a base class * Refactor BackblazeDataTransferClient and BackblazeDataTransferClientFactory to make use of BaseBackblazeS3ClientFactory In this commit I refactor BackblazeDataTransferClient and BackblazeDataTransferClientFactory to make use of BaseBackblazeS3ClientFactory * Add BackblazeDataTransferClientTest In the last commit I implement BackblazeDataTransferClientTest * Remove unused variable from BackblazeDataTransferClient Co-authored-by: William Morland --- .../build.gradle | 3 + .../backblaze/BackblazeTransferExtension.java | 16 +- .../common/BackblazeDataTransferClient.java | 55 ++-- .../BackblazeDataTransferClientFactory.java | 12 +- .../common/BackblazeS3ClientFactory.java | 23 ++ .../common/BaseBackblazeS3ClientFactory.java | 45 ++++ .../photos/BackblazePhotosImporter.java | 11 +- .../videos/BackblazeVideosImporter.java | 11 +- .../BackblazeDataTransferClientTest.java | 244 ++++++++++++++++++ .../photos/BackblazePhotosImporterTest.java | 152 +++++++++++ .../videos/BackblazeVideosImporterTest.java | 134 ++++++++++ .../src/test/resources/test.txt | 3 + 12 files changed, 669 insertions(+), 40 deletions(-) create mode 100644 extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeS3ClientFactory.java create mode 100644 extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BaseBackblazeS3ClientFactory.java create mode 100644 extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientTest.java create mode 100644 extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporterTest.java create mode 100644 extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporterTest.java create mode 100644 extensions/data-transfer/portability-data-transfer-backblaze/src/test/resources/test.txt diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/build.gradle b/extensions/data-transfer/portability-data-transfer-backblaze/build.gradle index ef8c24afb..eb9de76f1 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/build.gradle +++ b/extensions/data-transfer/portability-data-transfer-backblaze/build.gradle @@ -27,6 +27,9 @@ dependencies { compile('software.amazon.awssdk:s3:2.15.24') compile('org.apache.commons:commons-lang3:3.11') + compile("commons-io:commons-io:2.6") + + testCompile("org.mockito:mockito-core:${mockitoVersion}") } configurePublication(project) diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/BackblazeTransferExtension.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/BackblazeTransferExtension.java index 62fcb4854..ebda4bcfe 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/BackblazeTransferExtension.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/BackblazeTransferExtension.java @@ -22,12 +22,14 @@ import java.util.List; import org.datatransferproject.api.launcher.ExtensionContext; import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClientFactory; import org.datatransferproject.datatransfer.backblaze.photos.BackblazePhotosImporter; import org.datatransferproject.datatransfer.backblaze.videos.BackblazeVideosImporter; import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore; import org.datatransferproject.spi.transfer.extension.TransferExtension; import org.datatransferproject.spi.transfer.provider.Exporter; import org.datatransferproject.spi.transfer.provider.Importer; +import org.datatransferproject.transfer.ImageStreamProvider; public class BackblazeTransferExtension implements TransferExtension { public static final String SERVICE_ID = "Backblaze"; @@ -69,8 +71,18 @@ public void initialize(ExtensionContext context) { TemporaryPerJobDataStore jobStore = context.getService(TemporaryPerJobDataStore.class); ImmutableMap.Builder importerBuilder = ImmutableMap.builder(); - importerBuilder.put("PHOTOS", new BackblazePhotosImporter(monitor, jobStore)); - importerBuilder.put("VIDEOS", new BackblazeVideosImporter(monitor, jobStore)); + BackblazeDataTransferClientFactory backblazeDataTransferClientFactory = + new BackblazeDataTransferClientFactory(); + ImageStreamProvider isProvider = new ImageStreamProvider(); + + importerBuilder.put( + "PHOTOS", + new BackblazePhotosImporter( + monitor, jobStore, isProvider, backblazeDataTransferClientFactory)); + importerBuilder.put( + "VIDEOS", + new BackblazeVideosImporter( + monitor, jobStore, isProvider, backblazeDataTransferClientFactory)); importerMap = importerBuilder.build(); initialized = true; } diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java index d7164a2c5..a764c4594 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClient.java @@ -56,23 +56,32 @@ public class BackblazeDataTransferClient { private static final String DATA_TRANSFER_BUCKET_PREFIX_FORMAT_STRING = "%s-data-transfer"; - private static final String S3_ENDPOINT_FORMAT_STRING = "https://s3.%s.backblazeb2.com"; private static final int MAX_BUCKET_CREATION_ATTEMPTS = 10; private final List BACKBLAZE_REGIONS = Arrays.asList("us-west-000", "us-west-001", "us-west-002", "eu-central-003"); - private static final long SIZE_THRESHOLD_FOR_MULTIPART_UPLOAD = 20 * 1024 * 1024; // 20 MB. - private static final long PART_SIZE_FOR_MULTIPART_UPLOAD = 5 * 1024 * 1024; // 5 MB. - + private final long sizeThresholdForMultipartUpload; + private final long partSizeForMultiPartUpload; + private final BackblazeS3ClientFactory backblazeS3ClientFactory; private final Monitor monitor; private S3Client s3Client; private String bucketName; - public BackblazeDataTransferClient(Monitor monitor) { + public BackblazeDataTransferClient( + Monitor monitor, + BackblazeS3ClientFactory backblazeS3ClientFactory, + long sizeThresholdForMultipartUpload, + long partSizeForMultiPartUpload) { this.monitor = monitor; + this.backblazeS3ClientFactory = backblazeS3ClientFactory; + // Avoid infinite loops + if (partSizeForMultiPartUpload <= 0) + throw new IllegalArgumentException("Part size for multipart upload must be positive."); + this.sizeThresholdForMultipartUpload = sizeThresholdForMultipartUpload; + this.partSizeForMultiPartUpload = partSizeForMultiPartUpload; } - public void init(String keyId, String applicationKey) + public void init(String keyId, String applicationKey, String exportService) throws BackblazeCredentialsException, IOException { // Fetch all the available buckets and use that to find which region the user is in ListBucketsResponse listBucketsResponse = null; @@ -92,7 +101,7 @@ public void init(String keyId, String applicationKey) Throwable s3Exception = null; for (String region : BACKBLAZE_REGIONS) { try { - s3Client = getOrCreateS3Client(keyId, applicationKey, region); + s3Client = backblazeS3ClientFactory.createS3Client(keyId, applicationKey, region); listBucketsResponse = s3Client.listBuckets(); userRegion = region; @@ -113,7 +122,7 @@ public void init(String keyId, String applicationKey) "User's credentials or permissions are not valid for any regions available", s3Exception); } - bucketName = getOrCreateBucket(s3Client, listBucketsResponse, userRegion); + bucketName = getOrCreateBucket(s3Client, listBucketsResponse, userRegion, exportService); } public String uploadFile(String fileKey, File file) throws IOException { @@ -126,12 +135,12 @@ public String uploadFile(String fileKey, File file) throws IOException { monitor.debug( () -> String.format("Uploading '%s' with file size %d bytes", fileKey, contentLength)); - if (contentLength >= SIZE_THRESHOLD_FOR_MULTIPART_UPLOAD) { + if (contentLength >= sizeThresholdForMultipartUpload) { monitor.debug( () -> String.format( "File size is larger than %d bytes, so using multipart upload", - SIZE_THRESHOLD_FOR_MULTIPART_UPLOAD)); + sizeThresholdForMultipartUpload)); return uploadFileUsingMultipartUpload(fileKey, file, contentLength); } @@ -160,7 +169,7 @@ private String uploadFileUsingMultipartUpload(String fileKey, File file, long co try (InputStream fileInputStream = new FileInputStream(file)) { for (int i = 1; filePosition < contentLength; i++) { // Because the last part could be smaller than others, adjust the part size as needed - long partSize = Math.min(PART_SIZE_FOR_MULTIPART_UPLOAD, (contentLength - filePosition)); + long partSize = Math.min(partSizeForMultiPartUpload, (contentLength - filePosition)); UploadPartRequest uploadRequest = UploadPartRequest.builder() @@ -194,13 +203,16 @@ private String uploadFileUsingMultipartUpload(String fileKey, File file, long co } private String getOrCreateBucket( - S3Client s3Client, ListBucketsResponse listBucketsResponse, String region) + S3Client s3Client, + ListBucketsResponse listBucketsResponse, + String region, + String exportService) throws IOException { String fullPrefix = String.format( DATA_TRANSFER_BUCKET_PREFIX_FORMAT_STRING, - JobMetadata.getExportService().toLowerCase()); + exportService.toLowerCase()); try { for (Bucket bucket : listBucketsResponse.buckets()) { if (bucket.name().startsWith(fullPrefix)) { @@ -234,21 +246,4 @@ private String getOrCreateBucket( throw new IOException("Error while creating bucket", e); } } - - private S3Client getOrCreateS3Client(String accessKey, String secretKey, String region) { - AwsSessionCredentials awsCreds = AwsSessionCredentials.create(accessKey, secretKey, ""); - - ClientOverrideConfiguration clientOverrideConfiguration = - ClientOverrideConfiguration.builder().putHeader("User-Agent", "Facebook-DTP").build(); - - // Use any AWS region for the client, the Backblaze API does not care about it - Region awsRegion = Region.US_EAST_1; - - return S3Client.builder() - .credentialsProvider(StaticCredentialsProvider.create(awsCreds)) - .overrideConfiguration(clientOverrideConfiguration) - .endpointOverride(URI.create(String.format(S3_ENDPOINT_FORMAT_STRING, region))) - .region(awsRegion) - .build(); - } } diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientFactory.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientFactory.java index 653a33023..0bdaca7c3 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientFactory.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientFactory.java @@ -19,18 +19,26 @@ import java.io.IOException; import org.datatransferproject.api.launcher.Monitor; import org.datatransferproject.datatransfer.backblaze.exception.BackblazeCredentialsException; +import org.datatransferproject.transfer.JobMetadata; import org.datatransferproject.types.transfer.auth.TokenSecretAuthData; public class BackblazeDataTransferClientFactory { private BackblazeDataTransferClient b2Client; + private static final long SIZE_THRESHOLD_FOR_MULTIPART_UPLOAD = 20 * 1024 * 1024; // 20 MB. + private static final long PART_SIZE_FOR_MULTIPART_UPLOAD = 5 * 1024 * 1024; // 5 MB. public BackblazeDataTransferClient getOrCreateB2Client( Monitor monitor, TokenSecretAuthData authData) throws BackblazeCredentialsException, IOException { if (b2Client == null) { BackblazeDataTransferClient backblazeDataTransferClient = - new BackblazeDataTransferClient(monitor); - backblazeDataTransferClient.init(authData.getToken(), authData.getSecret()); + new BackblazeDataTransferClient( + monitor, + new BaseBackblazeS3ClientFactory(), + SIZE_THRESHOLD_FOR_MULTIPART_UPLOAD, + PART_SIZE_FOR_MULTIPART_UPLOAD); + String exportService = JobMetadata.getExportService(); + backblazeDataTransferClient.init(authData.getToken(), authData.getSecret(), exportService); b2Client = backblazeDataTransferClient; } return b2Client; diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeS3ClientFactory.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeS3ClientFactory.java new file mode 100644 index 000000000..3d04515f2 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeS3ClientFactory.java @@ -0,0 +1,23 @@ +/* + * Copyright 2021 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.datatransferproject.datatransfer.backblaze.common; + +import software.amazon.awssdk.services.s3.S3Client; + +public interface BackblazeS3ClientFactory { + S3Client createS3Client(String accessKey, String secretKey, String region); +} diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BaseBackblazeS3ClientFactory.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BaseBackblazeS3ClientFactory.java new file mode 100644 index 000000000..9c72c26ca --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/common/BaseBackblazeS3ClientFactory.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.datatransferproject.datatransfer.backblaze.common; + +import java.net.URI; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +public class BaseBackblazeS3ClientFactory implements BackblazeS3ClientFactory { + private static final String S3_ENDPOINT_FORMAT_STRING = "https://s3.%s.backblazeb2.com"; + + public S3Client createS3Client(String accessKey, String secretKey, String region) { + AwsSessionCredentials awsCreds = AwsSessionCredentials.create(accessKey, secretKey, ""); + + ClientOverrideConfiguration clientOverrideConfiguration = + ClientOverrideConfiguration.builder().putHeader("User-Agent", "Facebook-DTP").build(); + + // Use any AWS region for the client, the Backblaze API does not care about it + Region awsRegion = Region.US_EAST_1; + + return S3Client.builder() + .credentialsProvider(StaticCredentialsProvider.create(awsCreds)) + .overrideConfiguration(clientOverrideConfiguration) + .endpointOverride(URI.create(String.format(S3_ENDPOINT_FORMAT_STRING, region))) + .region(awsRegion) + .build(); + } +} diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java index 398eba743..9443f5b75 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporter.java @@ -41,14 +41,19 @@ public class BackblazePhotosImporter private static final String PHOTO_TRANSFER_MAIN_FOLDER = "Photo Transfer"; private final TemporaryPerJobDataStore jobStore; - private final ImageStreamProvider imageStreamProvider = new ImageStreamProvider(); + private final ImageStreamProvider imageStreamProvider; private final Monitor monitor; private final BackblazeDataTransferClientFactory b2ClientFactory; - public BackblazePhotosImporter(Monitor monitor, TemporaryPerJobDataStore jobStore) { + public BackblazePhotosImporter( + Monitor monitor, + TemporaryPerJobDataStore jobStore, + ImageStreamProvider imageStreamProvider, + BackblazeDataTransferClientFactory b2ClientFactory) { this.monitor = monitor; this.jobStore = jobStore; - this.b2ClientFactory = new BackblazeDataTransferClientFactory(); + this.imageStreamProvider = imageStreamProvider; + this.b2ClientFactory = b2ClientFactory; } @Override diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java index 97eda843c..38cb112a7 100644 --- a/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/main/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporter.java @@ -37,14 +37,19 @@ public class BackblazeVideosImporter private static final String VIDEO_TRANSFER_MAIN_FOLDER = "Video Transfer"; private final TemporaryPerJobDataStore jobStore; - private final ImageStreamProvider imageStreamProvider = new ImageStreamProvider(); + private final ImageStreamProvider imageStreamProvider; private final Monitor monitor; private final BackblazeDataTransferClientFactory b2ClientFactory; - public BackblazeVideosImporter(Monitor monitor, TemporaryPerJobDataStore jobStore) { + public BackblazeVideosImporter( + Monitor monitor, + TemporaryPerJobDataStore jobStore, + ImageStreamProvider imageStreamProvider, + BackblazeDataTransferClientFactory b2ClientFactory) { this.monitor = monitor; this.jobStore = jobStore; - this.b2ClientFactory = new BackblazeDataTransferClientFactory(); + this.imageStreamProvider = imageStreamProvider; + this.b2ClientFactory = b2ClientFactory; } @Override diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientTest.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientTest.java new file mode 100644 index 000000000..586779e57 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/common/BackblazeDataTransferClientTest.java @@ -0,0 +1,244 @@ +/* + * Copyright 2021 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.datatransferproject.datatransfer.backblaze.common; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.backblaze.exception.BackblazeCredentialsException; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.Bucket; +import software.amazon.awssdk.services.s3.model.BucketAlreadyExistsException; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; +import software.amazon.awssdk.services.s3.model.ListBucketsResponse; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; +import software.amazon.awssdk.services.s3.model.UploadPartResponse; + +@RunWith(MockitoJUnitRunner.class) +public class BackblazeDataTransferClientTest { + @Mock private Monitor monitor; + @Mock private BackblazeS3ClientFactory backblazeS3ClientFactory; + @Mock private S3Client s3Client; + private static File testFile; + private static final String KEY_ID = "keyId"; + private static final String APP_KEY = "appKey"; + private static final String EXPORT_SERVICE = "exp-serv"; + private static final String FILE_KEY = "fileKey"; + private static final String VALID_BUCKET_NAME = EXPORT_SERVICE + "-data-transfer-bucket"; + + @BeforeClass + public static void setUpClass() { + testFile = new File("src/test/resources/test.txt"); + } + + @Before + public void setUp() { + when(backblazeS3ClientFactory.createS3Client(anyString(), anyString(), anyString())) + .thenReturn(s3Client); + } + + private void createValidBucketList() { + Bucket bucket = Bucket.builder().name(VALID_BUCKET_NAME).build(); + when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().buckets(bucket).build()); + } + + private void createEmptyBucketList() { + when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().build()); + } + + private BackblazeDataTransferClient createDefaultClient() { + return new BackblazeDataTransferClient(monitor, backblazeS3ClientFactory, 1000, 500); + } + + @Test(expected = IllegalArgumentException.class) + public void testWrongPartSize() { + // Act + new BackblazeDataTransferClient(monitor, backblazeS3ClientFactory, 10, 0); + // Assert: expected exception + } + + @Test + public void testInitBucketNameMatches() throws BackblazeCredentialsException, IOException { + // Arrange + createValidBucketList(); + BackblazeDataTransferClient client = createDefaultClient(); + // Act + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); + // Assert: no bucket is created + verify(s3Client, times(0)).createBucket(any(CreateBucketRequest.class)); + } + + @Test + public void testInitBucketCreated() throws BackblazeCredentialsException, IOException { + // Arrange + Bucket bucket = Bucket.builder().name("invalid-name").build(); + when(s3Client.listBuckets()).thenReturn(ListBucketsResponse.builder().buckets(bucket).build()); + BackblazeDataTransferClient client = createDefaultClient(); + // Act + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); + // Assert: bucket created once + verify(s3Client, times(1)).createBucket(any(CreateBucketRequest.class)); + } + + @Test(expected = IOException.class) + public void testInitBucketNameExists() throws BackblazeCredentialsException, IOException { + // Arrange + createEmptyBucketList(); + when(s3Client.createBucket(any(CreateBucketRequest.class))) + .thenThrow(BucketAlreadyExistsException.builder().build()); + BackblazeDataTransferClient client = createDefaultClient(); + // Act + try { + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); + } catch (Exception ex) { + // Assert + verify(monitor, atLeast(1)).info(any()); + throw ex; + } + } + + @Test(expected = IOException.class) + public void testInitErrorCreatingBucket() throws BackblazeCredentialsException, IOException { + // Arrange + createEmptyBucketList(); + when(s3Client.createBucket(any(CreateBucketRequest.class))) + .thenThrow(AwsServiceException.builder().build()); + BackblazeDataTransferClient client = createDefaultClient(); + // Act + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); + // Assert: expected exception + } + + @Test(expected = BackblazeCredentialsException.class) + public void testInitListBucketException() throws BackblazeCredentialsException, IOException { + // Arrange + when(s3Client.listBuckets()).thenThrow(S3Exception.builder().statusCode(403).build()); + BackblazeDataTransferClient client = createDefaultClient(); + // Act + try { + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); + } catch (Exception ex) { + // Assert: closed client + verify(s3Client, atLeast(1)).close(); + verify(monitor, atLeast(1)).debug(any()); + throw ex; + } + } + + @Test(expected = IllegalStateException.class) + public void testUploadFileNonInitialized() throws IOException { + // Arrange + BackblazeDataTransferClient client = createDefaultClient(); + // Act + client.uploadFile(FILE_KEY, testFile); + // Assert: expected exception + } + + @Test + public void testUploadFileSingle() throws BackblazeCredentialsException, IOException { + // Arrange + final String expectedVersionId = "123"; + createValidBucketList(); + when(s3Client.putObject(any(PutObjectRequest.class), any(RequestBody.class))) + .thenReturn(PutObjectResponse.builder().versionId(expectedVersionId).build()); + BackblazeDataTransferClient client = createDefaultClient(); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); + // Act + String actualVersionId = client.uploadFile(FILE_KEY, testFile); + // Assert + verify(s3Client, times(1)).putObject(any(PutObjectRequest.class), any(RequestBody.class)); + assertEquals(expectedVersionId, actualVersionId); + } + + @Test(expected = IOException.class) + public void testUploadFileSingleException() throws BackblazeCredentialsException, IOException { + // Arrange + createValidBucketList(); + when(s3Client.putObject(any(PutObjectRequest.class), any(RequestBody.class))) + .thenThrow(AwsServiceException.builder().build()); + BackblazeDataTransferClient client = createDefaultClient(); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); + // Act + client.uploadFile(FILE_KEY, testFile); + // Assert: expected exception + } + + @Test + public void testUploadFileMultipart() throws BackblazeCredentialsException, IOException { + // Arrange + final String expectedVersionId = "123"; + createValidBucketList(); + when(s3Client.createMultipartUpload(any(CreateMultipartUploadRequest.class))) + .thenReturn(CreateMultipartUploadResponse.builder().uploadId("xyz").build()); + when(s3Client.uploadPart(any(UploadPartRequest.class), any(RequestBody.class))) + .thenReturn(UploadPartResponse.builder().build()); + when(s3Client.completeMultipartUpload(any(CompleteMultipartUploadRequest.class))) + .thenReturn(CompleteMultipartUploadResponse.builder().versionId(expectedVersionId).build()); + final long partSize = 10; + final long fileSize = testFile.length(); + final long expectedParts = fileSize / partSize + (fileSize % partSize == 0 ? 0 : 1); + BackblazeDataTransferClient client = + new BackblazeDataTransferClient(monitor, backblazeS3ClientFactory, fileSize / 2, partSize); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); + // Act + String actualVersionId = client.uploadFile(FILE_KEY, testFile); + // Assert: uploaded in 8 parts + verify(s3Client, times((int) expectedParts)) + .uploadPart(any(UploadPartRequest.class), any(RequestBody.class)); + assertEquals(expectedVersionId, actualVersionId); + } + + @Test(expected = IOException.class) + public void testUploadFileMultipartException() throws BackblazeCredentialsException, IOException { + // Arrange + createValidBucketList(); + when(s3Client.createMultipartUpload(any(CreateMultipartUploadRequest.class))) + .thenReturn(CreateMultipartUploadResponse.builder().uploadId("xyz").build()); + when(s3Client.uploadPart(any(UploadPartRequest.class), any(RequestBody.class))) + .thenThrow(AwsServiceException.builder().build()); + final long fileSize = testFile.length(); + BackblazeDataTransferClient client = + new BackblazeDataTransferClient(monitor, backblazeS3ClientFactory, fileSize / 2, fileSize / 8); + client.init(KEY_ID, APP_KEY, EXPORT_SERVICE); + // Act + client.uploadFile(FILE_KEY, testFile); + // Assert: expected exception + } +} diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporterTest.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporterTest.java new file mode 100644 index 000000000..b79139909 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporterTest.java @@ -0,0 +1,152 @@ +/* + * Copyright 2021 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.datatransferproject.datatransfer.backblaze.photos; + +import static junit.framework.TestCase.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.net.HttpURLConnection; +import java.util.ArrayList; +import java.util.UUID; +import java.util.concurrent.Callable; +import org.apache.commons.io.IOUtils; +import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClient; +import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClientFactory; +import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore; +import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor; +import org.datatransferproject.spi.transfer.provider.ImportResult; +import org.datatransferproject.transfer.ImageStreamProvider; +import org.datatransferproject.types.common.models.photos.PhotoAlbum; +import org.datatransferproject.types.common.models.photos.PhotoModel; +import org.datatransferproject.types.common.models.photos.PhotosContainerResource; +import org.datatransferproject.types.transfer.auth.TokenSecretAuthData; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +public class BackblazePhotosImporterTest { + Monitor monitor; + TemporaryPerJobDataStore dataStore; + ImageStreamProvider streamProvider; + BackblazeDataTransferClientFactory clientFactory; + IdempotentImportExecutor executor; + TokenSecretAuthData authData; + BackblazeDataTransferClient client; + + @Before + public void setUp() { + monitor = mock(Monitor.class); + dataStore = mock(TemporaryPerJobDataStore.class); + streamProvider = mock(ImageStreamProvider.class); + clientFactory = mock(BackblazeDataTransferClientFactory.class); + executor = mock(IdempotentImportExecutor.class); + authData = mock(TokenSecretAuthData.class); + client = mock(BackblazeDataTransferClient.class); + } + + @Test + public void testNullData() throws Exception { + BackblazePhotosImporter sut = + new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory); + ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, null); + assertEquals(ImportResult.OK, result); + } + + @Test + public void testNullPhotosAndAlbums() throws Exception { + PhotosContainerResource data = mock(PhotosContainerResource.class); + when(data.getAlbums()).thenReturn(null); + when(data.getPhotos()).thenReturn(null); + + BackblazePhotosImporter sut = + new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory); + ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, data); + assertEquals(ImportResult.OK, result); + } + + @Test + public void testEmptyPhotosAndAlbums() throws Exception { + PhotosContainerResource data = mock(PhotosContainerResource.class); + when(data.getAlbums()).thenReturn(new ArrayList<>()); + when(data.getPhotos()).thenReturn(new ArrayList<>()); + + BackblazePhotosImporter sut = + new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory); + ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, data); + assertEquals(ImportResult.OK, result); + } + + @Test + public void testImportPhoto() throws Exception { + String dataId = "dataId"; + String title = "title"; + String photoUrl = "photoUrl"; + String albumName = "albumName"; + String albumId = "albumId"; + String response = "response"; + + PhotoModel photoModel = new PhotoModel(title, photoUrl, "", "", dataId, albumId, false, null); + ArrayList photos = new ArrayList<>(); + photos.add(photoModel); + PhotosContainerResource data = mock(PhotosContainerResource.class); + when(data.getPhotos()).thenReturn(photos); + + when(executor.getCachedValue(albumId)).thenReturn(albumName); + + HttpURLConnection connection = mock(HttpURLConnection.class); + when(connection.getInputStream()).thenReturn(IOUtils.toInputStream("photo content", "UTF-8")); + when(streamProvider.getConnection(photoUrl)).thenReturn(connection); + + when(client.uploadFile(eq("Photo Transfer/albumName/dataId.jpg"), any())).thenReturn(response); + when(clientFactory.getOrCreateB2Client(monitor, authData)).thenReturn(client); + + BackblazePhotosImporter sut = + new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory); + sut.importItem(UUID.randomUUID(), executor, authData, data); + + ArgumentCaptor> importCapture = ArgumentCaptor.forClass(Callable.class); + verify(executor, times(1)) + .executeAndSwallowIOExceptions(eq(dataId), eq(title), importCapture.capture()); + + String actual = importCapture.getValue().call(); + assertEquals(response, actual); + } + + @Test + public void testImportAlbum() throws Exception { + String albumId = "albumId"; + PhotoAlbum album = new PhotoAlbum(albumId, "", ""); + ArrayList albums = new ArrayList<>(); + albums.add(album); + PhotosContainerResource data = mock(PhotosContainerResource.class); + when(data.getAlbums()).thenReturn(albums); + + BackblazePhotosImporter sut = + new BackblazePhotosImporter(monitor, dataStore, streamProvider, clientFactory); + sut.importItem(UUID.randomUUID(), executor, authData, data); + + verify(executor, times(1)) + .executeAndSwallowIOExceptions( + eq(albumId), eq("Caching album name for album 'albumId'"), any()); + } +} diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporterTest.java b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporterTest.java new file mode 100644 index 000000000..ce928567c --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/videos/BackblazeVideosImporterTest.java @@ -0,0 +1,134 @@ +/* + * Copyright 2021 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.datatransferproject.datatransfer.backblaze.videos; + +import static junit.framework.TestCase.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.net.HttpURLConnection; +import java.util.ArrayList; +import java.util.UUID; +import java.util.concurrent.Callable; +import org.apache.commons.io.IOUtils; +import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClient; +import org.datatransferproject.datatransfer.backblaze.common.BackblazeDataTransferClientFactory; +import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore; +import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor; +import org.datatransferproject.spi.transfer.provider.ImportResult; +import org.datatransferproject.transfer.ImageStreamProvider; +import org.datatransferproject.types.common.models.videos.VideoModel; +import org.datatransferproject.types.common.models.videos.VideosContainerResource; +import org.datatransferproject.types.transfer.auth.TokenSecretAuthData; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +public class BackblazeVideosImporterTest { + Monitor monitor; + TemporaryPerJobDataStore dataStore; + ImageStreamProvider streamProvider; + BackblazeDataTransferClientFactory clientFactory; + IdempotentImportExecutor executor; + TokenSecretAuthData authData; + BackblazeDataTransferClient client; + + @Before + public void setUp() { + monitor = mock(Monitor.class); + dataStore = mock(TemporaryPerJobDataStore.class); + streamProvider = mock(ImageStreamProvider.class); + clientFactory = mock(BackblazeDataTransferClientFactory.class); + executor = mock(IdempotentImportExecutor.class); + authData = mock(TokenSecretAuthData.class); + client = mock(BackblazeDataTransferClient.class); + } + + @Test + public void testNullData() throws Exception { + BackblazeVideosImporter sut = + new BackblazeVideosImporter(monitor, dataStore, streamProvider, clientFactory); + ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, null); + assertEquals(ImportResult.OK, result); + } + + @Test + public void testNullVideos() throws Exception { + VideosContainerResource data = mock(VideosContainerResource.class); + when(data.getVideos()).thenReturn(null); + + BackblazeVideosImporter sut = + new BackblazeVideosImporter(monitor, dataStore, streamProvider, clientFactory); + ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, data); + assertEquals(ImportResult.OK, result); + } + + @Test + public void testEmptyVideos() throws Exception { + VideosContainerResource data = mock(VideosContainerResource.class); + when(data.getVideos()).thenReturn(new ArrayList<>()); + + BackblazeVideosImporter sut = + new BackblazeVideosImporter(monitor, dataStore, streamProvider, clientFactory); + ImportResult result = sut.importItem(UUID.randomUUID(), executor, authData, data); + assertEquals(ImportResult.OK, result); + } + + @Test + public void testImportVideo() throws Exception { + String dataId = "dataId"; + String title = "title"; + String videoUrl = "videoUrl"; + String description = "description"; + String encodingFormat = "UTF-8"; + String albumName = "albumName"; + String albumId = "albumId"; + String response = "response"; + + VideoModel videoObject = + new VideoModel(title, videoUrl, description, encodingFormat, dataId, albumId, false); + ArrayList videos = new ArrayList<>(); + videos.add(videoObject); + VideosContainerResource data = mock(VideosContainerResource.class); + when(data.getVideos()).thenReturn(videos); + + when(executor.getCachedValue(albumId)).thenReturn(albumName); + + HttpURLConnection connection = mock(HttpURLConnection.class); + when(connection.getInputStream()).thenReturn(IOUtils.toInputStream("video content", "UTF-8")); + when(streamProvider.getConnection(videoUrl)).thenReturn(connection); + + when(client.uploadFile(eq("Video Transfer/dataId.mp4"), any())).thenReturn(response); + when(clientFactory.getOrCreateB2Client(monitor, authData)).thenReturn(client); + + BackblazeVideosImporter sut = + new BackblazeVideosImporter(monitor, dataStore, streamProvider, clientFactory); + sut.importItem(UUID.randomUUID(), executor, authData, data); + + ArgumentCaptor> importCapture = ArgumentCaptor.forClass(Callable.class); + verify(executor, times(1)) + .executeAndSwallowIOExceptions(eq(dataId), eq(title), importCapture.capture()); + + String actual = importCapture.getValue().call(); + assertEquals(response, actual); + } +} diff --git a/extensions/data-transfer/portability-data-transfer-backblaze/src/test/resources/test.txt b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/resources/test.txt new file mode 100644 index 000000000..0234876c7 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-backblaze/src/test/resources/test.txt @@ -0,0 +1,3 @@ +Oh, hi there! + +I'm just a test file, hanging around here waiting to be read.