support generating presigned url on demand (#91)

* support generating presigned url on demand

* cleanup

* fix tests

* add AWS_REGION for tests in CI

.github/workflows/feature.yml

@@ -23,7 +23,7 @@ jobs:
           cache: maven
 
       - name: Build with Maven
-        run: mvn -B package --file pom.xml
+        run: AWS_REGION="ew-west-2" mvn -B package --file pom.xml
 
       - name: DependencyCheck
         uses: dependency-check/Dependency-Check_Action@main

pom.xml

@@ -154,6 +154,11 @@
             <artifactId>aws-java-sdk-sqs</artifactId>
             <version>1.12.315</version>
         </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-s3</artifactId>
+            <version>1.12.310</version>
+        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.datatype</groupId>
             <artifactId>jackson-datatype-jsr310</artifactId>

src/main/java/gov/cabinetoffice/gap/adminbackend/controllers/SubmissionsController.java

@@ -1,13 +1,15 @@
 package gov.cabinetoffice.gap.adminbackend.controllers;
 
-import gov.cabinetoffice.gap.adminbackend.dtos.AddingSignedUrlDTO;
+import gov.cabinetoffice.gap.adminbackend.dtos.S3ObjectKeyDTO;
+import gov.cabinetoffice.gap.adminbackend.dtos.UrlDTO;
 import gov.cabinetoffice.gap.adminbackend.dtos.submission.LambdaSubmissionDefinition;
 import gov.cabinetoffice.gap.adminbackend.dtos.submission.SubmissionExportsDTO;
 import gov.cabinetoffice.gap.adminbackend.enums.GrantExportStatus;
 import gov.cabinetoffice.gap.adminbackend.exceptions.NotFoundException;
 import gov.cabinetoffice.gap.adminbackend.services.FileService;
 import gov.cabinetoffice.gap.adminbackend.services.SecretAuthService;
 import gov.cabinetoffice.gap.adminbackend.services.SubmissionsService;
+import gov.cabinetoffice.gap.adminbackend.services.S3Service;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.media.ArraySchema;
 import io.swagger.v3.oas.annotations.media.Content;
@@ -37,6 +39,8 @@ public class SubmissionsController {
 
     private final SubmissionsService submissionsService;
 
+    private final S3Service s3Service;
+
     private final SecretAuthService secretAuthService;
 
     private final FileService fileService;
@@ -129,20 +133,29 @@ public ResponseEntity updateExportRecordStatus(@PathVariable String batchExportI
         return new ResponseEntity<>(HttpStatus.NO_CONTENT);
     }
 
-    @PatchMapping("/{submissionId}/export-batch/{batchExportId}/signedUrl")
-    @Operation(summary = "Add AWS signed url to batch export for download")
+    @PostMapping("/signed-url")
+    @Operation(summary = "Get presigned link for S3 object key")
+    public ResponseEntity<UrlDTO> getPresignedUrl(@RequestBody S3ObjectKeyDTO s3ObjectKeyDTO) {
+        final String objectKey = s3ObjectKeyDTO.getS3ObjectKey();
+        final String presignedUrl = s3Service.generateExportDocSignedUrl(objectKey);
+        return ResponseEntity.ok(new UrlDTO(presignedUrl));
+    }
+
+    @PatchMapping("/{submissionId}/export-batch/{batchExportId}/s3-object-key")
+    @Operation(summary = "Add AWS S3 object key to batch export for download")
     @ApiResponses(value = {
-            @ApiResponse(responseCode = "204", description = "Successfully added signed url",
+            @ApiResponse(responseCode = "204", description = "Successfully added S3 key",
                     content = @Content(mediaType = "application/json")),
             @ApiResponse(responseCode = "400",
                     description = "Required path variables and body not provided in expected format",
                     content = @Content(mediaType = "application/json")),
-            @ApiResponse(responseCode = "500", description = "Something went wrong while updating signed url",
+            @ApiResponse(responseCode = "500", description = "Something went wrong while updating S3 key",
                     content = @Content(mediaType = "application/json")) })
     public ResponseEntity updateExportRecordLocation(@PathVariable UUID batchExportId, @PathVariable UUID submissionId,
-            @RequestBody AddingSignedUrlDTO signedUrlDTO, @RequestHeader(HttpHeaders.AUTHORIZATION) String authHeader) {
+            @RequestBody S3ObjectKeyDTO s3ObjectKeyDTO, @RequestHeader(HttpHeaders.AUTHORIZATION) String authHeader) {
         secretAuthService.authenticateSecret(authHeader);
-        submissionsService.addSignedUrlToSubmissionExport(submissionId, batchExportId, signedUrlDTO.getSignedUrl());
+        submissionsService.addS3ObjectKeyToSubmissionExport(submissionId, batchExportId,
+                s3ObjectKeyDTO.getS3ObjectKey());
         return ResponseEntity.noContent().build();
     }
 

src/main/java/gov/cabinetoffice/gap/adminbackend/dtos/AddingSignedUrlDTO.java → src/main/java/gov/cabinetoffice/gap/adminbackend/dtos/S3ObjectKeyDTO.java

@@ -7,8 +7,8 @@
 @Data
 @AllArgsConstructor
 @NoArgsConstructor
-public class AddingSignedUrlDTO {
+public class S3ObjectKeyDTO {
 
-    private String signedUrl;
+    private String s3ObjectKey;
 
 }

src/main/java/gov/cabinetoffice/gap/adminbackend/dtos/UrlDTO.java

@@ -0,0 +1,14 @@
+package gov.cabinetoffice.gap.adminbackend.dtos;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class UrlDTO {
+
+    private String url;
+
+}

src/main/java/gov/cabinetoffice/gap/adminbackend/dtos/submission/SubmissionExportsDTO.java

@@ -9,6 +9,6 @@ public class SubmissionExportsDTO {
 
     private String label;
 
-    private String url;
+    private String s3key;
 
 }

src/main/java/gov/cabinetoffice/gap/adminbackend/repositories/GrantExportRepository.java

@@ -32,9 +32,9 @@ Integer updateExportRecordStatus(@Param("submissionId") String submissionId,
 
     @Transactional
     @Modifying
-    @Query("UPDATE GrantExportEntity e SET e.location = :signedUrl WHERE e.id.exportBatchId = :exportBatchId AND e.id.submissionId = :submissionId")
+    @Query("UPDATE GrantExportEntity e SET e.location = :s3ObjectKey WHERE e.id.exportBatchId = :exportBatchId AND e.id.submissionId = :submissionId")
     void updateExportRecordLocation(@Param("submissionId") UUID submissionId,
-            @Param("exportBatchId") UUID exportBatchId, @Param("signedUrl") String signedUrl);
+            @Param("exportBatchId") UUID exportBatchId, @Param("s3ObjectKey") String s3ObjectKey);
 
     Long countByIdExportBatchIdAndStatusNot(UUID exportGrantId, GrantExportStatus status);
 

src/main/java/gov/cabinetoffice/gap/adminbackend/services/S3Service.java

@@ -0,0 +1,34 @@
+package gov.cabinetoffice.gap.adminbackend.services;
+
+import com.amazonaws.HttpMethod;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
+import lombok.RequiredArgsConstructor;
+import org.springframework.beans.factory.annotation.Value;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+
+import java.time.Instant;
+import java.util.Date;
+
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class S3Service {
+
+    @Value("${cloud.aws.s3.submissions-export-bucket-name}")
+    private String attachmentsBucket;
+
+    private final AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient();
+
+    public String generateExportDocSignedUrl(String objectKey) {
+        int linkTimeoutDuration = 604800;
+        GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(attachmentsBucket,
+                objectKey).withMethod(HttpMethod.GET)
+                        .withExpiration(Date.from(Instant.now().plusSeconds(linkTimeoutDuration)));
+
+        return s3Client.generatePresignedUrl(generatePresignedUrlRequest).toExternalForm();
+    }
+
+}

src/main/java/gov/cabinetoffice/gap/adminbackend/services/SubmissionsService.java

@@ -274,7 +274,7 @@ public List<SubmissionExportsDTO> getCompletedSubmissionExportsForBatch(UUID exp
         List<GrantExportEntity> exports = grantExportRepository.findAllByIdExportBatchIdAndStatusAndCreatedBy(
                 exportBatchId, GrantExportStatus.COMPLETE, adminSession.getGrantAdminId());
 
-        return exports.stream().map(entity -> SubmissionExportsDTO.builder().url(entity.getLocation())
+        return exports.stream().map(entity -> SubmissionExportsDTO.builder().s3key(entity.getLocation())
                 .label(getFilenameFromExportsSignedUrl(entity)).build()).toList();
     }
 
@@ -346,8 +346,8 @@ public void updateExportStatus(String submissionId, String batchExportId, GrantE
         }
     }
 
-    public void addSignedUrlToSubmissionExport(UUID submissionId, UUID exportId, String signedUrl) {
-        grantExportRepository.updateExportRecordLocation(submissionId, exportId, signedUrl);
+    public void addS3ObjectKeyToSubmissionExport(UUID submissionId, UUID exportId, String s3ObjectKey) {
+        grantExportRepository.updateExportRecordLocation(submissionId, exportId, s3ObjectKey);
     }
 
 }

src/test/java/gov/cabinetoffice/gap/adminbackend/controllers/SubmissionsControllerTest.java

@@ -1,6 +1,7 @@
 package gov.cabinetoffice.gap.adminbackend.controllers;
 
-import gov.cabinetoffice.gap.adminbackend.dtos.AddingSignedUrlDTO;
+import gov.cabinetoffice.gap.adminbackend.dtos.S3ObjectKeyDTO;
+import gov.cabinetoffice.gap.adminbackend.dtos.UrlDTO;
 import gov.cabinetoffice.gap.adminbackend.dtos.submission.LambdaSubmissionDefinition;
 import gov.cabinetoffice.gap.adminbackend.dtos.submission.SubmissionExportsDTO;
 import gov.cabinetoffice.gap.adminbackend.enums.GrantExportStatus;
@@ -53,6 +54,9 @@ class SubmissionsControllerTest {
     @MockBean
     private SubmissionsService submissionsService;
 
+    @MockBean
+    private S3Service s3Service;
+
     @MockBean
     private ApplicationFormService applicationFormService;
 
@@ -282,12 +286,12 @@ void updateExportRecordStatus_UnexpectedErrorOccurred() throws Exception {
 
         @Test
         void updateExportRecordLocation_SuccessfullyUpdate() throws Exception {
-            AddingSignedUrlDTO mockRequest = new AddingSignedUrlDTO("link_to_aws.com/path/filename.zip");
+            S3ObjectKeyDTO mockRequest = new S3ObjectKeyDTO("link_to_aws.com/path/filename.zip");
 
-            doNothing().when(submissionsService).addSignedUrlToSubmissionExport(any(), any(), anyString());
+            doNothing().when(submissionsService).addS3ObjectKeyToSubmissionExport(any(), any(), anyString());
 
             MvcResult res = mockMvc.perform(
-                    patch("/submissions/" + UUID.randomUUID() + "/export-batch/" + UUID.randomUUID() + "/signedUrl")
+                    patch("/submissions/" + UUID.randomUUID() + "/export-batch/" + UUID.randomUUID() + "/s3-object-key")
                             .contentType(MediaType.APPLICATION_JSON).content(HelperUtils.asJsonString(mockRequest))
                             .header(HttpHeaders.AUTHORIZATION, LAMBDA_AUTH_HEADER))
                     .andExpect(status().isNoContent()).andReturn();
@@ -297,6 +301,28 @@ void updateExportRecordLocation_SuccessfullyUpdate() throws Exception {
 
     }
 
+    @Nested
+    class getPresignedUrl {
+
+        void getPresignedUrl_HappyPath() throws Exception {
+            doReturn("www.fakeamazon.com/test_file_name").when(s3Service)
+                    .generateExportDocSignedUrl("path/filename.zip");
+            S3ObjectKeyDTO mockRequest = new S3ObjectKeyDTO("path/filename.zip");
+            UrlDTO mockResponse = new UrlDTO("www.fakeamazon.com/test_file_name");
+
+            mockMvc.perform(post("/submissions/signed-url").contentType(MediaType.APPLICATION_JSON)
+                    .content(HelperUtils.asJsonString(mockRequest))).andExpect(status().isOk())
+                    .andExpect(content().json(HelperUtils.asJsonString(mockResponse)));
+        }
+
+        void getPresignedUrl_BadRequest_Body() throws Exception {
+            UrlDTO mockRequest = new UrlDTO("www.doesntmatter.com");
+            mockMvc.perform(post("/submissions/signed-url").contentType(MediaType.APPLICATION_JSON)
+                    .content(HelperUtils.asJsonString(mockRequest))).andExpect(status().isBadRequest());
+        }
+
+    }
+
     @Nested
     class updateExportRecordLocation {
 
@@ -307,29 +333,29 @@ void beforeEach() {
 
         @Test
         void updateExportRecordLocation_BadRequest_PathVariables() throws Exception {
-            AddingSignedUrlDTO mockRequest = new AddingSignedUrlDTO("link_to_aws.com/path/filename.zip");
+            S3ObjectKeyDTO mockRequest = new S3ObjectKeyDTO("path/filename.zip");
 
-            mockMvc.perform(patch("/submissions/1234/export-batch/12345/signedUrl")
+            mockMvc.perform(patch("/submissions/1234/export-batch/12345/s3-object-key")
                     .contentType(MediaType.APPLICATION_JSON).content(HelperUtils.asJsonString(mockRequest))
                     .header(HttpHeaders.AUTHORIZATION, LAMBDA_AUTH_HEADER)).andExpect(status().isBadRequest());
         }
 
         @Test
         void updateExportRecordLocation_BadRequest_RequestBody() throws Exception {
-            mockMvc.perform(patch("/submissions/1234/export-batch/12345/signedUrl")
+            mockMvc.perform(patch("/submissions/1234/export-batch/12345/s3-object-key")
                     .contentType(MediaType.APPLICATION_JSON).content("\"link_to_aws.com/path/filename.zip\"")
                     .header(HttpHeaders.AUTHORIZATION, LAMBDA_AUTH_HEADER)).andExpect(status().isBadRequest());
         }
 
         @Test
-        void updateExportRecordLocation_UnexpectedErrorOccures() throws Exception {
-            AddingSignedUrlDTO mockRequest = new AddingSignedUrlDTO("link_to_aws.com/path/filename.zip");
+        void updateExportRecordLocation_UnexpectedErrorOccurs() throws Exception {
+            S3ObjectKeyDTO mockRequest = new S3ObjectKeyDTO("path/filename.zip");
 
-            doThrow(new RuntimeException()).when(submissionsService).addSignedUrlToSubmissionExport(any(), any(),
+            doThrow(new RuntimeException()).when(submissionsService).addS3ObjectKeyToSubmissionExport(any(), any(),
                     anyString());
 
             mockMvc.perform(
-                    patch("/submissions/" + UUID.randomUUID() + "/export-batch/" + UUID.randomUUID() + "/signedUrl")
+                    patch("/submissions/" + UUID.randomUUID() + "/export-batch/" + UUID.randomUUID() + "/s3-object-key")
                             .contentType(MediaType.APPLICATION_JSON).content(HelperUtils.asJsonString(mockRequest))
                             .header(HttpHeaders.AUTHORIZATION, LAMBDA_AUTH_HEADER))
                     .andExpect(status().isInternalServerError());

src/test/java/gov/cabinetoffice/gap/adminbackend/services/S3ServiceTest.java

@@ -0,0 +1,77 @@
+package gov.cabinetoffice.gap.adminbackend.services;
+
+import com.amazonaws.HttpMethod;
+import com.amazonaws.SdkClientException;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.MockedStatic;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.net.URL;
+import java.time.Instant;
+import java.util.Date;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.*;
+
+public class S3ServiceTest {
+
+    private static S3Service s3Service;
+
+    private static AmazonS3 mockS3Client;
+
+    ArgumentCaptor<GeneratePresignedUrlRequest> presignedUrlRequestCaptor = ArgumentCaptor
+            .forClass(GeneratePresignedUrlRequest.class);
+
+    @BeforeAll
+    static void beforeAll() {
+        s3Service = new S3Service();
+        mockS3Client = mock(AmazonS3.class);
+        ReflectionTestUtils.setField(s3Service, "s3Client", mockS3Client);
+    }
+
+    @BeforeEach
+    void resetMocks() {
+        reset(mockS3Client);
+    }
+
+    @Test
+    void successfullyGenerateExportSignedURL() throws Exception {
+
+        URL mockUrl = new URL("https://mock_url.co.uk/object_path");
+
+        when(mockS3Client.generatePresignedUrl(any())).thenReturn(mockUrl);
+
+        Instant currentInstant = Instant.now();
+        Date mockExpiryDate = Date.from(currentInstant.plusSeconds(604800));
+
+        try (MockedStatic<Date> mockedInstant = mockStatic(Date.class)) {
+            mockedInstant.when(() -> Date.from(any())).thenReturn(mockExpiryDate);
+
+            String response = s3Service.generateExportDocSignedUrl("object_path");
+
+            verify(mockS3Client).generatePresignedUrl(presignedUrlRequestCaptor.capture());
+            GeneratePresignedUrlRequest capturedValues = presignedUrlRequestCaptor.getValue();
+
+            assertEquals(mockUrl.toExternalForm(), response);
+            assertEquals(mockExpiryDate, capturedValues.getExpiration());
+            assertEquals(HttpMethod.GET, capturedValues.getMethod());
+
+        }
+
+    }
+
+    @Test
+    void unableToGenerateSignedURL() throws Exception {
+        when(mockS3Client.generatePresignedUrl(any())).thenThrow(SdkClientException.class);
+
+        assertThrows(SdkClientException.class, () -> s3Service.generateExportDocSignedUrl("object_path"));
+    }
+
+}