Skip to content
This repository has been archived by the owner on May 16, 2023. It is now read-only.

Commit

Permalink
Feat: Endpoint to export Archive (#264)
Browse files Browse the repository at this point in the history
* Work in progress

* Add Unit Test for CSV Export
Add SHA256, Entity-Count and File-Size of CSV to cancellation Entity

* Add Endpoint to download Archive for Partner
Small Refactoring

* Update Role Name
Add required profile to activate Archive Export Controller

* Update Role Name
Add required profile to activate Archive Export Controller

* Add Log Message for Archive Export

* Checkstyle

* Fix Header

* Fix Header
  • Loading branch information
f11h authored Jan 23, 2023
1 parent c812e3d commit fe0763c
Show file tree
Hide file tree
Showing 6 changed files with 400 additions and 62 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,32 @@ public List<Archive> findAll() {
*
* @return {@link List} of {@link Archive}
*/
public List<Archive> findAllByPocId(final String pocId) {
this.em.getTransaction().begin();
public List<Archive> findAllByPocId(final String pocId, final String tenantId) {
em.getTransaction().begin();

final List<Archive> result = em
.createQuery("SELECT a FROM Archive a WHERE a.pocId = ?1", Archive.class)
.setParameter(1, pocId).getResultList();
.createQuery("SELECT a FROM Archive a WHERE a.pocId = ?1 AND a.tenantId = ?2", Archive.class)
.setParameter(1, pocId)
.setParameter(2, tenantId)
.getResultList();

this.em.getTransaction().commit();
em.getTransaction().commit();
return result;
}

/**
* Returns all entries by tenantId.
*
* @return {@link List} of {@link Archive}
*/
public List<Archive> findAllByTenantId(final String tenantId) {
em.getTransaction().begin();

final List<Archive> result = em
.createQuery("SELECT a FROM Archive a WHERE a.tenantId = ?1", Archive.class)
.setParameter(1, tenantId).getResultList();

em.getTransaction().commit();
return result;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ public class SecurityConfig extends KeycloakWebSecurityConfigurerAdapter {
public static final String ROLE_TENANT_COUNTER = "ROLE_c19_quick_tenant_test_counter";
public static final String ROLE_POC_NAT_ADMIN = "ROLE_c19_quick_test_poc_nat_admin";
public static final String ROLE_TERMINATOR = "ROLE_c19_quick_test_terminator";
public static final String ROLE_ARCHIVE_OPERATOR = "ROLE_c19_quick_test_archive_operator";

private static final String API_ROUTE = "/api/**";
private static final String CONFIG_ROUTE = "/api/config/*";
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
/*-
* ---license-start
* Corona-Warn-App / cwa-quick-test-backend
* ---
* Copyright (C) 2021 - 2023 T-Systems International GmbH and all other contributors
* ---
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ---license-end
*/

package app.coronawarn.quicktest.controller;

import static app.coronawarn.quicktest.config.SecurityConfig.ROLE_ARCHIVE_OPERATOR;

import app.coronawarn.quicktest.service.ArchiveService;
import com.opencsv.exceptions.CsvDataTypeMismatchException;
import com.opencsv.exceptions.CsvRequiredFieldEmptyException;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.enums.ParameterIn;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.responses.ApiResponses;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Profile;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.annotation.Secured;
import org.springframework.security.core.Authentication;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.server.ResponseStatusException;


@Slf4j
@RestController
@RequestMapping(value = "/api/archive")
@RequiredArgsConstructor
@Profile("archive_export")
public class ArchiveExportController {

private final ArchiveService archiveService;

/**
* Endpoint for downloading archived entities.
*
* @return CSV with all archived data.
*/
@Operation(
summary = "Download Archive CSV-File",
description = "Creates a CSV-File with all archived data for whole Partner or just one POC ID.",
parameters = {
@Parameter(
in = ParameterIn.PATH,
name = "partnerId",
description = "Partner ID of the PArtner to download data of",
required = true),
@Parameter(
in = ParameterIn.QUERY,
name = "pocId",
description = "Filter for entities with given pocId")
}
)
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "Successful")
})
@GetMapping(value = "/{partnerId}", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE)
@Secured({ROLE_ARCHIVE_OPERATOR})
public ResponseEntity<byte[]> exportArchive(@PathVariable("partnerId") String partnerId,
Authentication authentication) {

try {
ArchiveService.CsvExportFile csv = archiveService.createCsv(partnerId);

log.info("Archive Export triggered for PartnerId: {} by {}, FileSize: {}",
partnerId, authentication.getName(), csv.getCsvBytes().length);

return ResponseEntity
.status(HttpStatus.OK)
.contentLength(csv.getCsvBytes().length)
.contentType(MediaType.APPLICATION_OCTET_STREAM)
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=quicktest_export.csv")
.body(csv.getCsvBytes());

} catch (CsvRequiredFieldEmptyException | CsvDataTypeMismatchException e) {
log.error("Failed to create CSV: {}", e.getMessage());
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Failed to create CSV.");
}
}
}
75 changes: 69 additions & 6 deletions src/main/java/app/coronawarn/quicktest/service/ArchiveService.java
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@
import app.coronawarn.quicktest.service.cryption.CryptionService;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.opencsv.CSVWriter;
import com.opencsv.bean.StatefulBeanToCsv;
import com.opencsv.bean.StatefulBeanToCsvBuilder;
import com.opencsv.exceptions.CsvDataTypeMismatchException;
import com.opencsv.exceptions.CsvRequiredFieldEmptyException;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
Expand All @@ -42,6 +48,7 @@
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import net.javacrumbs.shedlock.core.LockExtender;
Expand Down Expand Up @@ -168,13 +175,18 @@ public void moveToArchiveByTenantId(String tenantId) {
}

/**
* Get longterm archives by pocId.
* Get decrypted entities from longterm archive.
*
* @param pocId (optional) pocId to filter for
* @param tenantId tenantID to filter for
*/
public List<ArchiveCipherDtoV1> getQuicktestsFromLongterm(final String pocId, final String tenantId)
throws JsonProcessingException {
List<Archive> allByPocId = longTermArchiveRepository.findAllByPocId(createHash(pocId));
List<ArchiveCipherDtoV1> dtos = new ArrayList<>(allByPocId.size());
for (Archive archive : allByPocId) {
public List<ArchiveCipherDtoV1> getQuicktestsFromLongterm(final String pocId, final String tenantId) {
List<Archive> entities = pocId != null
? longTermArchiveRepository.findAllByPocId(createHash(pocId), createHash(tenantId))
: longTermArchiveRepository.findAllByTenantId(createHash(tenantId));

List<ArchiveCipherDtoV1> dtos = new ArrayList<>(entities.size());
for (Archive archive : entities) {
try {
final String decrypt = keyProvider.decrypt(archive.getSecret(), tenantId);
final String json = cryptionService.getAesCryption().decrypt(decrypt, archive.getCiphertext());
Expand All @@ -188,6 +200,57 @@ public List<ArchiveCipherDtoV1> getQuicktestsFromLongterm(final String pocId, fi
return dtos;
}

@RequiredArgsConstructor
@Getter
public static class CsvExportFile {
private final byte[] csvBytes;
private final int totalEntityCount;
}

/**
* Create a CSV containing given Quicktest-Archive-Entities.
*
* @param partnerId Partner for which the CSV should be created.
* @return byte-array representing a CSV.
*/
public CsvExportFile createCsv(String partnerId)
throws CsvRequiredFieldEmptyException, CsvDataTypeMismatchException {

StringWriter stringWriter = new StringWriter();
CSVWriter csvWriter = new CSVWriter(
stringWriter,
'\t',
CSVWriter.DEFAULT_QUOTE_CHARACTER,

'\\',
CSVWriter.DEFAULT_LINE_END);

StatefulBeanToCsv<ArchiveCipherDtoV1> beanToCsv =
new StatefulBeanToCsvBuilder<ArchiveCipherDtoV1>(csvWriter).build();

int page = 0;
int pageSize = 500;
int totalEntityCount = 0;
List<ArchiveCipherDtoV1> quicktests;
do {
log.info("Loading Archive Chunk {} for Partner {}", page, partnerId);
quicktests = getQuicktestsFromLongtermByTenantId(partnerId, page, pageSize);
totalEntityCount += quicktests.size();
log.info("Found {} Quicktests in Archive for Chunk {} for Partner {}", quicktests.size(), page, partnerId);
beanToCsv.write(quicktests);
page++;

try {
LockExtender.extendActiveLock(Duration.ofMinutes(10), Duration.ZERO);
} catch (LockExtender.NoActiveLockException ignored) {
// Exception will be thrown if Job is executed outside Scheduler Context
}
} while (!quicktests.isEmpty());
log.info("Got {} Quicktests for Partner {}", totalEntityCount, partnerId);

return new CsvExportFile(stringWriter.toString().getBytes(StandardCharsets.UTF_8), totalEntityCount);
}

/**
* Get longterm archives by tenantId.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,25 +20,17 @@

package app.coronawarn.quicktest.service;

import app.coronawarn.quicktest.archive.domain.ArchiveCipherDtoV1;
import app.coronawarn.quicktest.config.CsvUploadConfig;
import app.coronawarn.quicktest.domain.Cancellation;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.opencsv.CSVWriter;
import com.opencsv.bean.StatefulBeanToCsv;
import com.opencsv.bean.StatefulBeanToCsvBuilder;
import java.io.ByteArrayInputStream;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Duration;
import java.time.ZonedDateTime;
import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import net.javacrumbs.shedlock.core.LockExtender;
import net.javacrumbs.shedlock.spring.annotation.SchedulerLock;
import org.keycloak.representations.idm.GroupRepresentation;
import org.springframework.scheduling.annotation.Scheduled;
Expand Down Expand Up @@ -102,63 +94,29 @@ private void processCsvUploadBatchRecursion(List<Cancellation> cancellations) {
for (Cancellation cancellation : cancellations) {
log.info("Processing CSV for Partner {}", cancellation.getPartnerId());
try {
StringWriter stringWriter = new StringWriter();
CSVWriter csvWriter = new CSVWriter(
stringWriter,
'\t',
CSVWriter.DEFAULT_QUOTE_CHARACTER,

'\\',
CSVWriter.DEFAULT_LINE_END);

StatefulBeanToCsv<ArchiveCipherDtoV1> beanToCsv =
new StatefulBeanToCsvBuilder<ArchiveCipherDtoV1>(csvWriter).build();

int page = 0;
int pageSize = 500;
int totalEntityCount = 0;
List<ArchiveCipherDtoV1> quicktests;
do {
log.info("Loading Archive Chunk {} for Partner {}", page, cancellation.getPartnerId());
quicktests = archiveService.getQuicktestsFromLongtermByTenantId(
cancellation.getPartnerId(), page, pageSize);
totalEntityCount += quicktests.size();
log.info("Found {} Quicktests in Archive for Chunk {} for Partner {}",
quicktests.size(), page, cancellation.getPartnerId());
beanToCsv.write(quicktests);
page++;

try {
LockExtender.extendActiveLock(Duration.ofMinutes(10), Duration.ZERO);
} catch (LockExtender.NoActiveLockException ignored) {
// Exception will be thrown if Job is executed outside Sheduler Context
}
} while (!quicktests.isEmpty());
log.info("Got {} Quicktests for Partner {}", totalEntityCount, cancellation.getPartnerId());

byte[] csvBytes = stringWriter.toString().getBytes(StandardCharsets.UTF_8);
ArchiveService.CsvExportFile csv = archiveService.createCsv(cancellation.getPartnerId());
String objectId = cancellation.getPartnerId() + ".csv";
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(csvBytes.length);
metadata.setContentLength(csv.getCsvBytes().length);

s3Client.putObject(
s3Config.getBucketName(),
objectId,
new ByteArrayInputStream(csvBytes), metadata);
new ByteArrayInputStream(csv.getCsvBytes()), metadata);

log.info("File stored to S3 with id: {}, size: {}, hash: {}",
objectId, csvBytes.length, getHash(csvBytes));
objectId, csv.getCsvBytes().length, getHash(csv.getCsvBytes()));

if (cancellation.getDbEntityCount() == totalEntityCount) {
if (cancellation.getDbEntityCount() == csv.getTotalEntityCount()) {
cancellationService.updateCsvCreated(cancellation, ZonedDateTime.now(), objectId,
getHash(csvBytes), totalEntityCount, csvBytes.length);
getHash(csv.getCsvBytes()), csv.getTotalEntityCount(), csv.getCsvBytes().length);
} else {
log.error("Difference between actual and expected EntityCount in CSV File for partner {}. "
+ "Expected: {}, Acutal: {}, CSV Export will not be marked as finished.",
cancellation.getPartnerId(), cancellation.getDbEntityCount(), totalEntityCount);
+ "Expected: {}, Actual: {}, CSV Export will not be marked as finished.",
cancellation.getPartnerId(), cancellation.getDbEntityCount(), csv.getTotalEntityCount());

cancellationService.updateDataExportError(cancellation, "CSV Export Delta detected. "
+ "Expected: " + cancellation.getDbEntityCount() + " Actual: " + totalEntityCount);
+ "Expected: " + cancellation.getDbEntityCount() + " Actual: " + csv.getTotalEntityCount());
}
} catch (Exception e) {
String errorMessage = e.getClass().getName() + ": " + e.getMessage();
Expand Down
Loading

0 comments on commit fe0763c

Please sign in to comment.