Merge pull request #14 from eu-digital-green-certificates/feat/improv…

…e_download Feat/improve download
eu-digital-green-certificates · Apr 11, 2022 · bec3ed0 · bec3ed0
2 parents e451476 + 7e24350
commit bec3ed0
Show file tree

Hide file tree

Showing 18 changed files with 628 additions and 48 deletions.
diff --git a/pom.xml b/pom.xml
@@ -34,14 +34,14 @@
 		<lombok.version>1.18.22</lombok.version>
 		<springdoc.version>1.6.0</springdoc.version>
 		<mapstruct.version>1.4.2.Final</mapstruct.version>
-		<junit.version>5.8.1</junit.version>
+		<junit.jupiter.version>5.8.2</junit.jupiter.version>
 		<mockito.version>4.1.0</mockito.version>
 		<bcpkix.version>1.70</bcpkix.version>
 		<okhttp.version>4.9.1</okhttp.version>
 		<shedlock.version>4.30.0</shedlock.version>
 		<nimbusds.version>9.9.2</nimbusds.version>
 		<dgc.lib.version>1.1.13</dgc.lib.version>
-		<dgc.bloomfilter.version>0.0.0-09cb38e</dgc.bloomfilter.version>
+		<dgc.bloomfilter.version>0.0.0-29d785a</dgc.bloomfilter.version>
 		<dgc.partialvarriablehashfilter.version>0.0.0-613ab9b</dgc.partialvarriablehashfilter.version>
 		<sap.cloud.sdk.version>3.57.0</sap.cloud.sdk.version>
 		<jjwt.version>0.11.2</jjwt.version>
@@ -144,6 +144,13 @@
 				<type>pom</type>
 				<scope>import</scope>
 			</dependency>
+			<dependency>
+				<groupId>io.zonky.test.postgres</groupId>
+				<artifactId>embedded-postgres-binaries-bom</artifactId>
+				<version>9.6.16</version>
+				<type>pom</type>
+				<scope>import</scope>
+			</dependency>
 		</dependencies>
 	</dependencyManagement>
 
@@ -195,7 +202,7 @@
 		<dependency>
 			<groupId>org.junit.jupiter</groupId>
 			<artifactId>junit-jupiter-api</artifactId>
-			<version>${junit.version}</version>
+			<version>${junit.jupiter.version}</version>
 			<scope>test</scope>
 		</dependency>
 		<dependency>
@@ -301,6 +308,18 @@
 			<version>0.11.2</version>
 			<scope>runtime</scope>
 		</dependency>
+		<dependency>
+			<groupId>io.zonky.test</groupId>
+			<artifactId>embedded-database-spring-test</artifactId>
+			<version>2.1.1</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>io.zonky.test</groupId>
+			<artifactId>embedded-postgres</artifactId>
+			<version>1.2.10</version>
+			<scope>test</scope>
+		</dependency>
 	</dependencies>
 
 	<build>

diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/config/DgcConfigProperties.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/config/DgcConfigProperties.java
@@ -41,6 +41,7 @@ public class DgcConfigProperties {
     @Setter
     public static class GatewayDownload {
         private Integer timeInterval;
+        private Integer downloadLimit;
         private Integer lockLimit;
     }
 

diff --git a/...ain/java/eu/europa/ec/dgc/revocationdistribution/controller/RevocationListController.java b/...ain/java/eu/europa/ec/dgc/revocationdistribution/controller/RevocationListController.java
@@ -79,7 +79,7 @@ public class RevocationListController {
     @GetMapping(path = "lists", produces = MediaType.APPLICATION_JSON_VALUE)
     @Operation(
         summary = "Returns an overview about all available revocation lists.",
-        description = "This method returns an over about available revocation lists for each KID. The response "
+        description = "This method returns an overview about available revocation lists for each KID. The response "
             + "contains for all available KIDs the last modification date, the used hash types etc.",
         tags = {"Revocation Lists"},
         parameters = {
@@ -763,7 +763,7 @@ private SliceType getSliceDataType(String sliceDataTypeHeader) {
             try {
                 return SliceType.valueOf(sliceDataTypeHeader);
             } catch (IllegalArgumentException e) {
-                log.info("Unkown slice data type requested {}", sliceDataTypeHeader);
+                log.info("Unknown slice data type requested {}", sliceDataTypeHeader);
                 throw new BadRequestException("Requested slice data type unknown: " + sliceDataTypeHeader);
             }
         }

diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/HashesEntity.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/HashesEntity.java
@@ -20,8 +20,10 @@
 
 package eu.europa.ec.dgc.revocationdistribution.entity;
 
+import java.util.UUID;
 import javax.persistence.Column;
 import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
 import javax.persistence.Id;
 import javax.persistence.JoinColumn;
 import javax.persistence.OneToOne;
@@ -30,6 +32,8 @@
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
+import org.hibernate.annotations.GenericGenerator;
+import org.springframework.data.domain.Persistable;
 
 
 @Getter
@@ -38,12 +42,17 @@
 @Table(name = "hashes")
 @AllArgsConstructor
 @NoArgsConstructor
-public class HashesEntity {
+public class HashesEntity implements Persistable<UUID> {
+
+
+    @Id
+    @GeneratedValue(generator = "uuid2")
+    @GenericGenerator(name = "uuid2", strategy = "uuid2")
+    private UUID id;
 
     /**
      * The revoked hash.
      */
-    @Id
     @Column(name = "hash", nullable = false)
     private String hash;
 
@@ -87,4 +96,13 @@ public class HashesEntity {
     @Column(name = "updated")
     private boolean updated;
 
+    @Override
+    public UUID getId() {
+        return id;
+    }
+
+    @Override
+    public boolean isNew() {
+        return true;
+    }
 }
diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/ChangeList.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/ChangeList.java
@@ -32,6 +32,6 @@
 public class ChangeList {
     private List<ChangeListItem> created = new ArrayList<>();
     private List<ChangeListItem> updated = new ArrayList<>();
-    private List<ChangeListItem> deleted = new ArrayList<>();
+    private List<String> deletedKids = new ArrayList<>();
 }
 
diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/ChangeListItem.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/ChangeListItem.java
@@ -32,20 +32,17 @@ public class ChangeListItem {
     /**
      * Constructor to create a new Change List item with the data from a kid view entity.
      * @param kve The kid view entity to get the data from.
-     * @param oldStorageMode The old storage mode [POINT, VECTOR, COORDINATE] of the item, if present.
      */
-    public ChangeListItem(KidViewEntity kve, String oldStorageMode) {
+    public ChangeListItem(KidViewEntity kve) {
         this.kidId = kve.getKid();
         this.lastUpdated = kve.getLastUpdated();
         this.expired = kve.getExpired();
         this.newStorageMode = kve.getStorageMode();
-        this.oldStorageMode = oldStorageMode;
+
     }
 
     private String kidId;
 
-    private String oldStorageMode;
-
     private String newStorageMode;
 
     private ZonedDateTime lastUpdated;

diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/repository/HashesRepository.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/repository/HashesRepository.java
@@ -38,10 +38,10 @@ public interface HashesRepository extends JpaRepository<HashesEntity, String> {
     @Query("DELETE HashesEntity h WHERE  h.batch = null")
     void deleteAllOrphanedHashes();
 
-    @Query("SELECT h.id FROM HashesEntity h WHERE h.id IN :hashes")
+    @Query("SELECT h.hash FROM HashesEntity h WHERE h.hash IN :hashes")
     List<String> getHashesPresentInListAndDb(@Param("hashes") List<String> hashes);
 
-    @Query("SELECT h.id FROM HashesEntity h INNER JOIN h.batch b WHERE h.id IN :hashes AND b.expires > :checkTime")
+    @Query("SELECT h.hash FROM HashesEntity h INNER JOIN h.batch b WHERE h.hash IN :hashes AND b.expires > :checkTime")
     List<String> getHashesPresentInListAndDbAndNotExpired(
         @Param("hashes") List<String> hashes,
         @Param("checkTime") ZonedDateTime checkTime);

diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/GeneratorService.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/GeneratorService.java
@@ -36,6 +36,7 @@
 import eu.europa.ec.dgc.revocationdistribution.repository.PointViewRepository;
 import eu.europa.ec.dgc.revocationdistribution.repository.SliceRepository;
 import eu.europa.ec.dgc.revocationdistribution.repository.VectorViewRepository;
+import eu.europa.ec.dgc.revocationdistribution.utils.HelperFunctions;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -78,6 +79,8 @@ public class GeneratorService {
 
     private final CoordinateViewMapper coordinateViewMapper;
 
+    private final HelperFunctions helperFunctions;
+
     private String etag;
     private String oldEtag;
 
@@ -128,52 +131,68 @@ private ChangeList generateList() {
         Map<String, RevocationListJsonResponseItemDto> itemsMap =
             items.stream().collect(Collectors.toMap(i -> i.getKid(), i -> i));
 
+        List<String> goneKids = new ArrayList<>(itemsMap.keySet());
+        goneKids.removeAll(kidViewEntityList.stream().map(KidViewEntity::getKid).collect(Collectors.toList()));
+        changeList.getDeletedKids().addAll(goneKids);
+        log.debug("Gone kid entries: {}", goneKids);
+
+
+        log.trace("Update items start");
         //Update Items
         kidViewEntityList.stream().forEach(kve -> {
 
             if (kve.getTypes().isEmpty() && kve.getExpired() == null) {
                 log.debug("Delete kid entry : {} ", kve.getKid());
                 if (itemsMap.remove(kve.getKid()) != null) {
-                    changeList.getDeleted().add(new ChangeListItem(kve, null));
+                    changeList.getDeletedKids().add(kve.getKid());
                 }
             } else {
-                if (kve.isUpdated()) {
-                    RevocationListJsonResponseItemDto item;
-                    item = new RevocationListJsonResponseItemDto();
-                    item.setKid(kve.getKid());
-                    item.setLastUpdated(kve.getLastUpdated());
-                    item.setHashTypes(kve.getTypes());
-                    item.setMode(kve.getStorageMode());
-                    item.setExpires(kve.getExpired());
 
+                RevocationListJsonResponseItemDto item = getRevocationListJsonItem(kve);
+
+                if (kve.isUpdated() || (!itemsMap.containsKey(kve.getKid()))) {
+
+                    itemsMap.put(item.getKid(), item);
+                    changeList.getUpdated().add(new ChangeListItem(kve));
+
+                } else {
                     RevocationListJsonResponseItemDto oldItem;
-                    oldItem = itemsMap.put(item.getKid(), item);
+                    oldItem = itemsMap.get(kve.getKid());
 
-                    if (oldItem != null) {
-                        changeList.getUpdated().add(new ChangeListItem(kve, oldItem.getMode()));
-                    } else {
-                        changeList.getCreated().add(new ChangeListItem(kve, null));
+                    if (!helperFunctions.compareRevocationListItems(item, oldItem)) {
+                        itemsMap.put(item.getKid(), item);
+                        changeList.getUpdated().add(new ChangeListItem(kve));
                     }
                 }
             }
         });
-
+        log.trace("update items stop");
         RevocationListJsonEntity revocationListJsonEntity = new RevocationListJsonEntity();
         revocationListJsonEntity.setEtag(etag);
         revocationListJsonEntity.setJsonData(new ArrayList<>(itemsMap.values()));
-
+        log.trace("before save");
         revocationListService.saveRevocationListJson(revocationListJsonEntity);
-
-        log.trace(itemsMap.values().toString());
+        log.trace("create list finished");
+        //log.trace(itemsMap.values().toString());
         return changeList;
     }
 
+    private RevocationListJsonResponseItemDto getRevocationListJsonItem(KidViewEntity kve) {
+        RevocationListJsonResponseItemDto item = new RevocationListJsonResponseItemDto();
+
+        item.setKid(kve.getKid());
+        item.setLastUpdated(kve.getLastUpdated());
+        item.setHashTypes(kve.getTypes());
+        item.setMode(kve.getStorageMode());
+        item.setExpires(kve.getExpired());
+
+        return item;
+    }
+
+
     private void handleChangeList(ChangeList changeList) {
         //handle deleted kIds
-        List<String> deletedKids =
-            changeList.getDeleted().stream().map(ChangeListItem::getKidId).collect(Collectors.toList());
-
-        markDataForRemoval(deletedKids);
+        markDataForRemoval(changeList.getDeletedKids());
 
         //handle updated kIds
         List<String> updatedKids =
@@ -238,7 +257,7 @@ private void generatePartitionsForKidInVectorMode(ChangeListItem changeItem) {
         //get all ids for kId
         List<String> partitionIds = vectorViewRepository.findDistinctIdsByKid(changeItem.getKidId());
 
-        log.debug("PartionIds {}", partitionIds);
+        log.debug("PartitionIds {}", partitionIds);
 
         for (String partitionId : partitionIds) {
             List<ChunkMetaViewDto> entities =
@@ -257,7 +276,7 @@ private void generatePartitionsForKidInCoordinateMode(ChangeListItem changeItem)
         //get all ids for kId
         List<String> partitionIds = coordinateViewRepository.findDistinctIdsByKid(changeItem.getKidId());
 
-        log.debug("PartionIds {}", partitionIds);
+        log.debug("PartitionIds {}", partitionIds);
 
         for (String partitionId : partitionIds) {
             List<ChunkMetaViewDto> entities =

diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/PartitionGeneratorService.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/PartitionGeneratorService.java
@@ -66,7 +66,7 @@ public class PartitionGeneratorService {
 
     public void generatePartition(String etag, List<ChunkMetaViewDto> entities,
                                    String kid, String id, String storageMode) {
-
+        log.info("Generate Partition of entities: {}, kId: {}, ID: {}, etag: {}", entities.size(), kid, id, etag);
         if (sliceCalculationBloomFilter.isPresent()) {
             generatePartition(etag, entities, kid, id, sliceCalculationBloomFilter.get(), storageMode);
         }
@@ -98,7 +98,7 @@ private void generatePartition(String etag, List<ChunkMetaViewDto> entities,
             if (!Objects.equals(mve.getKid(), kid) || !Objects.equals(mve.getId(), id)) {
                 log.error("Kid and/or id does not match: kid: {} , {} id {}, {}", kid, mve.getKid(), id, mve.getId());
             } else {
-
+                log.info("Number of hashes per slice: {}",mve.getHashes().length);
                 SliceDataDto sliceDataDto = sliceCalculation.calculateSlice(mve.getHashes(), storageMode);
                 if (sliceDataDto != null) {
                     Map<String, PartitionChunksJsonItemDto> chunkItemsMap;

diff --git a/...uropa/ec/dgc/revocationdistribution/service/RevocationListDownloadServiceGatewayImpl.java b/...uropa/ec/dgc/revocationdistribution/service/RevocationListDownloadServiceGatewayImpl.java
@@ -99,9 +99,9 @@ public void downloadRevocationList() {
 
         boolean needsCalculation = getNeedsCalculation();
 
-        int timeInterval = properties.getRevocationListDownload().getTimeInterval();
+        int downloadLimit = properties.getRevocationListDownload().getDownloadLimit();
 
-        ZonedDateTime abortTime =  ZonedDateTime.now().plusSeconds((timeInterval / 1000) / 2);
+        ZonedDateTime abortTime =  ZonedDateTime.now().plusSeconds(downloadLimit / 1000);
 
         DgcGatewayRevocationListDownloadIterator revocationListIterator;
 
@@ -133,7 +133,7 @@ public void downloadRevocationList() {
                         RevocationBatchDto revocationBatchDto =
                             dgcGatewayDownloadConnector.getRevocationListBatchById(batchListItem.getBatchId());
 
-                        log.trace(revocationBatchDto.toString());
+                        //log.trace(revocationBatchDto.toString());
 
                         revocationListservice.updateRevocationListBatch(batchListItem.getBatchId(), revocationBatchDto);
                         log.info("Downloaded batch: {}", batchListItem.getBatchId());

diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/RevocationListService.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/RevocationListService.java
@@ -76,7 +76,7 @@ public class RevocationListService {
      */
     @Transactional
     public void updateRevocationListBatch(String batchId, RevocationBatchDto revocationBatchDto) {
-
+        log.trace("start batchupdate");
         BatchListEntity batchListEntity =  saveBatchList(batchId, revocationBatchDto);
 
         List<HashesEntity> hashes = new ArrayList<>();
@@ -96,7 +96,9 @@ public void updateRevocationListBatch(String batchId, RevocationBatchDto revocat
                 log.warn("Batch ({}) includes hash with null value, hash is ignored", batchId);
             }
         }
+        log.trace("before db save");
         hashesRepository.saveAll(hashes);
+        log.trace("Stop batchupdate");
     }