From bba6ebf159438f268eb1309549a1ed3bd36e7617 Mon Sep 17 00:00:00 2001 From: slaurenz <82034561+slaurenz@users.noreply.github.com> Date: Thu, 3 Mar 2022 13:32:32 +0100 Subject: [PATCH] Added data type to partitions and slices --- .../entity/PartitionEntity.java | 10 ++++++++++ .../revocationdistribution/entity/SliceEntity.java | 14 ++++++++++++-- .../revocationdistribution/model/SliceType.java | 2 +- .../service/GeneratorService.java | 8 ++++++-- .../service/SliceCalculation.java | 4 ++++ .../service/SliceCalculationBloomFilterImpl.java | 6 ++++++ .../service/SliceCalculationHashListImpl.java | 7 ++++++- .../db/changelog/006_create_partitions_table.sql | 1 + .../db/changelog/008_create_slices_table.sql | 1 + 9 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/PartitionEntity.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/PartitionEntity.java index 5bfe5f5..2624146 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/PartitionEntity.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/PartitionEntity.java @@ -23,10 +23,13 @@ import com.fasterxml.jackson.annotation.JsonFormat; import com.vladmihalcea.hibernate.type.json.JsonBinaryType; import eu.europa.ec.dgc.revocationdistribution.dto.PartitionChunksJsonItemDto; +import eu.europa.ec.dgc.revocationdistribution.model.SliceType; import java.time.ZonedDateTime; import java.util.Map; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; @@ -106,6 +109,13 @@ public class PartitionEntity { @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ssXXX", timezone = "UTC") private ZonedDateTime expired; + /** + * The type of the binary data e.g. BLOOMFILTER or VARHASHLIST + */ + @Column(name = "data_type") + @Enumerated(EnumType.STRING) + private SliceType dataType; + /** * The hashes of the chunk. */ diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/SliceEntity.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/SliceEntity.java index 2a2373e..2c9d8a1 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/SliceEntity.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/SliceEntity.java @@ -21,9 +21,12 @@ package eu.europa.ec.dgc.revocationdistribution.entity; import com.fasterxml.jackson.annotation.JsonFormat; +import eu.europa.ec.dgc.revocationdistribution.model.SliceType; import java.time.ZonedDateTime; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; @@ -60,7 +63,7 @@ public class SliceEntity { private String kid; /** - * Id . + * Id. */ @Column(name = "partition_id") private String id; @@ -72,7 +75,7 @@ public class SliceEntity { private String chunk; /** - * chunk of slice. + * hash of slice. */ @Column(name = "hash") private String hash; @@ -97,6 +100,13 @@ public class SliceEntity { @Column(name = "slice_binary_data") private byte[] binaryData; + /** + * The type of the binary data e.g. BLOOMFILTER or VARHASHLIST + */ + @Column(name = "data_type") + @Enumerated(EnumType.STRING) + private SliceType dataType; + /** * Indicates if the slice needs to be deleted on etag change. */ diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/SliceType.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/SliceType.java index 1def4f7..026c143 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/SliceType.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/SliceType.java @@ -2,5 +2,5 @@ public enum SliceType { BLOOMFILTER, - HASHLIST + VARHASHLIST } diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/GeneratorService.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/GeneratorService.java index fbb1e51..b89d94f 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/GeneratorService.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/GeneratorService.java @@ -34,6 +34,7 @@ import eu.europa.ec.dgc.revocationdistribution.mapper.VectorViewMapper; import eu.europa.ec.dgc.revocationdistribution.model.ChangeList; import eu.europa.ec.dgc.revocationdistribution.model.ChangeListItem; +import eu.europa.ec.dgc.revocationdistribution.model.SliceType; import eu.europa.ec.dgc.revocationdistribution.repository.CoordinateViewRepository; import eu.europa.ec.dgc.revocationdistribution.repository.KidViewRepository; import eu.europa.ec.dgc.revocationdistribution.repository.PartitionRepository; @@ -313,7 +314,8 @@ private void generatePartition(List entities, chunksJson.put(mve.getChunk(), chunkItemsMap); saveSlice(mve.getKid(), id, mve.getChunk(), sliceDataDto.getMetaData().getHash(), - mve.getLastUpdated(), mve.getExpired(), sliceDataDto.getBinaryData()); + mve.getLastUpdated(), mve.getExpired(), sliceCalculationService.getSliceType(), + sliceDataDto.getBinaryData()); x = mve.getX(); y = mve.getY(); @@ -331,7 +333,7 @@ private void generatePartition(List entities, } private void saveSlice(String kid, String id, String chunk, String hash, - ZonedDateTime lastUpdated, ZonedDateTime expired, byte[] binaryData) { + ZonedDateTime lastUpdated, ZonedDateTime expired, SliceType dataType, byte[] binaryData) { SliceEntity sliceEntity = new SliceEntity(); @@ -342,6 +344,7 @@ private void saveSlice(String kid, String id, String chunk, String hash, sliceEntity.setHash(hash); sliceEntity.setLastUpdated(lastUpdated); sliceEntity.setExpired(expired); + sliceEntity.setDataType(dataType); sliceEntity.setBinaryData(binaryData); sliceEntity.setToBeDeleted(false); @@ -364,6 +367,7 @@ private void savePartition(String kid, String id, String x, String y, String z, partitionEntity.setZ(z); partitionEntity.setLastUpdated(lastUpdated); partitionEntity.setExpired(expired); + partitionEntity.setDataType(SliceType.BLOOMFILTER); partitionEntity.setChunks(chunksJson); partitionEntity.setToBeDeleted(false ); diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculation.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculation.java index 3171c14..4bc2303 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculation.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculation.java @@ -21,9 +21,13 @@ package eu.europa.ec.dgc.revocationdistribution.service; import eu.europa.ec.dgc.revocationdistribution.dto.SliceDataDto; +import eu.europa.ec.dgc.revocationdistribution.model.SliceType; public interface SliceCalculation { + SliceType getSliceType(); + SliceDataDto calculateSlice(String[] hashes); + } diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculationBloomFilterImpl.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculationBloomFilterImpl.java index b046cd8..58072b7 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculationBloomFilterImpl.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculationBloomFilterImpl.java @@ -45,6 +45,12 @@ public class SliceCalculationBloomFilterImpl implements SliceCalculation { private final DgcConfigProperties properties; private final HelperFunctions helperFunctions; + + @Override + public SliceType getSliceType() { + return SliceType.BLOOMFILTER; + } + @Override public SliceDataDto calculateSlice(String[] hashes) { if (hashes.length <= 0) { diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculationHashListImpl.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculationHashListImpl.java index 1ddb55f..c9de9ca 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculationHashListImpl.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/SliceCalculationHashListImpl.java @@ -46,6 +46,11 @@ public class SliceCalculationHashListImpl implements SliceCalculation { private final HelperFunctions helperFunctions; private static final int NUMBER_OF_BYTES_TO_STORE = 2; + @Override + public SliceType getSliceType() { + return SliceType.VARHASHLIST; + } + @Override public SliceDataDto calculateSlice(String[] hashes) { if (hashes.length <= 0) { @@ -56,7 +61,7 @@ public SliceDataDto calculateSlice(String[] hashes) { SliceDataDto sliceDataDto = new SliceDataDto(); - sliceDataDto.getMetaData().setType(SliceType.HASHLIST.name()); + sliceDataDto.getMetaData().setType(SliceType.VARHASHLIST.name()); sliceDataDto.getMetaData().setVersion(properties.getHashList().getVersion()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/src/main/resources/db/changelog/006_create_partitions_table.sql b/src/main/resources/db/changelog/006_create_partitions_table.sql index 247c3f9..687676b 100644 --- a/src/main/resources/db/changelog/006_create_partitions_table.sql +++ b/src/main/resources/db/changelog/006_create_partitions_table.sql @@ -13,6 +13,7 @@ CREATE TABLE IF NOT EXISTS partitions expired timestamp with time zone, lastupdated timestamp with time zone, to_be_deleted boolean, + data_type text COLLATE pg_catalog."default", chunks_json_data jsonb, CONSTRAINT partitions_pkey PRIMARY KEY (db_id) ) diff --git a/src/main/resources/db/changelog/008_create_slices_table.sql b/src/main/resources/db/changelog/008_create_slices_table.sql index 600a9df..a140681 100644 --- a/src/main/resources/db/changelog/008_create_slices_table.sql +++ b/src/main/resources/db/changelog/008_create_slices_table.sql @@ -12,6 +12,7 @@ CREATE TABLE IF NOT EXISTS slices expired timestamp with time zone, lastupdated timestamp with time zone, to_be_deleted boolean, + data_type text COLLATE pg_catalog."default", slice_binary_data bytea, CONSTRAINT slices_pkey PRIMARY KEY (db_id) )