diff --git a/pom.xml b/pom.xml index 6514ee6..e227ccd 100644 --- a/pom.xml +++ b/pom.xml @@ -34,14 +34,14 @@ 1.18.22 1.6.0 1.4.2.Final - 5.8.1 + 5.8.2 4.1.0 1.70 4.9.1 4.30.0 9.9.2 1.1.13 - 0.0.0-09cb38e + 0.0.0-29d785a 0.0.0-613ab9b 3.57.0 0.11.2 @@ -144,6 +144,13 @@ pom import + + io.zonky.test.postgres + embedded-postgres-binaries-bom + 9.6.16 + pom + import + @@ -195,7 +202,7 @@ org.junit.jupiter junit-jupiter-api - ${junit.version} + ${junit.jupiter.version} test @@ -301,6 +308,18 @@ 0.11.2 runtime + + io.zonky.test + embedded-database-spring-test + 2.1.1 + test + + + io.zonky.test + embedded-postgres + 1.2.10 + test + diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/config/DgcConfigProperties.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/config/DgcConfigProperties.java index cf86908..bbf3af7 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/config/DgcConfigProperties.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/config/DgcConfigProperties.java @@ -41,6 +41,7 @@ public class DgcConfigProperties { @Setter public static class GatewayDownload { private Integer timeInterval; + private Integer downloadLimit; private Integer lockLimit; } diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/controller/RevocationListController.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/controller/RevocationListController.java index 9bfc528..15ab91c 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/controller/RevocationListController.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/controller/RevocationListController.java @@ -79,7 +79,7 @@ public class RevocationListController { @GetMapping(path = "lists", produces = MediaType.APPLICATION_JSON_VALUE) @Operation( summary = "Returns an overview about all available revocation lists.", - description = "This method returns an over about available revocation lists for each KID. The response " + description = "This method returns an overview about available revocation lists for each KID. The response " + "contains for all available KIDs the last modification date, the used hash types etc.", tags = {"Revocation Lists"}, parameters = { @@ -763,7 +763,7 @@ private SliceType getSliceDataType(String sliceDataTypeHeader) { try { return SliceType.valueOf(sliceDataTypeHeader); } catch (IllegalArgumentException e) { - log.info("Unkown slice data type requested {}", sliceDataTypeHeader); + log.info("Unknown slice data type requested {}", sliceDataTypeHeader); throw new BadRequestException("Requested slice data type unknown: " + sliceDataTypeHeader); } } diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/HashesEntity.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/HashesEntity.java index b804a82..50c756d 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/HashesEntity.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/entity/HashesEntity.java @@ -20,8 +20,10 @@ package eu.europa.ec.dgc.revocationdistribution.entity; +import java.util.UUID; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.GeneratedValue; import javax.persistence.Id; import javax.persistence.JoinColumn; import javax.persistence.OneToOne; @@ -30,6 +32,8 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +import org.hibernate.annotations.GenericGenerator; +import org.springframework.data.domain.Persistable; @Getter @@ -38,12 +42,17 @@ @Table(name = "hashes") @AllArgsConstructor @NoArgsConstructor -public class HashesEntity { +public class HashesEntity implements Persistable { + + + @Id + @GeneratedValue(generator = "uuid2") + @GenericGenerator(name = "uuid2", strategy = "uuid2") + private UUID id; /** * The revoked hash. */ - @Id @Column(name = "hash", nullable = false) private String hash; @@ -87,4 +96,13 @@ public class HashesEntity { @Column(name = "updated") private boolean updated; + @Override + public UUID getId() { + return id; + } + + @Override + public boolean isNew() { + return true; + } } diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/ChangeList.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/ChangeList.java index 407e235..0fc36e3 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/ChangeList.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/ChangeList.java @@ -32,6 +32,6 @@ public class ChangeList { private List created = new ArrayList<>(); private List updated = new ArrayList<>(); - private List deleted = new ArrayList<>(); + private List deletedKids = new ArrayList<>(); } diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/ChangeListItem.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/ChangeListItem.java index a162ed8..d67e6d4 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/ChangeListItem.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/model/ChangeListItem.java @@ -32,20 +32,17 @@ public class ChangeListItem { /** * Constructor to create a new Change List item with the data from a kid view entity. * @param kve The kid view entity to get the data from. - * @param oldStorageMode The old storage mode [POINT, VECTOR, COORDINATE] of the item, if present. */ - public ChangeListItem(KidViewEntity kve, String oldStorageMode) { + public ChangeListItem(KidViewEntity kve) { this.kidId = kve.getKid(); this.lastUpdated = kve.getLastUpdated(); this.expired = kve.getExpired(); this.newStorageMode = kve.getStorageMode(); - this.oldStorageMode = oldStorageMode; + } private String kidId; - private String oldStorageMode; - private String newStorageMode; private ZonedDateTime lastUpdated; diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/repository/HashesRepository.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/repository/HashesRepository.java index 3f1ade1..2d817cf 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/repository/HashesRepository.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/repository/HashesRepository.java @@ -38,10 +38,10 @@ public interface HashesRepository extends JpaRepository { @Query("DELETE HashesEntity h WHERE h.batch = null") void deleteAllOrphanedHashes(); - @Query("SELECT h.id FROM HashesEntity h WHERE h.id IN :hashes") + @Query("SELECT h.hash FROM HashesEntity h WHERE h.hash IN :hashes") List getHashesPresentInListAndDb(@Param("hashes") List hashes); - @Query("SELECT h.id FROM HashesEntity h INNER JOIN h.batch b WHERE h.id IN :hashes AND b.expires > :checkTime") + @Query("SELECT h.hash FROM HashesEntity h INNER JOIN h.batch b WHERE h.hash IN :hashes AND b.expires > :checkTime") List getHashesPresentInListAndDbAndNotExpired( @Param("hashes") List hashes, @Param("checkTime") ZonedDateTime checkTime); diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/GeneratorService.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/GeneratorService.java index ede9a30..e8a6994 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/GeneratorService.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/GeneratorService.java @@ -36,6 +36,7 @@ import eu.europa.ec.dgc.revocationdistribution.repository.PointViewRepository; import eu.europa.ec.dgc.revocationdistribution.repository.SliceRepository; import eu.europa.ec.dgc.revocationdistribution.repository.VectorViewRepository; +import eu.europa.ec.dgc.revocationdistribution.utils.HelperFunctions; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -78,6 +79,8 @@ public class GeneratorService { private final CoordinateViewMapper coordinateViewMapper; + private final HelperFunctions helperFunctions; + private String etag; private String oldEtag; @@ -128,52 +131,68 @@ private ChangeList generateList() { Map itemsMap = items.stream().collect(Collectors.toMap(i -> i.getKid(), i -> i)); + List goneKids = new ArrayList<>(itemsMap.keySet()); + goneKids.removeAll(kidViewEntityList.stream().map(KidViewEntity::getKid).collect(Collectors.toList())); + changeList.getDeletedKids().addAll(goneKids); + log.debug("Gone kid entries: {}", goneKids); + + + log.trace("Update items start"); //Update Items kidViewEntityList.stream().forEach(kve -> { if (kve.getTypes().isEmpty() && kve.getExpired() == null) { log.debug("Delete kid entry : {} ", kve.getKid()); if (itemsMap.remove(kve.getKid()) != null) { - changeList.getDeleted().add(new ChangeListItem(kve, null)); + changeList.getDeletedKids().add(kve.getKid()); } } else { - if (kve.isUpdated()) { - RevocationListJsonResponseItemDto item; - item = new RevocationListJsonResponseItemDto(); - item.setKid(kve.getKid()); - item.setLastUpdated(kve.getLastUpdated()); - item.setHashTypes(kve.getTypes()); - item.setMode(kve.getStorageMode()); - item.setExpires(kve.getExpired()); + RevocationListJsonResponseItemDto item = getRevocationListJsonItem(kve); + + if (kve.isUpdated() || (!itemsMap.containsKey(kve.getKid()))) { + + itemsMap.put(item.getKid(), item); + changeList.getUpdated().add(new ChangeListItem(kve)); + + } else { RevocationListJsonResponseItemDto oldItem; - oldItem = itemsMap.put(item.getKid(), item); + oldItem = itemsMap.get(kve.getKid()); - if (oldItem != null) { - changeList.getUpdated().add(new ChangeListItem(kve, oldItem.getMode())); - } else { - changeList.getCreated().add(new ChangeListItem(kve, null)); + if (!helperFunctions.compareRevocationListItems(item, oldItem)) { + itemsMap.put(item.getKid(), item); + changeList.getUpdated().add(new ChangeListItem(kve)); } } } }); - + log.trace("update items stop"); RevocationListJsonEntity revocationListJsonEntity = new RevocationListJsonEntity(); revocationListJsonEntity.setEtag(etag); revocationListJsonEntity.setJsonData(new ArrayList<>(itemsMap.values())); - + log.trace("before save"); revocationListService.saveRevocationListJson(revocationListJsonEntity); - - log.trace(itemsMap.values().toString()); + log.trace("create list finished"); + //log.trace(itemsMap.values().toString()); return changeList; } + private RevocationListJsonResponseItemDto getRevocationListJsonItem(KidViewEntity kve) { + RevocationListJsonResponseItemDto item = new RevocationListJsonResponseItemDto(); + + item.setKid(kve.getKid()); + item.setLastUpdated(kve.getLastUpdated()); + item.setHashTypes(kve.getTypes()); + item.setMode(kve.getStorageMode()); + item.setExpires(kve.getExpired()); + + return item; + } + + private void handleChangeList(ChangeList changeList) { //handle deleted kIds - List deletedKids = - changeList.getDeleted().stream().map(ChangeListItem::getKidId).collect(Collectors.toList()); - - markDataForRemoval(deletedKids); + markDataForRemoval(changeList.getDeletedKids()); //handle updated kIds List updatedKids = @@ -238,7 +257,7 @@ private void generatePartitionsForKidInVectorMode(ChangeListItem changeItem) { //get all ids for kId List partitionIds = vectorViewRepository.findDistinctIdsByKid(changeItem.getKidId()); - log.debug("PartionIds {}", partitionIds); + log.debug("PartitionIds {}", partitionIds); for (String partitionId : partitionIds) { List entities = @@ -257,7 +276,7 @@ private void generatePartitionsForKidInCoordinateMode(ChangeListItem changeItem) //get all ids for kId List partitionIds = coordinateViewRepository.findDistinctIdsByKid(changeItem.getKidId()); - log.debug("PartionIds {}", partitionIds); + log.debug("PartitionIds {}", partitionIds); for (String partitionId : partitionIds) { List entities = diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/PartitionGeneratorService.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/PartitionGeneratorService.java index a437864..af464d5 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/PartitionGeneratorService.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/PartitionGeneratorService.java @@ -66,7 +66,7 @@ public class PartitionGeneratorService { public void generatePartition(String etag, List entities, String kid, String id, String storageMode) { - + log.info("Generate Partition of entities: {}, kId: {}, ID: {}, etag: {}", entities.size(), kid, id, etag); if (sliceCalculationBloomFilter.isPresent()) { generatePartition(etag, entities, kid, id, sliceCalculationBloomFilter.get(), storageMode); } @@ -98,7 +98,7 @@ private void generatePartition(String etag, List entities, if (!Objects.equals(mve.getKid(), kid) || !Objects.equals(mve.getId(), id)) { log.error("Kid and/or id does not match: kid: {} , {} id {}, {}", kid, mve.getKid(), id, mve.getId()); } else { - + log.info("Number of hashes per slice: {}",mve.getHashes().length); SliceDataDto sliceDataDto = sliceCalculation.calculateSlice(mve.getHashes(), storageMode); if (sliceDataDto != null) { Map chunkItemsMap; diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/RevocationListDownloadServiceGatewayImpl.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/RevocationListDownloadServiceGatewayImpl.java index 9f313fa..e9d49f7 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/RevocationListDownloadServiceGatewayImpl.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/RevocationListDownloadServiceGatewayImpl.java @@ -99,9 +99,9 @@ public void downloadRevocationList() { boolean needsCalculation = getNeedsCalculation(); - int timeInterval = properties.getRevocationListDownload().getTimeInterval(); + int downloadLimit = properties.getRevocationListDownload().getDownloadLimit(); - ZonedDateTime abortTime = ZonedDateTime.now().plusSeconds((timeInterval / 1000) / 2); + ZonedDateTime abortTime = ZonedDateTime.now().plusSeconds(downloadLimit / 1000); DgcGatewayRevocationListDownloadIterator revocationListIterator; @@ -133,7 +133,7 @@ public void downloadRevocationList() { RevocationBatchDto revocationBatchDto = dgcGatewayDownloadConnector.getRevocationListBatchById(batchListItem.getBatchId()); - log.trace(revocationBatchDto.toString()); + //log.trace(revocationBatchDto.toString()); revocationListservice.updateRevocationListBatch(batchListItem.getBatchId(), revocationBatchDto); log.info("Downloaded batch: {}", batchListItem.getBatchId()); diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/RevocationListService.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/RevocationListService.java index 7343892..c50d6e0 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/RevocationListService.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/service/RevocationListService.java @@ -76,7 +76,7 @@ public class RevocationListService { */ @Transactional public void updateRevocationListBatch(String batchId, RevocationBatchDto revocationBatchDto) { - + log.trace("start batchupdate"); BatchListEntity batchListEntity = saveBatchList(batchId, revocationBatchDto); List hashes = new ArrayList<>(); @@ -96,7 +96,9 @@ public void updateRevocationListBatch(String batchId, RevocationBatchDto revocat log.warn("Batch ({}) includes hash with null value, hash is ignored", batchId); } } + log.trace("before db save"); hashesRepository.saveAll(hashes); + log.trace("Stop batchupdate"); } diff --git a/src/main/java/eu/europa/ec/dgc/revocationdistribution/utils/HelperFunctions.java b/src/main/java/eu/europa/ec/dgc/revocationdistribution/utils/HelperFunctions.java index aa5ecb2..7377f10 100644 --- a/src/main/java/eu/europa/ec/dgc/revocationdistribution/utils/HelperFunctions.java +++ b/src/main/java/eu/europa/ec/dgc/revocationdistribution/utils/HelperFunctions.java @@ -20,11 +20,13 @@ package eu.europa.ec.dgc.revocationdistribution.utils; +import eu.europa.ec.dgc.revocationdistribution.dto.RevocationListJsonResponseDto; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; import lombok.extern.slf4j.Slf4j; import org.bouncycastle.util.encoders.Hex; import org.springframework.stereotype.Service; @@ -62,4 +64,23 @@ public byte[] getBytesFromHexString(String hex) { return Hex.decode(hex); } + + /** + * Compare two RevocationListItems for equality. + * @param item1 item to be compared + * @param item2 item to be compared + * @return true if equal + */ + public boolean compareRevocationListItems( + RevocationListJsonResponseDto.RevocationListJsonResponseItemDto item1, + RevocationListJsonResponseDto.RevocationListJsonResponseItemDto item2) { + + return item1.getKid().equals(item2.getKid()) + && item1.getHashTypes().equals(item2.getHashTypes()) + && item1.getMode().equals(item2.getMode()) + && item1.getExpires().truncatedTo(ChronoUnit.SECONDS) + .isEqual(item2.getExpires().truncatedTo(ChronoUnit.SECONDS)) + && item1.getLastUpdated().truncatedTo(ChronoUnit.SECONDS) + .isEqual(item2.getLastUpdated().truncatedTo(ChronoUnit.SECONDS)); + } } diff --git a/src/main/resources/application-debug.yml b/src/main/resources/application-debug.yml new file mode 100644 index 0000000..92f40ec --- /dev/null +++ b/src/main/resources/application-debug.yml @@ -0,0 +1,13 @@ +logging: + level: + eu.europa.ec.dgc: TRACE + default: TRACE + org.hibernate.SQL: INFO + org.hibernate.type.descriptor.sql: INFO + +spring: + jpa: + properties: + hibernate: + generate_statistics: true + show-sql: true diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 004530d..5ede266 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,6 +1,7 @@ logging: level: - default: TRACE + eu.europa.ec.dgc: INFO + default: INFO org.hibernate.SQL: INFO org.hibernate.type.descriptor.sql: INFO server: @@ -17,6 +18,12 @@ spring: database-platform: org.hibernate.dialect.PostgreSQLDialect hibernate: ddl-auto: validate + properties: + hibernate: + jdbc.batch_size: 2000 + order_inserts: true + format_sql: true + liquibase: change-log: classpath:db/changelog.xml database-change-log-table: RD_CHANGELOG @@ -55,14 +62,18 @@ springdoc: api-docs: path: /api/docs enabled: true + writer-with-order-by-keys: true swagger-ui: path: /swagger - tagsSorter=alpha: + tagsSorter: alpha + operationsSorter: alpha + dgc: revocationListDownload: timeInterval: 1800000 - lockLimit: 3600000 + lockLimit: 14400000 + downloadLimit: 90000 #max download time should be less than timeInterval defaultRevocationDataType: BLOOMFILTER # Possible values are BLOOMFILTER and VARHASHLIST bloomFilter: enabled: true diff --git a/src/main/resources/db/changelog.xml b/src/main/resources/db/changelog.xml index 1b06c6b..7d9b118 100644 --- a/src/main/resources/db/changelog.xml +++ b/src/main/resources/db/changelog.xml @@ -20,5 +20,6 @@ + \ No newline at end of file diff --git a/src/main/resources/db/changelog/016_reset_database_alter_hashes.sql b/src/main/resources/db/changelog/016_reset_database_alter_hashes.sql new file mode 100644 index 0000000..80f216d --- /dev/null +++ b/src/main/resources/db/changelog/016_reset_database_alter_hashes.sql @@ -0,0 +1,320 @@ +--liquibase formatted sql +--changeset slaurenz:recreate database splitStatements:false + +-- Drop existing data +drop TABLE IF EXISTS hashes cascade; +drop TABLE IF EXISTS batch_list cascade; +drop TABLE IF EXISTS configuration cascade; +drop TABLE IF EXISTS info cascade; +drop TABLE IF EXISTS partitions cascade; +drop TABLE IF EXISTS revocation_list_json cascade; +drop TABLE IF EXISTS shedlock_rd cascade; +drop TABLE IF EXISTS slices cascade; +DROP FUNCTION IF EXISTS set_last_updated_function(); +DROP FUNCTION IF EXISTS set_new_etag(text); + + +-- recreate batch list + +CREATE TABLE IF NOT EXISTS batch_list +( + batch_id character varying(36) COLLATE pg_catalog."default" NOT NULL, + country character varying(2) COLLATE pg_catalog."default" NOT NULL, + expires timestamp with time zone NOT NULL, + kid character varying(12) COLLATE pg_catalog."default", + type character varying(255) COLLATE pg_catalog."default" NOT NULL, + created_at timestamp with time zone, + CONSTRAINT batch_list_pkey PRIMARY KEY (batch_id) +) +WITH ( + OIDS = FALSE +); + + +-- recreate configuration +CREATE TABLE IF NOT EXISTS configuration +( + key text COLLATE pg_catalog."default" NOT NULL, + value text COLLATE pg_catalog."default", + value2 text COLLATE pg_catalog."default", + CONSTRAINT configuration_pkey PRIMARY KEY (key) +) +WITH ( + OIDS = FALSE +); + +-- recreate +CREATE OR REPLACE FUNCTION set_last_updated_function() + RETURNS trigger + LANGUAGE 'plpgsql' + COST 100 + VOLATILE NOT LEAKPROOF +AS $BODY$ +BEGIN + NEW.last_updated := NOW(); + NEW.updated := true; + RETURN NEW; +END; +$BODY$; + +-- recreate hashes +CREATE TABLE IF NOT EXISTS hashes +( + id uuid NOT NULL, + hash character varying(255) COLLATE pg_catalog."default" NOT NULL, + batch_id character varying(36) COLLATE pg_catalog."default", + kid character varying(12) COLLATE pg_catalog."default", + updated boolean, + x character(1) COLLATE pg_catalog."default" NOT NULL, + y character(1) COLLATE pg_catalog."default" NOT NULL, + z character(1) COLLATE pg_catalog."default" NOT NULL, + last_updated timestamp with time zone DEFAULT now(), + CONSTRAINT hashes_pkey PRIMARY KEY (id), + CONSTRAINT fk_batch_id FOREIGN KEY (batch_id) + REFERENCES batch_list (batch_id) MATCH SIMPLE + ON UPDATE NO ACTION + ON DELETE SET NULL + NOT VALID +) +WITH ( + OIDS = FALSE +); + +-- Index: fki_fk_batch_id +-- DROP INDEX IF EXISTS fki_fk_batch_id; + +CREATE INDEX IF NOT EXISTS fki_fk_batch_id + ON hashes USING btree + (batch_id COLLATE pg_catalog."default" ASC NULLS LAST) + TABLESPACE pg_default; + +-- Trigger: set_last_updated_trigger +-- DROP TRIGGER IF EXISTS set_last_updated_trigger ON hashes; + +CREATE TRIGGER set_last_updated_trigger + BEFORE UPDATE + ON hashes + FOR EACH ROW + WHEN (new.updated IS TRUE OR new.batch_id IS NULL) + EXECUTE PROCEDURE set_last_updated_function(); + +-- recreate info +CREATE TABLE IF NOT EXISTS info +( + key text COLLATE pg_catalog."default" NOT NULL, + value text COLLATE pg_catalog."default", + CONSTRAINT info_pkey PRIMARY KEY (key) +) +WITH ( + OIDS = FALSE +); + +-- recreate partitions +CREATE TABLE IF NOT EXISTS partitions +( + db_id BIGSERIAL, + etag text COLLATE pg_catalog."default" NOT NULL, + kid text COLLATE pg_catalog."default" NOT NULL, + partition_id text COLLATE pg_catalog."default", + x text COLLATE pg_catalog."default", + y text COLLATE pg_catalog."default", + z text COLLATE pg_catalog."default", + expired timestamp with time zone, + lastupdated timestamp with time zone, + to_be_deleted boolean, + data_type text COLLATE pg_catalog."default", + chunks_json_data jsonb, + CONSTRAINT partitions_pkey PRIMARY KEY (db_id) +) +WITH ( + OIDS = FALSE +); + +-- recreate revocation list json + +CREATE TABLE IF NOT EXISTS revocation_list_json +( + etag text COLLATE pg_catalog."default" NOT NULL, + created_at timestamp with time zone, + json_data jsonb, + CONSTRAINT revocation_list_json_pkey PRIMARY KEY (etag) +) +WITH ( + OIDS = FALSE +); + + +-- recreate slices +CREATE TABLE IF NOT EXISTS slices +( + db_id BIGSERIAL, + etag text COLLATE pg_catalog."default" NOT NULL, + kid text COLLATE pg_catalog."default" NOT NULL, + partition_id text COLLATE pg_catalog."default", + chunk text COLLATE pg_catalog."default", + hash text COLLATE pg_catalog."default", + expired timestamp with time zone, + lastupdated timestamp with time zone, + to_be_deleted boolean, + data_type text COLLATE pg_catalog."default", + slice_binary_data bytea, + CONSTRAINT slices_pkey PRIMARY KEY (db_id) +) +WITH ( + OIDS = FALSE +); + +--recreate views +CREATE OR REPLACE VIEW coordinate_view + AS + SELECT row_number() OVER ()::text AS row_id, + CASE + WHEN hashes.kid IS NULL THEN 'UNKNOWN_KID'::character varying + ELSE hashes.kid + END AS kid, + max(date_trunc('minute'::text, batch_list.expires))::timestamp with time zone AS expired, + max(hashes.last_updated) AS lastupdated, + array_agg(DISTINCT hashes.hash) AS hashes, + hashes.z::text AS chunk, + concat(hashes.x, hashes.y) AS partition_id, + hashes.x::text AS x, + hashes.y::text AS y + FROM hashes + LEFT JOIN batch_list ON hashes.batch_id::text = batch_list.batch_id::text + WHERE hashes.batch_id IS NOT NULL + GROUP BY hashes.kid, hashes.x, hashes.y, hashes.z, (date_trunc('minute'::text, batch_list.expires)) + ORDER BY hashes.kid, (concat(hashes.x, hashes.y)), (hashes.z::text), (date_trunc('minute'::text, batch_list.expires)); + + +CREATE OR REPLACE VIEW kid_view + AS + WITH configuration AS ( + SELECT + CASE + WHEN configuration_1.key = 'POINTLIMIT'::text THEN 'POINT'::text + WHEN configuration_1.key = 'VECTORLIMIT'::text THEN 'VECTOR'::text + WHEN configuration_1.key = 'COORDINATELIMIT'::text THEN 'COORDINATE'::text + ELSE NULL::text + END AS storage_mode, + to_number(configuration_1.value, '999999999999'::text) AS minlimit, + to_number(configuration_1.value2, '999999999999'::text) AS maxlimit + FROM configuration configuration_1 + WHERE configuration_1.key = ANY (ARRAY['POINTLIMIT'::text, 'VECTORLIMIT'::text, 'COORDINATELIMIT'::text]) + ) + SELECT a.kid, + a.hashtypes, + configuration.storage_mode, + a.lastupdated, + a.expired, + a.updated + FROM ( SELECT + CASE + WHEN hashes.kid IS NULL THEN 'UNKNOWN_KID'::character varying + ELSE hashes.kid + END AS kid, + count(*) AS c, + array_to_string(array_agg(DISTINCT batch_list.type), ','::text) AS hashtypes, + bool_or(hashes.updated) AS updated, + max(batch_list.expires) AS expired, + max(hashes.last_updated) AS lastupdated + FROM hashes + LEFT JOIN batch_list ON hashes.batch_id::text = batch_list.batch_id::text + GROUP BY hashes.kid) a, + configuration + WHERE a.c::numeric >= configuration.minlimit AND a.c::numeric <= configuration.maxlimit; + + +CREATE OR REPLACE VIEW point_view + AS + SELECT row_number() OVER ()::text AS row_id, + CASE + WHEN hashes.kid IS NULL THEN 'UNKNOWN_KID'::character varying + ELSE hashes.kid + END AS kid, + max(date_trunc('minute'::text, batch_list.expires))::timestamp with time zone AS expired, + max(hashes.last_updated) AS lastupdated, + array_agg(DISTINCT hashes.hash) AS hashes, + hashes.x::text AS chunk, + NULL::text AS partition_id, + NULL::text AS x, + NULL::text AS y + FROM hashes + LEFT JOIN batch_list ON hashes.batch_id::text = batch_list.batch_id::text + WHERE hashes.batch_id IS NOT NULL + GROUP BY hashes.kid, hashes.x, (date_trunc('minute'::text, batch_list.expires)); + + + +CREATE OR REPLACE VIEW vector_view + AS + SELECT row_number() OVER ()::text AS row_id, + CASE + WHEN hashes.kid IS NULL THEN 'UNKNOWN_KID'::character varying + ELSE hashes.kid + END AS kid, + max(date_trunc('minute'::text, batch_list.expires))::timestamp with time zone AS expired, + max(hashes.last_updated) AS lastupdated, + array_agg(DISTINCT hashes.hash) AS hashes, + hashes.y::text AS chunk, + hashes.x::text AS partition_id, + hashes.x::text AS x, + NULL::text AS y + FROM hashes + LEFT JOIN batch_list ON hashes.batch_id::text = batch_list.batch_id::text + WHERE hashes.batch_id IS NOT NULL + GROUP BY hashes.kid, hashes.x, hashes.y, (date_trunc('minute'::text, batch_list.expires)) + ORDER BY hashes.kid, (hashes.x::text), NULL::text, (date_trunc('minute'::text, batch_list.expires)); + + +-- create set new etag function +CREATE OR REPLACE FUNCTION set_new_etag( + new_etag text) + RETURNS integer + LANGUAGE 'plpgsql' + COST 100 + VOLATILE PARALLEL UNSAFE +AS $BODY$ + BEGIN + -- Update etag in info table + INSERT INTO info (key, value) + VALUES('CURRENTETAG', new_etag) + ON CONFLICT (key) + DO UPDATE SET value = EXCLUDED.value; + + -- Delete old slices and partitions + DELETE FROM partitions WHERE to_be_deleted=true; + DELETE FROM slices WHERE to_be_deleted=true; + + -- Update etag field of slices and partitions + Update partitions SET etag = new_etag; + Update slices SET etag = new_etag; + + -- Update etag in info table + INSERT INTO info (key, value) + VALUES('CURRENTETAG', new_etag) + ON CONFLICT (key) + DO UPDATE SET value = EXCLUDED.value; + + RETURN 1; + END; +$BODY$; + +-- reload boundries +INSERT INTO configuration (key, value, value2) VALUES ('POINTLIMIT', '0', '100000'); +INSERT INTO configuration (key, value, value2) VALUES ('VECTORLIMIT', '100001', '1600000'); +INSERT INTO configuration (key, value, value2) VALUES ('COORDINATELIMIT', '1600001', '999999999999'); + +-- recreate schedlock +CREATE TABLE IF NOT EXISTS shedlock_rd +( + id BIGSERIAL, + lock_until timestamp without time zone NOT NULL, + locked_at timestamp without time zone NOT NULL, + locked_by character varying(255) COLLATE pg_catalog."default" NOT NULL, + name character varying(64) COLLATE pg_catalog."default" NOT NULL, + CONSTRAINT shedlock_rd_pkey PRIMARY KEY (id), + CONSTRAINT uk_2ad9gyjxfy85r5k5yssh63e63 UNIQUE (name) +) +WITH ( + OIDS = FALSE +); diff --git a/src/test/java/eu/europa/ec/dgc/revocationdistribution/OpenApiTest.java b/src/test/java/eu/europa/ec/dgc/revocationdistribution/OpenApiTest.java new file mode 100644 index 0000000..63fcf7b --- /dev/null +++ b/src/test/java/eu/europa/ec/dgc/revocationdistribution/OpenApiTest.java @@ -0,0 +1,82 @@ +/*- + * ---license-start + * eu-digital-green-certificates / dgca-businessrule-service + * --- + * Copyright (C) 2021 T-Systems International GmbH and all other contributors + * --- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---license-end + */ + +package eu.europa.ec.dgc.revocationdistribution; + + +import eu.europa.ec.dgc.gateway.connector.DgcGatewayCountryListDownloadConnector; +import eu.europa.ec.dgc.gateway.connector.DgcGatewayValidationRuleDownloadConnector; +import eu.europa.ec.dgc.gateway.connector.DgcGatewayValueSetDownloadConnector; +import eu.europa.ec.dgc.gateway.connector.client.DgcGatewayConnectorRestClientConfig; +import eu.europa.ec.dgc.revocationdistribution.client.IssuanceDgciRestClient; +import io.zonky.test.db.AutoConfigureEmbeddedDatabase; +import java.io.BufferedInputStream; +import java.io.FileOutputStream; +import java.net.URL; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; + +import static io.zonky.test.db.AutoConfigureEmbeddedDatabase.DatabaseProvider.ZONKY; + +@Slf4j +@SpringBootTest( + properties = { + "server.port=8080", + "springdoc.api-docs.enabled=true", + "springdoc.api-docs.path=/openapi", + "dgc.gateway.connector.enabled=false" + }, + webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT +) +@AutoConfigureEmbeddedDatabase(provider = ZONKY) +class OpenApiTest { + + @MockBean + DgcGatewayConnectorRestClientConfig dgcGatewayConnectorRestClientConfig; + + @MockBean + IssuanceDgciRestClient issuanceDgciRestClient; + @MockBean + DgcGatewayValidationRuleDownloadConnector dgcGatewayValidationRuleDownloadConnector; + + @MockBean + DgcGatewayValueSetDownloadConnector dgcGatewayValueSetDownloadConnector; + + @MockBean + DgcGatewayCountryListDownloadConnector dgcGatewayCountryListDownloadConnector; + + @Test + void apiDocs() { + try (BufferedInputStream in = new BufferedInputStream(new URL("http://localhost:8080/openapi").openStream()); + FileOutputStream out = new FileOutputStream("target/openapi.json")) { + byte[] buffer = new byte[1024]; + int read; + while ((read = in.read(buffer, 0, buffer.length)) != -1) { + out.write(buffer, 0, read); + } + } catch (Exception e) { + log.error("Failed to download openapi specification.", e); + Assertions.fail(); + } + } +} diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml new file mode 100644 index 0000000..088591e --- /dev/null +++ b/src/test/resources/application.yml @@ -0,0 +1,76 @@ +logging: + level: + default: TRACE + org.hibernate.SQL: INFO + org.hibernate.type.descriptor.sql: INFO +server: + port: 8080 +spring: + application: + name: dgca-revocation-distribution-service + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://localhost:5432/postgres + username: postgres + password: postgres + jpa: + database-platform: org.hibernate.dialect.PostgreSQLDialect + hibernate: + ddl-auto: validate + liquibase: + change-log: classpath:db/changelog.xml + database-change-log-table: RD_CHANGELOG + database-change-log-lock-table: RD_CHANGELOG_LOCK + h2: + console: + enabled: false + path: /h2-console + task: + scheduling: + pool: + size: 5 +management: + server: + ssl: + enabled: false + port: 8081 + endpoint: + info: + enabled: true + health: + enabled: true + endpoints: + enabled-by-default: false + web: + exposure: + include: info,health + jmx: + exposure: + include: info,health + health: + probes: + enabled: true + +info: + name: ${spring.application.name} + profiles: ${spring.profiles.active} +springdoc: + api-docs: + path: /api/docs + enabled: true + swagger-ui: + path: /swagger + +dgc: + revocationListDownload: + timeInterval: 1800000 + lockLimit: 3600000 + downloadLimit: 60000 #max download time should be less than timeInterval + bloomFilter: + enabled: true + type: Bloom + version: 1.0 + probRate: 0.000000000001f + issuance: + dgci: + endpoint: ${DGC_ISSUANCE_DGCI_ENDPOINT} \ No newline at end of file