From b478bb6ceac91f4dff9cbf931652e35f1b869658 Mon Sep 17 00:00:00 2001 From: Srdjan Stevanetic Date: Fri, 1 Dec 2023 16:21:46 +0100 Subject: [PATCH 01/22] update organization fields (unmap from zoho or permanently remove) --- .../definitions/model/Organization.java | 41 ++------------- .../vocabulary/EntityFieldsTypes.java | 4 -- .../vocabulary/OrganizationSolrFields.java | 4 -- .../vocabulary/WebEntityFields.java | 2 - .../web/xml/model/XmlConstants.java | 2 - .../web/xml/model/XmlOrganizationImpl.java | 31 ++---------- .../repository/EntityRecordRepository.java | 22 +++++--- .../solr/model/SolrOrganization.java | 50 +++---------------- .../service/DereferenceServiceIT.java | 1 - .../entitymanagement/solr/SolrServiceIT.java | 29 ++++++----- .../testutils/ZohoRecordTestDeserializer.java | 37 +++++++++++--- .../web/EntityRegistrationIT.java | 7 +-- .../entitymanagement/web/UtilityTests.java | 44 ++++++++++++++++ .../ZohoOrganizationConverter.java | 47 +++-------------- .../zoho/utils/ZohoConstants.java | 2 - 15 files changed, 126 insertions(+), 197 deletions(-) create mode 100644 entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java index b24c461c4..651de4186 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java @@ -11,27 +11,24 @@ import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.FOAF_LOGO; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.FOAF_MBOX; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.FOAF_PHONE; -import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.GEOGRAPHIC_LEVEL; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.HAS_ADDRESS; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.HIDDEN_LABEL; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ID; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.IDENTIFIER; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.LANGUAGE; -import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ORGANIZATION_DOMAIN; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.PREF_LABEL; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.SAME_AS; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.TYPE; - -import com.fasterxml.jackson.annotation.JsonGetter; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import com.fasterxml.jackson.annotation.JsonSetter; -import eu.europeana.entitymanagement.vocabulary.EntityTypes; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonSetter; +import eu.europeana.entitymanagement.vocabulary.EntityTypes; /** This class defines base organization type of an entity. */ @JsonInclude(value = JsonInclude.Include.NON_EMPTY) @@ -47,8 +44,6 @@ DESCRIPTION, FOAF_LOGO, EUROPEANA_ROLE, - ORGANIZATION_DOMAIN, - GEOGRAPHIC_LEVEL, COUNTRY, LANGUAGE, FOAF_HOMEPAGE, @@ -68,8 +63,6 @@ public class Organization extends Entity { private List phone; private List mbox; private Map> europeanaRole; - private Map> organizationDomain; - private Map geographicLevel; private String country; private Address hasAddress; private List sameAs; @@ -89,10 +82,6 @@ public Organization(Organization copy) { if (copy.getMbox() != null) this.mbox = new ArrayList<>(copy.getMbox()); if (copy.getEuropeanaRole() != null) this.europeanaRole = new HashMap<>(copy.getEuropeanaRole()); - if (copy.getOrganizationDomain() != null) - this.organizationDomain = new HashMap<>(copy.getOrganizationDomain()); - if (copy.getGeographicLevel() != null) - this.geographicLevel = new HashMap<>(copy.getGeographicLevel()); this.country = copy.getCountry(); if (copy.getAddress() != null) this.hasAddress = new Address(copy.getAddress()); if (copy.sameAs != null) this.sameAs = (new ArrayList<>(copy.sameAs)); @@ -169,26 +158,6 @@ public void setAddress(Address hasAddress) { this.hasAddress = hasAddress; } - @JsonGetter(GEOGRAPHIC_LEVEL) - public Map getGeographicLevel() { - return geographicLevel; - } - - @JsonSetter(GEOGRAPHIC_LEVEL) - public void setGeographicLevel(Map geographicLevel) { - this.geographicLevel = geographicLevel; - } - - @JsonGetter(ORGANIZATION_DOMAIN) - public Map> getOrganizationDomain() { - return organizationDomain; - } - - @JsonSetter(ORGANIZATION_DOMAIN) - public void setOrganizationDomain(Map> organizationDomain) { - this.organizationDomain = organizationDomain; - } - @JsonGetter(FOAF_HOMEPAGE) public String getHomepage() { return homepage; diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java index 2abefdbab..9ca78fd8c 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java @@ -101,10 +101,6 @@ public enum EntityFieldsTypes { logo(EntityFieldsTypes.FIELD_TYPE_WEB_RESOURCE, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), europeanaRole( EntityFieldsTypes.FIELD_TYPE_KEYWORD, true, EntityFieldsTypes.FIELD_CARDINALITY_0_INFINITE), - organizationDomain( - EntityFieldsTypes.FIELD_TYPE_KEYWORD, true, EntityFieldsTypes.FIELD_CARDINALITY_0_INFINITE), - geographicLevel( - EntityFieldsTypes.FIELD_TYPE_KEYWORD, true, EntityFieldsTypes.FIELD_CARDINALITY_0_1), country(EntityFieldsTypes.FIELD_TYPE_TEXT, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), homepage(EntityFieldsTypes.FIELD_TYPE_URI, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), phone( diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java index 2587ce40e..f286ec96c 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java @@ -16,10 +16,6 @@ public interface OrganizationSolrFields extends EntitySolrFields { public static final String FOAF_MBOX = "foaf_mbox"; public static final String EUROPEANA_ROLE = "europeanaRole"; public static final String EUROPEANA_ROLE_ALL = EUROPEANA_ROLE + EXTENSION_ALL; - public static final String ORGANIZATION_DOMAIN = "organizationDomain"; - public static final String ORGANIZATION_DOMAIN_ALL = ORGANIZATION_DOMAIN + EXTENSION_ALL; - public static final String GEOGRAPHIC_LEVEL = "geographicLevel"; - public static final String GEOGRAPHIC_LEVEL_ALL = GEOGRAPHIC_LEVEL + EXTENSION_ALL; public static final String COUNTRY = "country"; public static final String VCARD_HAS_ADDRESS = "vcard_hasAddress.1"; public static final String VCARD_STREET_ADDRESS = "vcard_streetAddress.1"; diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/WebEntityFields.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/WebEntityFields.java index 1a4c0b8a4..7c290e3b1 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/WebEntityFields.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/WebEntityFields.java @@ -90,9 +90,7 @@ public interface WebEntityFields { public static final String DESCRIPTION = "description"; public static final String ACRONYM = "acronym"; public static final String COUNTRY = "country"; - public static final String ORGANIZATION_DOMAIN = "organizationDomain"; public static final String EUROPEANA_ROLE = "europeanaRole"; - public static final String GEOGRAPHIC_LEVEL = "geographicLevel"; public static final String FOAF_LOGO = "logo"; public static final String FOAF_HOMEPAGE = "homepage"; public static final String FOAF_PHONE = "phone"; diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlConstants.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlConstants.java index 239ecb76a..b2d15fafd 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlConstants.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlConstants.java @@ -82,8 +82,6 @@ public final class XmlConstants extends XmlFields { public static final String XML_DESCRIPTION = "description"; public static final String XML_LOGO = "logo"; public static final String XML_EUROPEANA_ROLE = "europeanaRole"; - public static final String XML_ORGANIZATION_DOMAIN = "organizationDomain"; - public static final String XML_GEOGRAPHIC_LEVEL = "geographicLevel"; public static final String XML_COUNTRY = "country"; public static final String XML_HOMEPAGE = "homepage"; public static final String XML_PHONE = "phone"; diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java index 837c52aba..07b654073 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java @@ -15,7 +15,6 @@ import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_COUNTRY; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_DESCRIPTION; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_EUROPEANA_ROLE; -import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_GEOGRAPHIC_LEVEL; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_HAS_ADDRESS; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_HOMEPAGE; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_IDENTIFIER; @@ -23,13 +22,8 @@ import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_LOGO; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_MBOX; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_ORGANIZATION; -import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_ORGANIZATION_DOMAIN; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_PHONE; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_SAME_AS; - -import eu.europeana.entitymanagement.definitions.exceptions.EntityModelCreationException; -import eu.europeana.entitymanagement.definitions.model.Organization; -import eu.europeana.entitymanagement.vocabulary.EntityTypes; import java.util.ArrayList; import java.util.List; import javax.xml.bind.annotation.XmlAccessType; @@ -38,6 +32,9 @@ import javax.xml.bind.annotation.XmlRootElement; import javax.xml.bind.annotation.XmlType; import org.apache.commons.collections.CollectionUtils; +import eu.europeana.entitymanagement.definitions.exceptions.EntityModelCreationException; +import eu.europeana.entitymanagement.definitions.model.Organization; +import eu.europeana.entitymanagement.vocabulary.EntityTypes; @XmlRootElement(namespace = NAMESPACE_EDM, name = XML_ORGANIZATION) @XmlAccessorType(XmlAccessType.FIELD) @@ -53,8 +50,6 @@ XML_DESCRIPTION, XML_LOGO, XML_EUROPEANA_ROLE, - XML_ORGANIZATION_DOMAIN, - XML_GEOGRAPHIC_LEVEL, XML_COUNTRY, XML_LANGUAGE, XML_HOMEPAGE, @@ -82,12 +77,6 @@ public class XmlOrganizationImpl extends XmlBaseEntityImpl { @XmlElement(namespace = NAMESPACE_EDM, name = XML_EUROPEANA_ROLE) private List europeanaRole = new ArrayList<>(); - @XmlElement(namespace = NAMESPACE_EDM, name = XML_ORGANIZATION_DOMAIN) - private List organizationDomain = new ArrayList<>(); - - @XmlElement(namespace = NAMESPACE_EDM, name = XML_GEOGRAPHIC_LEVEL) - private List geographicLevel = new ArrayList<>(); - @XmlElement(namespace = NAMESPACE_EDM, name = XML_COUNTRY) private String country; @@ -119,10 +108,6 @@ public XmlOrganizationImpl(Organization organization) { } this.europeanaRole = RdfXmlUtils.convertToXmlMultilingualString(organization.getEuropeanaRole()); - this.organizationDomain = - RdfXmlUtils.convertToXmlMultilingualString(organization.getOrganizationDomain()); - this.geographicLevel = - RdfXmlUtils.convertMapToXmlMultilingualString(organization.getGeographicLevel()); this.country = organization.getCountry(); if (organization.getHomepage() != null) { this.homepage = new LabelledResource(organization.getHomepage()); @@ -151,8 +136,6 @@ public Organization toEntityModel() throws EntityModelCreationException { entity.setDescription(RdfXmlUtils.toLanguageMap(getDescription())); entity.setLogo(XmlWebResourceWrapper.toWebResource(getLogo())); entity.setEuropeanaRole(RdfXmlUtils.toLanguageMapList(getEuropeanaRole())); - entity.setOrganizationDomain(RdfXmlUtils.toLanguageMapList(getOrganizationDomain())); - entity.setGeographicLevel(RdfXmlUtils.toLanguageMap(getGeographicLevel())); entity.setCountry(getCountry()); if (getHomepage() != null) { entity.setHomepage(getHomepage().getResource()); @@ -189,14 +172,6 @@ public List getEuropeanaRole() { return europeanaRole; } - public List getOrganizationDomain() { - return organizationDomain; - } - - public List getGeographicLevel() { - return geographicLevel; - } - public String getCountry() { return country; } diff --git a/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/EntityRecordRepository.java b/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/EntityRecordRepository.java index 91593db87..17b0f2991 100644 --- a/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/EntityRecordRepository.java +++ b/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/EntityRecordRepository.java @@ -11,7 +11,13 @@ import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_MODIFIED; import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_SAME_AS; import static eu.europeana.entitymanagement.mongo.utils.MorphiaUtils.MULTI_UPDATE_OPTS; - +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Repository; import com.mongodb.client.result.UpdateResult; import dev.morphia.query.FindOptions; import dev.morphia.query.Query; @@ -23,13 +29,6 @@ import eu.europeana.entitymanagement.definitions.model.EntityRecord; import eu.europeana.entitymanagement.definitions.web.EntityIdDisabledStatus; import eu.europeana.entitymanagement.mongo.utils.MorphiaUtils; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Repository; /** Repository for retrieving the EntityRecord objects. */ @Repository(AppConfigConstants.BEAN_ENTITY_RECORD_REPO) @@ -182,6 +181,13 @@ public List findWithFilters(int start, int count, Filter[] filters .toList(); } + public List findAll(int start, int count) { + return getDataStore() + .find(EntityRecord.class) + .iterator(new FindOptions().skip(start).sort(ascending(ENTITY_MODIFIED)).limit(count)) + .toList(); + } + public long deleteBulk(List entityIds) { return getDataStore() .find(EntityRecord.class) diff --git a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java index 72759b1e9..cca559f92 100644 --- a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java +++ b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java @@ -1,17 +1,17 @@ package eu.europeana.entitymanagement.solr.model; -import eu.europeana.entitymanagement.definitions.model.Address; -import eu.europeana.entitymanagement.definitions.model.Organization; -import eu.europeana.entitymanagement.solr.SolrUtils; -import eu.europeana.entitymanagement.utils.EntityUtils; -import eu.europeana.entitymanagement.vocabulary.EntitySolrFields; -import eu.europeana.entitymanagement.vocabulary.OrganizationSolrFields; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.commons.collections.MapUtils; import org.apache.solr.client.solrj.beans.Field; +import eu.europeana.entitymanagement.definitions.model.Address; +import eu.europeana.entitymanagement.definitions.model.Organization; +import eu.europeana.entitymanagement.solr.SolrUtils; +import eu.europeana.entitymanagement.utils.EntityUtils; +import eu.europeana.entitymanagement.vocabulary.EntitySolrFields; +import eu.europeana.entitymanagement.vocabulary.OrganizationSolrFields; public class SolrOrganization extends SolrEntity { @@ -39,12 +39,6 @@ public class SolrOrganization extends SolrEntity { @Field(OrganizationSolrFields.EUROPEANA_ROLE_ALL) private Map> europeanaRole; - @Field(OrganizationSolrFields.ORGANIZATION_DOMAIN_ALL) - private Map> organizationDomain; - - @Field(OrganizationSolrFields.GEOGRAPHIC_LEVEL_ALL) - private Map geographicLevel; - @Field(OrganizationSolrFields.COUNTRY) private String country; @@ -88,8 +82,6 @@ public SolrOrganization(Organization organization) { this.phone = organization.getPhone(); if (organization.getMbox() != null) this.mbox = new ArrayList<>(organization.getMbox()); setEuropeanaRole(organization.getEuropeanaRole()); - setOrganizationDomain(organization.getOrganizationDomain()); - setGeographicLevel(organization.getGeographicLevel()); this.country = organization.getCountry(); if (organization.getSameReferenceLinks() != null) { this.sameAs = new ArrayList<>(organization.getSameReferenceLinks()); @@ -126,28 +118,6 @@ private void setAcronym(Map> acronym) { } } - private void setGeographicLevel(Map geographicLevel) { - if (MapUtils.isNotEmpty(geographicLevel)) { - this.geographicLevel = - new HashMap<>( - SolrUtils.normalizeStringMapByAddingPrefix( - OrganizationSolrFields.GEOGRAPHIC_LEVEL - + EntitySolrFields.DYNAMIC_FIELD_SEPARATOR, - geographicLevel)); - } - } - - private void setOrganizationDomain(Map> organizationDomain) { - if (MapUtils.isNotEmpty(organizationDomain)) { - this.organizationDomain = - new HashMap<>( - SolrUtils.normalizeStringListMapByAddingPrefix( - OrganizationSolrFields.ORGANIZATION_DOMAIN - + EntitySolrFields.DYNAMIC_FIELD_SEPARATOR, - organizationDomain)); - } - } - private void setEuropeanaRole(Map> europeanaRole) { if (MapUtils.isNotEmpty(europeanaRole)) { this.europeanaRole = @@ -186,14 +156,6 @@ public Map> getEuropeanaRole() { return europeanaRole; } - public Map> getOrganizationDomain() { - return organizationDomain; - } - - public Map getGeographicLevel() { - return geographicLevel; - } - public String getCountry() { return country; } diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java index 442156780..34ab3cec3 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java @@ -79,7 +79,6 @@ public void zohoOrganizationDereferenceTest() throws Exception { assertNull(org.getAltLabel()); assertEquals(1, org.getAcronym().size()); assertEquals(1, org.getEuropeanaRole().size()); - assertEquals(1, org.getGeographicLevel().size()); assertEquals(1, org.getAcronym().size()); assertEquals("FR", org.getCountry()); Assertions.assertNotNull(org.getHomepage()); diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/solr/SolrServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/solr/SolrServiceIT.java index 6c5f022e5..d4fb9d432 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/solr/SolrServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/solr/SolrServiceIT.java @@ -5,7 +5,18 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasSize; import static org.junit.jupiter.api.Assertions.assertEquals; - +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.context.SpringBootTest; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -31,18 +42,6 @@ import eu.europeana.entitymanagement.testutils.IntegrationTestUtils; import eu.europeana.entitymanagement.utils.EntityRecordUtils; import eu.europeana.entitymanagement.vocabulary.EntityTypes; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; -import org.hamcrest.Matchers; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.test.context.SpringBootTest; @SpringBootTest public class SolrServiceIT extends AbstractIntegrationTest { @@ -52,7 +51,7 @@ public class SolrServiceIT extends AbstractIntegrationTest { @Qualifier(AppConfig.BEAN_EM_SOLR_SERVICE) @Autowired private SolrService emSolrService; - + @Qualifier(AppConfigConstants.BEAN_JSON_MAPPER) @Autowired private ObjectMapper objectMapper; @@ -84,7 +83,6 @@ void verifyPayload(SolrEntity entity) { // for organizations verify country if (EntityTypes.Organization.getEntityType().equals(entity.getType())) { assertThat(payload, Matchers.containsString("\"country\"")); - assertThat(payload, Matchers.containsString("\"organizationDomain\"")); } } @@ -247,4 +245,5 @@ private List> getSolrEntities(String searchQuery) throws Exception return solrEntities; } + } diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/ZohoRecordTestDeserializer.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/ZohoRecordTestDeserializer.java index 9be3d5395..ff9018298 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/ZohoRecordTestDeserializer.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/ZohoRecordTestDeserializer.java @@ -1,16 +1,41 @@ package eu.europeana.entitymanagement.testutils; -import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.*; - +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.ACCOUNT_NAME_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.ACRONYM_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.ALTERNATIVE_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.CITY_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.COUNTRY_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.HIDDEN_LABEL1_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.HIDDEN_LABEL2_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.HIDDEN_LABEL3_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.HIDDEN_LABEL4_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.HIDDEN_LABEL_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.ID_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.INDUSTRY_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.LANGUAGE_CODE_LENGTH; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.LANG_ACRONYM_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.LANG_ALTERNATIVE_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.LANG_ORGANIZATION_NAME_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.LATITUDE_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.LOGO_LINK_TO_WIKIMEDIACOMMONS_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.LONGITUDE_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.OFFICIAL_LANGUAGE_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.ORGANIZATION_COUNTRY_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.ORGANIZATION_ROLE_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.PO_BOX_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.SAME_AS_CODE_LENGTH; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.SAME_AS_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.WEBSITE_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.ZIP_CODE_FIELD; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import com.zoho.crm.api.record.Record; import com.zoho.crm.api.util.Choice; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; /** Helper class to deserialize JSON into Zoho {@link Record} to make testing easier */ public class ZohoRecordTestDeserializer extends StdDeserializer { @@ -32,8 +57,6 @@ public class ZohoRecordTestDeserializer extends StdDeserializer { ACRONYM_FIELD, LOGO_LINK_TO_WIKIMEDIACOMMONS_FIELD, WEBSITE_FIELD, - DOMAIN_FIELD, - GEOGRAPHIC_LEVEL_FIELD, ORGANIZATION_COUNTRY_FIELD, CITY_FIELD, COUNTRY_FIELD, diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java index 9f4e95511..30aafb182 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java @@ -6,7 +6,6 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.matchesRegex; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; @@ -280,8 +279,7 @@ public void registerZohoOrganizationBergerShouldBeSuccessful() throws Exception // check if indexing is successfull by searching the organization in solr SolrOrganization org = emSolrService.searchById(SolrOrganization.class, WebEntityFields.BASE_DATA_EUROPEANA_URI + "organization/1"); - assertNotNull(org.getHasGeo()); - assertFalse(org.getHasGeo().startsWith("geo:")); + assertNotNull(org.getHasAddress()); } @Test @@ -328,10 +326,9 @@ void registerZohoOrganizationBnfWithNewFieldsShouldBeSuccessful() throws Excepti .contentType(MediaType.APPLICATION_JSON_VALUE)); response .andExpect(status().isAccepted()) - .andExpect(jsonPath("$.hasAddress.hasGeo").isNotEmpty()) + .andExpect(jsonPath("$.hasAddress.postalCode").isNotEmpty()) .andExpect(jsonPath("$.language", everyItem(matchesRegex("[a-z]+")))) .andExpect(jsonPath("$.hiddenLabel", hasSize(3))) - .andExpect(jsonPath("$.organizationDomain[*]", hasSize(1))) .andExpect(jsonPath("$.id", any(String.class))); } diff --git a/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java b/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java new file mode 100644 index 000000000..6a56a16ff --- /dev/null +++ b/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java @@ -0,0 +1,44 @@ +package eu.europeana.entitymanagement.web; + +import static eu.europeana.entitymanagement.solr.SolrUtils.createSolrEntity; +import java.util.List; +import org.junit.jupiter.api.Disabled; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.context.SpringBootTest; +import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.definitions.model.EntityRecord; +import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException; +import eu.europeana.entitymanagement.mongo.repository.EntityRecordRepository; +import eu.europeana.entitymanagement.solr.exception.SolrServiceException; +import eu.europeana.entitymanagement.solr.service.SolrService; + +@SpringBootTest +@Disabled("Excluded from automated runs") +public class UtilityTests { + + @Qualifier(AppConfig.BEAN_EM_SOLR_SERVICE) + @Autowired + private SolrService emSolrService; + + @Autowired + private EntityRecordRepository entityRecordRepository; + + /** + * This test can be used to reindex the local/other solr from mongo, when the schema fields change. + * @throws EntityUpdateException + */ +// @Test + public void reindexSolrFromMongo() throws EntityUpdateException { + List allRecords = entityRecordRepository.findAll(0, 100); + for(EntityRecord er : allRecords) { + try { + emSolrService.storeEntity(createSolrEntity(er)); + } catch (SolrServiceException e) { + throw new EntityUpdateException( + "Cannot create solr record for entity with id: " + er, e); + } + } + } + +} diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java index 59ddd0f29..2b7752d79 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java @@ -1,24 +1,22 @@ package eu.europeana.entitymanagement.zoho.organization; import static eu.europeana.entitymanagement.zoho.utils.ZohoUtils.toIsoLanguage; - -import com.zoho.crm.api.record.Record; -import com.zoho.crm.api.users.User; -import eu.europeana.entitymanagement.definitions.model.Address; -import eu.europeana.entitymanagement.definitions.model.Organization; -import eu.europeana.entitymanagement.definitions.model.WebResource; -import eu.europeana.entitymanagement.utils.EntityUtils; -import eu.europeana.entitymanagement.zoho.utils.ZohoConstants; -import eu.europeana.entitymanagement.zoho.utils.ZohoUtils; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import com.zoho.crm.api.record.Record; +import com.zoho.crm.api.users.User; +import eu.europeana.entitymanagement.definitions.model.Address; +import eu.europeana.entitymanagement.definitions.model.Organization; +import eu.europeana.entitymanagement.definitions.model.WebResource; +import eu.europeana.entitymanagement.utils.EntityUtils; +import eu.europeana.entitymanagement.zoho.utils.ZohoConstants; +import eu.europeana.entitymanagement.zoho.utils.ZohoUtils; public class ZohoOrganizationConverter { @@ -58,17 +56,6 @@ public static Organization convertToOrganizationEntity(Record zohoRecord, String ZohoUtils.createLanguageMapOfStringList( Locale.ENGLISH.getLanguage(), organizationRoleStringList)); } - org.setOrganizationDomain( - ZohoUtils.createMapWithLists( - Locale.ENGLISH.getLanguage(), - ZohoUtils.stringFieldSupplier(zohoRecord.getKeyValue(ZohoConstants.DOMAIN_FIELD)))); - - List geographicLevel = - ZohoUtils.stringListSupplier(zohoRecord.getKeyValue(ZohoConstants.GEOGRAPHIC_LEVEL_FIELD)); - if (!geographicLevel.isEmpty()) { - org.setGeographicLevel( - ZohoUtils.createMap(Locale.ENGLISH.getLanguage(), geographicLevel.get(0))); - } String organizationCountry = toEdmCountry(getStringFieldValue(zohoRecord, ZohoConstants.ORGANIZATION_COUNTRY_FIELD)); @@ -84,16 +71,6 @@ public static Organization convertToOrganizationEntity(Record zohoRecord, String ZohoUtils.stringFieldSupplier(zohoRecord.getKeyValue(ZohoConstants.COUNTRY_FIELD))); address.setVcardPostalCode( ZohoUtils.stringFieldSupplier(zohoRecord.getKeyValue(ZohoConstants.ZIP_CODE_FIELD))); - address.setVcardPostOfficeBox( - ZohoUtils.stringFieldSupplier(zohoRecord.getKeyValue(ZohoConstants.PO_BOX_FIELD))); - - String lat = - ZohoUtils.stringFieldSupplier(zohoRecord.getKeyValue(ZohoConstants.LATITUDE_FIELD)); - String lon = - ZohoUtils.stringFieldSupplier(zohoRecord.getKeyValue(ZohoConstants.LONGITUDE_FIELD)); - if (lat != null && lon != null) { - address.setVcardHasGeo(EntityUtils.toGeoUri(lat, lon)); - } // only set address if it contains metadata properties. if (address.hasMetadataProperties()) { @@ -125,14 +102,6 @@ public static Organization convertToOrganizationEntity(Record zohoRecord, String org.setHiddenLabel(hiddenLabels); } - List industry = - ZohoUtils.stringListSupplier(zohoRecord.getKeyValue(ZohoConstants.INDUSTRY_FIELD)); - if (!industry.isEmpty()) { - Map> orgDomain = new HashMap>(); - orgDomain.put("en", industry); - org.setOrganizationDomain(orgDomain); - } - return org; } diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/utils/ZohoConstants.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/utils/ZohoConstants.java index f39b9b369..2904846d5 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/utils/ZohoConstants.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/utils/ZohoConstants.java @@ -30,10 +30,8 @@ public final class ZohoConstants { public static final String ACRONYM_FIELD = "Acronym"; public static final String LOGO_LINK_TO_WIKIMEDIACOMMONS_FIELD = "Logo_link_to_WikimediaCommons"; public static final String WEBSITE_FIELD = "Website"; - public static final String DOMAIN_FIELD = "Domain2"; public static final String SECTOR_FIELD = "Sector"; public static final String SCOPE_FIELD = "Scope"; - public static final String GEOGRAPHIC_LEVEL_FIELD = "Geographic_Level"; public static final String ORGANIZATION_COUNTRY_FIELD = "Country1"; public static final String SAME_AS_FIELD = "SameAs"; public static final String STREET_FIELD = "Street"; From 8d0926dcf1698824ca645a50b70be01bf169f0ad Mon Sep 17 00:00:00 2001 From: SrdjanStevanetic Date: Fri, 1 Dec 2023 16:25:32 +0100 Subject: [PATCH 02/22] Update build_test_analyse.yml --- .github/workflows/build_test_analyse.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build_test_analyse.yml b/.github/workflows/build_test_analyse.yml index 3ed897ee8..06e6148e9 100644 --- a/.github/workflows/build_test_analyse.yml +++ b/.github/workflows/build_test_analyse.yml @@ -6,22 +6,19 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: # Shallow clones should be disabled for a better relevancy of analysis fetch-depth: 0 - - name: Set up JDK 11 - uses: actions/setup-java@v1 + - name: Set up JDK 17 + uses: actions/setup-java@v3 with: - java-version: 11 - - name: Cache Maven packages - uses: actions/cache@v1 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 + distribution: 'temurin' #should use the same as in the docker file + java-version: 17 + cache: 'maven' + cache-dependency-path: 'sub-project/pom.xml' # optional - name: Cache SonarCloud packages - uses: actions/cache@v1 + uses: actions/cache@v3 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar @@ -32,4 +29,4 @@ jobs: # Needed to get some information about the pull request, if any GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # SonarCloud access token should be generated from https://sonarcloud.io/account/security/ - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} From 0577e7cc0404845a63ad63c2e28645b30d314f60 Mon Sep 17 00:00:00 2001 From: Srdjan Stevanetic Date: Tue, 5 Dec 2023 17:32:24 +0100 Subject: [PATCH 03/22] new zoho country field (dereferenced using the europeana uri) --- development_tips.txt | 3 + .../common/vocabulary/AppConfigConstants.java | 2 +- .../definitions/model/Country.java | 59 ++++++++ .../definitions/model/Organization.java | 8 +- .../vocabulary/OrganizationSolrFields.java | 4 +- .../web/xml/model/XmlOrganizationImpl.java | 21 ++- .../solr/model/SolrOrganization.java | 33 ++++- .../AbstractIntegrationTest.java | 6 +- .../service/DereferenceServiceIT.java | 8 +- .../testutils/IntegrationTestUtils.java | 36 ++--- .../testutils/TestConfig.java | 19 +-- .../web/BaseWebControllerTest.java | 3 +- .../web/EntityRegistrationIT.java | 11 +- .../resources/content/organization.json | 5 +- .../content/place_register_sweden.json | 4 + .../resources/metis-deref/place_sweden.xml | 20 +++ .../resources/solr-docker/conf/schema.xml | 5 +- .../resources/zoho_country_mapping_test.json | 3 + .../web/service/BaseZohoAccess.java | 20 +-- .../web/service/EntityRecordService.java | 24 +++- .../web/service/ZohoSyncService.java | 15 +- entity-management-zoho/pom.xml | 6 + .../wikidata/WikidataDereferenceService.java | 7 +- ...figuration.java => ZohoConfiguration.java} | 18 ++- .../organization/ZohoDereferenceService.java | 16 ++- .../ZohoOrganizationConverter.java | 135 ++++++++++++------ .../main/resources/zoho_country_mapping.json | 3 + .../resources/zoho_import.properties.template | 3 +- 28 files changed, 371 insertions(+), 126 deletions(-) create mode 100644 entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Country.java create mode 100644 entity-management-tests/src/integration-test/resources/content/place_register_sweden.json create mode 100644 entity-management-tests/src/integration-test/resources/metis-deref/place_sweden.xml create mode 100644 entity-management-tests/src/integration-test/resources/zoho_country_mapping_test.json rename entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/{ZohoAccessConfiguration.java => ZohoConfiguration.java} (86%) create mode 100644 entity-management-zoho/src/main/resources/zoho_country_mapping.json diff --git a/development_tips.txt b/development_tips.txt index 4e2d4ea31..9b9bc00ef 100644 --- a/development_tips.txt +++ b/development_tips.txt @@ -15,3 +15,6 @@ public ZohoDereferenceService(ZohoAccessConfiguration zohoAccessConfiguration, } The printed response can then be saved to a file in test/resources. Also make sure to adjust the fields to be either string or list type (no map). + +2. Whenever introducing new field types for any of the model definition classes, e.g. Organization, make sure to properly hanlde the merging of the entities in the method combineEntities. + diff --git a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/vocabulary/AppConfigConstants.java b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/vocabulary/AppConfigConstants.java index 9d4de83e9..75c45177c 100644 --- a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/vocabulary/AppConfigConstants.java +++ b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/vocabulary/AppConfigConstants.java @@ -55,7 +55,7 @@ public class AppConfigConstants { public static final String BEAN_SOLR_ENTITY_SUGGESTER_FILTER = "solrEntityFilter"; - public static final String BEAN_ZOHO_ACCESS_CONFIGURATION = "zohoAccessConfiguration"; + public static final String BEAN_ZOHO_CONFIGURATION = "zohoConfiguration"; public static final String BEAN_WIKIDATA_ACCESS_SERVICE = "wikidataAccessService"; public static final String BEAN_WIKIDATA_ACCESS_DAO = "wikidataAccessDao"; } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Country.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Country.java new file mode 100644 index 000000000..29ec37cab --- /dev/null +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Country.java @@ -0,0 +1,59 @@ +package eu.europeana.entitymanagement.definitions.model; + +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.CONTEXT; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ID; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.PREF_LABEL; +import java.util.HashMap; +import java.util.Map; +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonSetter; +import dev.morphia.annotations.Embedded; +import eu.europeana.entitymanagement.vocabulary.WebEntityFields; + +@Embedded +@JsonInclude(value = JsonInclude.Include.NON_EMPTY) +@JsonPropertyOrder({ + CONTEXT, + ID, + PREF_LABEL +}) +public class Country { + + public Country() { + super(); + } + + public Country(Country copy) { + super(); + this.id = copy.getId(); + if(copy.getPrefLabel()!=null) { + this.prefLabel=new HashMap<>(copy.getPrefLabel()); + } + } + + private String id; + private Map prefLabel; + + @JsonSetter(ID) + public void setId(String id) { + this.id = id; + } + + @JsonGetter(ID) + public String getId() { + return id; + } + + @JsonGetter(WebEntityFields.PREF_LABEL) + public Map getPrefLabel() { + return prefLabel; + } + + @JsonSetter(WebEntityFields.PREF_LABEL) + public void setPrefLabel(Map prefLabel) { + this.prefLabel = prefLabel; + } + +} diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java index 651de4186..2c4a17268 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java @@ -63,7 +63,7 @@ public class Organization extends Entity { private List phone; private List mbox; private Map> europeanaRole; - private String country; + private Country country; private Address hasAddress; private List sameAs; private List language; @@ -82,7 +82,7 @@ public Organization(Organization copy) { if (copy.getMbox() != null) this.mbox = new ArrayList<>(copy.getMbox()); if (copy.getEuropeanaRole() != null) this.europeanaRole = new HashMap<>(copy.getEuropeanaRole()); - this.country = copy.getCountry(); + if (copy.getCountry() != null) this.country = new Country(copy.getCountry()); if (copy.getAddress() != null) this.hasAddress = new Address(copy.getAddress()); if (copy.sameAs != null) this.sameAs = (new ArrayList<>(copy.sameAs)); if (copy.language != null) this.language = (new ArrayList<>(copy.language)); @@ -139,12 +139,12 @@ public void setMbox(List mbox) { } @JsonGetter(COUNTRY) - public String getCountry() { + public Country getCountry() { return country; } @JsonSetter(COUNTRY) - public void setCountry(String country) { + public void setCountry(Country country) { this.country = country; } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java index f286ec96c..cb58c3b46 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java @@ -16,7 +16,9 @@ public interface OrganizationSolrFields extends EntitySolrFields { public static final String FOAF_MBOX = "foaf_mbox"; public static final String EUROPEANA_ROLE = "europeanaRole"; public static final String EUROPEANA_ROLE_ALL = EUROPEANA_ROLE + EXTENSION_ALL; - public static final String COUNTRY = "country"; + public static final String COUNTRY_ID = "countryId"; + public static final String COUNTRY_PREF_LABEL_ALL = "countryPrefLabel.*"; + public static final String COUNTRY_PREF_LABEL = "countryPrefLabel"; public static final String VCARD_HAS_ADDRESS = "vcard_hasAddress.1"; public static final String VCARD_STREET_ADDRESS = "vcard_streetAddress.1"; public static final String VCARD_LOCALITY = "vcard_locality.1"; diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java index 07b654073..b1950cc83 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java @@ -25,7 +25,9 @@ import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_PHONE; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_SAME_AS; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; @@ -33,6 +35,7 @@ import javax.xml.bind.annotation.XmlType; import org.apache.commons.collections.CollectionUtils; import eu.europeana.entitymanagement.definitions.exceptions.EntityModelCreationException; +import eu.europeana.entitymanagement.definitions.model.Country; import eu.europeana.entitymanagement.definitions.model.Organization; import eu.europeana.entitymanagement.vocabulary.EntityTypes; @@ -79,7 +82,7 @@ public class XmlOrganizationImpl extends XmlBaseEntityImpl { @XmlElement(namespace = NAMESPACE_EDM, name = XML_COUNTRY) private String country; - + @XmlElement(namespace = NAMESPACE_FOAF, name = XML_HOMEPAGE) private LabelledResource homepage; @@ -108,7 +111,11 @@ public XmlOrganizationImpl(Organization organization) { } this.europeanaRole = RdfXmlUtils.convertToXmlMultilingualString(organization.getEuropeanaRole()); - this.country = organization.getCountry(); + + if(organization.getCountry()!=null && organization.getCountry().getPrefLabel()!=null) { + this.country = organization.getCountry().getPrefLabel().get("en"); + } + if (organization.getHomepage() != null) { this.homepage = new LabelledResource(organization.getHomepage()); } @@ -136,7 +143,15 @@ public Organization toEntityModel() throws EntityModelCreationException { entity.setDescription(RdfXmlUtils.toLanguageMap(getDescription())); entity.setLogo(XmlWebResourceWrapper.toWebResource(getLogo())); entity.setEuropeanaRole(RdfXmlUtils.toLanguageMapList(getEuropeanaRole())); - entity.setCountry(getCountry()); + + if(country!=null) { + Country orgCountry = new Country(); + Map countryPrefLabel = new HashMap<>(); + countryPrefLabel.put("en", country); + orgCountry.setPrefLabel(countryPrefLabel); + entity.setCountry(orgCountry); + } + if (getHomepage() != null) { entity.setHomepage(getHomepage().getResource()); } diff --git a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java index cca559f92..7792360b4 100644 --- a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java +++ b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java @@ -39,8 +39,11 @@ public class SolrOrganization extends SolrEntity { @Field(OrganizationSolrFields.EUROPEANA_ROLE_ALL) private Map> europeanaRole; - @Field(OrganizationSolrFields.COUNTRY) - private String country; + @Field(OrganizationSolrFields.COUNTRY_ID) + private String countryId; + + @Field(OrganizationSolrFields.COUNTRY_PREF_LABEL_ALL) + private Map countryPrefLabel; @Field(OrganizationSolrFields.VCARD_HAS_ADDRESS) private String hasAddress; @@ -82,7 +85,9 @@ public SolrOrganization(Organization organization) { this.phone = organization.getPhone(); if (organization.getMbox() != null) this.mbox = new ArrayList<>(organization.getMbox()); setEuropeanaRole(organization.getEuropeanaRole()); - this.country = organization.getCountry(); + this.countryId = organization.getCountry().getId(); + setCountryPrefLabel(organization.getCountry().getPrefLabel()); + if (organization.getSameReferenceLinks() != null) { this.sameAs = new ArrayList<>(organization.getSameReferenceLinks()); } @@ -127,6 +132,16 @@ private void setEuropeanaRole(Map> europeanaRole) { europeanaRole)); } } + + private void setCountryPrefLabel(Map prefLabel) { + if (MapUtils.isNotEmpty(prefLabel)) { + this.countryPrefLabel = + new HashMap<>( + SolrUtils.normalizeStringMapByAddingPrefix( + OrganizationSolrFields.COUNTRY_PREF_LABEL + EntitySolrFields.DYNAMIC_FIELD_SEPARATOR, + prefLabel)); + } + } public Map getDescription() { return description; @@ -156,10 +171,6 @@ public Map> getEuropeanaRole() { return europeanaRole; } - public String getCountry() { - return country; - } - public String getHasAddress() { return hasAddress; } @@ -196,4 +207,12 @@ public String getHasGeo() { protected void setSameReferenceLinks(ArrayList uris) { this.sameAs = uris; } + + public String getCountryId() { + return countryId; + } + + public Map getCountryPrefLabel() { + return countryPrefLabel; + } } diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java index 1ff939800..7017e1486 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java @@ -18,7 +18,8 @@ import eu.europeana.entitymanagement.web.service.ConceptSchemeService; import eu.europeana.entitymanagement.web.service.EntityRecordService; import eu.europeana.entitymanagement.web.xml.model.XmlBaseEntityImpl; -import eu.europeana.entitymanagement.zoho.organization.ZohoAccessConfiguration; +import eu.europeana.entitymanagement.zoho.organization.ZohoConfiguration; +import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -64,7 +65,8 @@ public abstract class AbstractIntegrationTest { @Autowired protected EntityRecordRepository entityRecordRepository; @Autowired protected ConceptSchemeService emConceptSchemeService; @Autowired protected EntityManagementConfiguration emConfig; - @Autowired protected ZohoAccessConfiguration zohoAccessConfiguration; + @Autowired protected ZohoConfiguration zohoConfiguration; + @Autowired protected ZohoOrganizationConverter zohoOrgConverter; static { MONGO_CONTAINER = diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java index 34ab3cec3..a56338230 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java @@ -20,7 +20,6 @@ import eu.europeana.entitymanagement.testutils.IntegrationTestUtils; import eu.europeana.entitymanagement.testutils.TestConfig; import eu.europeana.entitymanagement.web.service.DereferenceServiceLocator; -import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter; import eu.europeana.entitymanagement.zoho.utils.ZohoConstants; import eu.europeana.entitymanagement.zoho.utils.ZohoException; @@ -30,7 +29,7 @@ // enable tests only on local machine // @Disabled @SpringBootTest -class DereferenceServiceIT extends AbstractIntegrationTest { +public class DereferenceServiceIT extends AbstractIntegrationTest { @Autowired private DereferenceServiceLocator dereferenceServiceLocator; @@ -90,7 +89,7 @@ public void zohoOrganizationDereferenceTest() throws Exception { Assertions.assertTrue(org.getHiddenLabel().contains("French National Library")); } - // @Test +// @Test public void zohoOrganizationDereferenceGFMTest() throws Exception { String organizationId = IntegrationTestUtils.ORGANIZATION_GFM_URI_ZOHO; Dereferencer dereferencer = @@ -98,7 +97,6 @@ public void zohoOrganizationDereferenceGFMTest() throws Exception { Optional orgOptional = dereferencer.dereferenceEntityById(organizationId); Assertions.assertTrue(orgOptional.isPresent()); - Organization org = (Organization) orgOptional.get(); assertEquals(2, org.getPrefLabel().size()); assertNotNull(org.getSameReferenceLinks()); @@ -133,7 +131,7 @@ public void zohoOrganizationDereferenceLabelsTest() throws Exception { // choice = new Choice("EN"); // record.addKeyValue(ZohoConstants.LANG_ALTERNATIVE_FIELD + "_4", choice); - Organization org = ZohoOrganizationConverter.convertToOrganizationEntity(record, zohoAccessConfiguration.getZohoBaseUrl()); + Organization org = zohoOrgConverter.convertToOrganizationEntity(record); Assertions.assertEquals(2, org.getPrefLabel().size()); Assertions.assertEquals(1, org.getAltLabel().size()); diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java index 4737b8597..aa984a6ff 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java @@ -1,16 +1,16 @@ package eu.europeana.entitymanagement.testutils; -import com.fasterxml.jackson.core.Version; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.module.SimpleModule; -import com.zoho.crm.api.record.Record; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.Objects; import java.util.Optional; import org.apache.commons.io.IOUtils; +import com.fasterxml.jackson.core.Version; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.zoho.crm.api.record.Record; public class IntegrationTestUtils { @@ -73,6 +73,7 @@ public class IntegrationTestUtils { public static final String ORGANIZATION_REGISTER_PCCE_ZOHO_JSON = "/content/organization_register_zoho_pcce.json"; public static final String PLACE_REGISTER_PARIS_JSON = "/content/place_register_paris.json"; + public static final String PLACE_REGISTER_SWEDEN_JSON = "/content/place_register_sweden.json"; public static final String PLACE_REGISTER_HAGENBACH_JSON = "/content/place_register_hagenbach.json"; @@ -105,6 +106,7 @@ public class IntegrationTestUtils { public static final String ORGANIZATION_BNF_XML = "/metis-deref/organization_bnf.xml"; public static final String PLACE_PARIS_XML = "/metis-deref/place_paris.xml"; public static final String PLACE_HAGENBACH_XML = "/metis-deref/place_hagenbach_redirect.xml"; + public static final String PLACE_SWEDEN_XML = "/metis-deref/place_sweden.xml"; public static final String TIMESPAN_1ST_CENTURY_XML = "/metis-deref/timespan_1st_century.xml"; public static final String TIMESPAN_1_CENTURY_SEARCH_AND_RECORD_JSON = @@ -156,6 +158,7 @@ public class IntegrationTestUtils { "http://www.wikidata.org/entity/Q41264"; public static final String PLACE_PARIS_URI = "https://sws.geonames.org/2988507/"; public static final String PLACE_HAGENBACH_URI = "http://www.wikidata.org/entity/Q32050320"; + public static final String PLACE_SWEDEN_URI = "http://www.wikidata.org/entity/Q34"; public static final String PLACE_HAGENBACH_UPDATED_URI = "http://www.wikidata.org/entity/Q541669"; public static final String TIMESPAN_1ST_CENTURY_URI = "http://www.wikidata.org/entity/Q8106"; @@ -250,17 +253,18 @@ public class IntegrationTestUtils { /** Maps Metis dereferenciation URIs to mocked XML responses */ public static final Map METIS_RESPONSE_MAP = - Map.of( - AGENT_DA_VINCI_URI, AGENT_DA_VINCI_XML, - AGENT_STALIN_URI, AGENT_STALIN_XML, - PLACE_PARIS_URI, PLACE_PARIS_XML, - PLACE_HAGENBACH_URI, PLACE_HAGENBACH_XML, - TIMESPAN_1ST_CENTURY_URI, TIMESPAN_1ST_CENTURY_XML, - CONCEPT_BATHTUB_URI, CONCEPT_BATHTUB_XML, - AGENT_JAN_VERMEER_VIAF_URI, AGENT_JAN_VERMEER_XML_VIAF, - AGENT_JAN_VERMEER_WIKIDATA_URI, AGENT_JAN_VERMEER_XML_WIKIDATA, - AGENT_SCHEGK_URI, AGENT_SCHEGK_XML, - AGENT_BIRCH_URI, AGENT_BIRCH_XML); + Map.ofEntries( + Map.entry(AGENT_DA_VINCI_URI, AGENT_DA_VINCI_XML), + Map.entry(AGENT_STALIN_URI, AGENT_STALIN_XML), + Map.entry(PLACE_PARIS_URI, PLACE_PARIS_XML), + Map.entry(PLACE_HAGENBACH_URI, PLACE_HAGENBACH_XML), + Map.entry(PLACE_SWEDEN_URI, PLACE_SWEDEN_XML), + Map.entry(TIMESPAN_1ST_CENTURY_URI, TIMESPAN_1ST_CENTURY_XML), + Map.entry(CONCEPT_BATHTUB_URI, CONCEPT_BATHTUB_XML), + Map.entry(AGENT_JAN_VERMEER_VIAF_URI, AGENT_JAN_VERMEER_XML_VIAF), + Map.entry(AGENT_JAN_VERMEER_WIKIDATA_URI, AGENT_JAN_VERMEER_XML_WIKIDATA), + Map.entry(AGENT_SCHEGK_URI, AGENT_SCHEGK_XML), + Map.entry(AGENT_BIRCH_URI, AGENT_BIRCH_XML)); public static String loadFile(String resourcePath) throws IOException { return IOUtils.toString( diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java index 224db3fad..cdc11eb55 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java @@ -1,8 +1,5 @@ package eu.europeana.entitymanagement.testutils; -import com.zoho.crm.api.record.Record; -import eu.europeana.entitymanagement.zoho.ZohoAccessClient; -import eu.europeana.entitymanagement.zoho.organization.ZohoAccessConfiguration; import java.util.Optional; import org.mockito.ArgumentMatchers; import org.mockito.Mockito; @@ -10,23 +7,29 @@ import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; +import com.zoho.crm.api.record.Record; +import eu.europeana.entitymanagement.zoho.ZohoAccessClient; +import eu.europeana.entitymanagement.zoho.organization.ZohoConfiguration; @TestConfiguration public class TestConfig { public static final String MOCK_ZOHO_BASE_URL = "https://crm.zoho.com/crm/org51823723/tab/Accounts/"; + public static final String MOCK_ZOHO_COUNTRY_MAPPING_FILE = "/zoho_country_mapping_test.json"; + /** * Since requests to Zoho are done via its SDK, and require authentication first, we mock out the * entire flow with Mockito. */ @Primary @Bean - public ZohoAccessConfiguration configureZoho() throws Exception { - ZohoAccessConfiguration zohoAccessConfiguration = Mockito.mock(ZohoAccessConfiguration.class); + public ZohoConfiguration configureZoho() throws Exception { + ZohoConfiguration zohoConfiguration = Mockito.mock(ZohoConfiguration.class); ZohoAccessClient zohoClient = Mockito.mock(ZohoAccessClient.class); - Mockito.when(zohoAccessConfiguration.getZohoAccessClient()).thenReturn(zohoClient); - Mockito.when(zohoAccessConfiguration.getZohoBaseUrl()). thenReturn(MOCK_ZOHO_BASE_URL); + Mockito.when(zohoConfiguration.getZohoAccessClient()).thenReturn(zohoClient); + Mockito.when(zohoConfiguration.getZohoBaseUrl()).thenReturn(MOCK_ZOHO_BASE_URL); + Mockito.when(zohoConfiguration.getZohoCountryMappingFile()).thenReturn(MOCK_ZOHO_COUNTRY_MAPPING_FILE); // find matching JSON file based on zohoId argument, then create a Record object for it Mockito.doAnswer( @@ -38,6 +41,6 @@ public ZohoAccessConfiguration configureZoho() throws Exception { .when(zohoClient) .getZohoRecordOrganizationById(ArgumentMatchers.any(String.class)); - return zohoAccessConfiguration; + return zohoConfiguration; } } diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java index 815827a3f..8c9950fc9 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java @@ -34,7 +34,6 @@ import eu.europeana.entitymanagement.testutils.IntegrationTestUtils; import eu.europeana.entitymanagement.testutils.TestConfig; import eu.europeana.entitymanagement.web.xml.model.XmlBaseEntityImpl; -import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter; @Import(TestConfig.class) abstract class BaseWebControllerTest extends AbstractIntegrationTest { @@ -172,7 +171,7 @@ protected EntityRecord createOrganization(String europeanaProxyEntityStr, Record Entity europeanaProxyEntity = objectMapper.readValue(europeanaProxyEntityStr, Entity.class); DataSource dataSource = datasources.verifyDataSource(europeanaProxyEntity.getEntityId(), false); Organization zohoOrganization = - ZohoOrganizationConverter.convertToOrganizationEntity(zohoRecord, zohoAccessConfiguration.getZohoBaseUrl()); + zohoOrgConverter.convertToOrganizationEntity(zohoRecord); EntityRecord savedRecord = entityRecordService.createEntityFromRequest( europeanaProxyEntity, zohoOrganization, dataSource); diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java index 30aafb182..a383f5964 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java @@ -226,6 +226,14 @@ public void registerZohoOrganizationShouldBeSuccessful() throws Exception { @Test public void registerZohoOrganizationGFMShouldBeSuccessful() throws Exception { + //1. create a place "Sweden" to be used to dereference zoho country for the zoho GFM org + mockMvc + .perform( + MockMvcRequestBuilders.post(IntegrationTestUtils.BASE_SERVICE_URL) + .content(loadFile(IntegrationTestUtils.PLACE_REGISTER_SWEDEN_JSON)) + .contentType(MediaType.APPLICATION_JSON_VALUE)); + + //2. register zoho GFM org ResultActions response = mockMvc.perform( MockMvcRequestBuilders.post(IntegrationTestUtils.BASE_SERVICE_URL) @@ -248,7 +256,8 @@ public void registerZohoOrganizationGFMShouldBeSuccessful() throws Exception { IntegrationTestUtils.ORGANIZATION_GFM_URI_WIKIDATA_URI))) .andExpect(jsonPath("$.prefLabel[*]", hasSize(2))) // should have Europeana, Zoho and Wikidata proxies - .andExpect(jsonPath("$.proxies", hasSize(3))); + .andExpect(jsonPath("$.proxies", hasSize(3))) + .andExpect(jsonPath("$.country.prefLabel").isNotEmpty()); } @Test diff --git a/entity-management-tests/src/integration-test/resources/content/organization.json b/entity-management-tests/src/integration-test/resources/content/organization.json index fc94efbd9..1ecd4fd6c 100644 --- a/entity-management-tests/src/integration-test/resources/content/organization.json +++ b/entity-management-tests/src/integration-test/resources/content/organization.json @@ -81,7 +81,10 @@ "GLAM" ] }, - "country": "NL", + "country" : { + "_t" : "Country", + "id" : "http://data.europeana.eu/place/4" + }, "homepage": "https://www.naturalis.nl/nl/", "phone": [ "+31-71-751-9600" diff --git a/entity-management-tests/src/integration-test/resources/content/place_register_sweden.json b/entity-management-tests/src/integration-test/resources/content/place_register_sweden.json new file mode 100644 index 000000000..609ceb1f9 --- /dev/null +++ b/entity-management-tests/src/integration-test/resources/content/place_register_sweden.json @@ -0,0 +1,4 @@ +{ + "type": "Place", + "id": "http://www.wikidata.org/entity/Q34" +} \ No newline at end of file diff --git a/entity-management-tests/src/integration-test/resources/metis-deref/place_sweden.xml b/entity-management-tests/src/integration-test/resources/metis-deref/place_sweden.xml new file mode 100644 index 000000000..2f56a6fb2 --- /dev/null +++ b/entity-management-tests/src/integration-test/resources/metis-deref/place_sweden.xml @@ -0,0 +1,20 @@ + + + + + Шведска + ISwidi + Sweden + Sweden + country in Northern Europe + Staat in Nordeuropa + + + + + 59.4 + 18.1 + +52 + + + \ No newline at end of file diff --git a/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml b/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml index 8f92f66c6..709bb9fd7 100644 --- a/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml +++ b/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml @@ -509,8 +509,11 @@ - + + diff --git a/entity-management-tests/src/integration-test/resources/zoho_country_mapping_test.json b/entity-management-tests/src/integration-test/resources/zoho_country_mapping_test.json new file mode 100644 index 000000000..29ce8a206 --- /dev/null +++ b/entity-management-tests/src/integration-test/resources/zoho_country_mapping_test.json @@ -0,0 +1,3 @@ +{ + "Sweden":"http://data.europeana.eu/place/1" +} \ No newline at end of file diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java index 7467a8849..f12a102c1 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java @@ -38,7 +38,7 @@ import eu.europeana.entitymanagement.web.model.Operation; import eu.europeana.entitymanagement.web.model.ZohoSyncReport; import eu.europeana.entitymanagement.web.model.ZohoSyncReportFields; -import eu.europeana.entitymanagement.zoho.organization.ZohoAccessConfiguration; +import eu.europeana.entitymanagement.zoho.organization.ZohoConfiguration; import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter; import eu.europeana.entitymanagement.zoho.utils.ZohoUtils; @@ -56,7 +56,9 @@ public class BaseZohoAccess { final DataSource zohoDataSource; - final ZohoAccessConfiguration zohoAccessConfiguration; + final ZohoConfiguration zohoConfiguration; + + protected final ZohoOrganizationConverter zohoOrgConverter; final ZohoSyncRepository zohoSyncRepo; @@ -68,7 +70,8 @@ public BaseZohoAccess( EntityRecordRepository entityRecordRepository, EntityManagementConfiguration emConfiguration, DataSources datasources, - ZohoAccessConfiguration zohoAccessConfiguration, + ZohoConfiguration zohoConfiguration, + ZohoOrganizationConverter zohoOrgConverter, SolrService solrService, ZohoSyncRepository zohoSyncRepo) { this.entityRecordService = entityRecordService; @@ -76,7 +79,8 @@ public BaseZohoAccess( this.entityRecordRepository = entityRecordRepository; this.emConfiguration = emConfiguration; this.datasources = datasources; - this.zohoAccessConfiguration = zohoAccessConfiguration; + this.zohoConfiguration = zohoConfiguration; + this.zohoOrgConverter = zohoOrgConverter; this.zohoDataSource = initZohoDataSource(); this.zohoSyncRepo = zohoSyncRepo; } @@ -201,7 +205,7 @@ void performDeprecationOperations(SortedSet deprecateOperations, Zoho } String generateZohoOrganizationUrl(Long zohoRecordId) { - return ZohoUtils.buildZohoOrganizationId(zohoAccessConfiguration.getZohoBaseUrl(), zohoRecordId); + return ZohoUtils.buildZohoOrganizationId(zohoConfiguration.getZohoBaseUrl(), zohoRecordId); } private void performDeprecation(ZohoSyncReport zohoSyncReport, Operation operation) { @@ -327,7 +331,7 @@ List performEntityRegistration(SortedSet createOperations, Zo */ private void performEntityRegistration(Operation operation, ZohoSyncReport zohoSyncReport, List entitiesToUpdate) { Organization zohoOrganization = - ZohoOrganizationConverter.convertToOrganizationEntity(operation.getZohoRecord(), zohoAccessConfiguration.getZohoBaseUrl()); + zohoOrgConverter.convertToOrganizationEntity(operation.getZohoRecord()); entitiesToUpdate.add(null); @@ -381,7 +385,7 @@ Optional findDupplicateOrganization(Operation operation, allCorefs.add(operation.getOrganizationId()); } allCorefs.add(zohoOrganization.getAbout()); - String Europeana_ID = ZohoOrganizationConverter.getEuropeanaIdFieldValue(operation.getZohoRecord()); + String Europeana_ID = zohoOrgConverter.getEuropeanaIdFieldValue(operation.getZohoRecord()); if(Europeana_ID != null) { allCorefs.add(Europeana_ID); } @@ -409,7 +413,7 @@ Optional findDupplicateOrganization(Operation operation, * @return */ protected boolean hasRequiredOwnership(Record zohoRecord) { - String ownerName = ZohoOrganizationConverter.getOwnerName(zohoRecord); + String ownerName = zohoOrgConverter.getOwnerName(zohoRecord); return ownerName.equals(emConfiguration.getZohoSyncOwnerFilter()); } diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java index d74a8cd4c..f414e4177 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java @@ -40,6 +40,7 @@ import eu.europeana.entitymanagement.definitions.model.Aggregation; import eu.europeana.entitymanagement.definitions.model.Concept; import eu.europeana.entitymanagement.definitions.model.ConceptScheme; +import eu.europeana.entitymanagement.definitions.model.Country; import eu.europeana.entitymanagement.definitions.model.Entity; import eu.europeana.entitymanagement.definitions.model.EntityProxy; import eu.europeana.entitymanagement.definitions.model.EntityRecord; @@ -66,7 +67,7 @@ import eu.europeana.entitymanagement.vocabulary.EntityTypes; import eu.europeana.entitymanagement.vocabulary.WebEntityConstants; import eu.europeana.entitymanagement.vocabulary.WebEntityFields; -import eu.europeana.entitymanagement.zoho.organization.ZohoAccessConfiguration; +import eu.europeana.entitymanagement.zoho.organization.ZohoConfiguration; import eu.europeana.entitymanagement.zoho.utils.WikidataUtils; import eu.europeana.entitymanagement.zoho.utils.ZohoConstants; import eu.europeana.entitymanagement.zoho.utils.ZohoException; @@ -83,7 +84,7 @@ public class EntityRecordService { private final SolrService solrService; - private final ZohoAccessConfiguration zohoAccessConfiguration; + private final ZohoConfiguration zohoConfiguration; private static final Logger logger = LogManager.getLogger(EntityRecordService.class); @@ -93,12 +94,12 @@ public class EntityRecordService { @Autowired public EntityRecordService(EntityRecordRepository entityRecordRepository, EntityManagementConfiguration emConfiguration, - ZohoAccessConfiguration zohoAccessConfiguration, + ZohoConfiguration zohoConfiguration, DataSources datasources, SolrService solrService) { this.entityRecordRepository = entityRecordRepository; this.emConfiguration = emConfiguration; - this.zohoAccessConfiguration = zohoAccessConfiguration; + this.zohoConfiguration = zohoConfiguration; this.datasources = datasources; this.solrService = solrService; } @@ -822,6 +823,8 @@ private Entity combineEntities(Entity primary, Entity secondary, List fie mergeWebResources(primary, secondary, field, consolidatedEntity); } else if (Address.class.isAssignableFrom(fieldType)) { mergeAddress(primary, secondary, field, consolidatedEntity); + } else if (Country.class.isAssignableFrom(fieldType)) { + mergeCountry(primary, secondary, field, consolidatedEntity); } } @@ -855,6 +858,17 @@ private void mergeWebResources(Entity primary, Entity secondary, Field field, } } + private void mergeCountry(Entity primary, Entity secondary, Field field, + Entity consolidatedEntity) throws IllegalAccessException { + Country primaryCountry = (Country) primary.getFieldValue(field); + Country secondaryCountry = (Country) secondary.getFieldValue(field); + if (primaryCountry == null && secondaryCountry != null) { + consolidatedEntity.setFieldValue(field, new Country(secondaryCountry)); + } else if (primaryCountry != null) { + consolidatedEntity.setFieldValue(field, new Country(primaryCountry)); + } + } + /** * Will combine the address * @@ -1283,7 +1297,7 @@ else if (StringUtils.equals(type, EntityFieldsTypes.FIELD_TYPE_URI) void updateEuropeanaIDFieldInZoho(String zohoOrganizationUrl, String europeanaId) throws EntityCreationException { try { - zohoAccessConfiguration.getZohoAccessClient().updateZohoRecordOrganizationStringField(zohoOrganizationUrl, ZohoConstants.EUROPEANA_ID_FIELD, europeanaId); + zohoConfiguration.getZohoAccessClient().updateZohoRecordOrganizationStringField(zohoOrganizationUrl, ZohoConstants.EUROPEANA_ID_FIELD, europeanaId); } catch (ZohoException e) { String message = "Updating EuropeanaID field in Zoho faild for Organization: " diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java index 2be4b6965..c6285cb75 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java @@ -31,7 +31,7 @@ import eu.europeana.entitymanagement.web.model.Operation; import eu.europeana.entitymanagement.web.model.ZohoSyncReport; import eu.europeana.entitymanagement.web.model.ZohoSyncReportFields; -import eu.europeana.entitymanagement.zoho.organization.ZohoAccessConfiguration; +import eu.europeana.entitymanagement.zoho.organization.ZohoConfiguration; import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter; import eu.europeana.entitymanagement.zoho.utils.ZohoException; @@ -42,12 +42,13 @@ public class ZohoSyncService extends BaseZohoAccess { public ZohoSyncService(EntityRecordService entityRecordService, EntityUpdateService entityUpdateService, EntityRecordRepository entityRecordRepository, EntityManagementConfiguration emConfiguration, DataSources datasources, - ZohoAccessConfiguration zohoAccessConfiguration, + ZohoConfiguration zohoConfiguration, + ZohoOrganizationConverter zohoOrgConverter, SolrService solrService, ZohoSyncRepository zohoSyncRepo) { super(entityRecordService, entityUpdateService, entityRecordRepository, emConfiguration, - datasources, zohoAccessConfiguration, solrService, zohoSyncRepo); + datasources, zohoConfiguration, zohoOrgConverter, solrService, zohoSyncRepo); } /** @@ -113,7 +114,7 @@ void synchronizeZohoOrganizations(@NotNull OffsetDateTime modifiedSince, // OffsetDateTime offsetDateTime = modifiedSince.toInstant() // .atOffset(ZoneOffset.UTC); try { - orgList = zohoAccessConfiguration.getZohoAccessClient().getZcrmRecordOrganizations(page, + orgList = zohoConfiguration.getZohoAccessClient().getZcrmRecordOrganizations(page, pageSize, modifiedSince); logExecutionProgress(orgList, page, pageSize); @@ -168,7 +169,7 @@ void synchronizeDeletedZohoOrganizations(OffsetDateTime modifiedSince, while (hasNext) { try { // list of (europeana) organizations ids - deletedRecordsInZoho = zohoAccessConfiguration.getZohoAccessClient() + deletedRecordsInZoho = zohoConfiguration.getZohoAccessClient() .getZohoDeletedRecordOrganizations(modifiedSince, startPage, pageSize); currentPageSize = deletedRecordsInZoho.size(); @@ -267,10 +268,10 @@ private void addOperation(BatchOperations operations, Long zohoId, Record zohoOr // String zohoBasedEntityId = // EntityRecordUtils.buildEntityIdUri(EntityTypes.Organization, zohoId.toString()); - String zohoRecordEuropeanaID = ZohoOrganizationConverter.getEuropeanaIdFieldValue(zohoOrg); + String zohoRecordEuropeanaID = zohoOrgConverter.getEuropeanaIdFieldValue(zohoOrg); boolean hasDpsOwner = hasRequiredOwnership(zohoOrg); - boolean markedForDeletion = ZohoOrganizationConverter.isMarkedForDeletion(zohoOrg); + boolean markedForDeletion = zohoOrgConverter.isMarkedForDeletion(zohoOrg); String emOperation = identifyOperationType(zohoId, zohoRecordEuropeanaID, entityRecord, hasDpsOwner, markedForDeletion); diff --git a/entity-management-zoho/pom.xml b/entity-management-zoho/pom.xml index 08058a1b2..7ba29f6e8 100644 --- a/entity-management-zoho/pom.xml +++ b/entity-management-zoho/pom.xml @@ -50,6 +50,12 @@ entity-management-common 1.6.3-SNAPSHOT + + eu.europeana.api + entity-management-mongo + 1.6.3-SNAPSHOT + + org.junit.jupiter junit-jupiter-api diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/wikidata/WikidataDereferenceService.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/wikidata/WikidataDereferenceService.java index 33452e8e2..9538886cb 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/wikidata/WikidataDereferenceService.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/wikidata/WikidataDereferenceService.java @@ -170,7 +170,12 @@ private String getEntityFromURL(String urlToRead) throws WikidataAccessException } private WikidataOrganization parse(String xml) throws JAXBException { + //this commented out code caused exception, i.e. Caused by: org.xml.sax.SAXParseException: Premature end of file. +// InputStream stream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)); +// return (WikidataOrganization) unmarshaller.get().unmarshal(stream); + InputStream stream = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)); - return (WikidataOrganization) unmarshaller.get().unmarshal(stream); + return (WikidataOrganization) unmarshaller.get().unmarshal( new StreamSource( stream ) ); + } } diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoAccessConfiguration.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoConfiguration.java similarity index 86% rename from entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoAccessConfiguration.java rename to entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoConfiguration.java index 805295027..b0e9e3900 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoAccessConfiguration.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoConfiguration.java @@ -1,16 +1,16 @@ package eu.europeana.entitymanagement.zoho.organization; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants; import eu.europeana.entitymanagement.zoho.ZohoAccessClient; import eu.europeana.entitymanagement.zoho.utils.ZohoException; import eu.europeana.entitymanagement.zoho.utils.ZohoInMemoryTokenStore; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.PropertySource; -@Configuration(AppConfigConstants.BEAN_ZOHO_ACCESS_CONFIGURATION) +@Configuration(AppConfigConstants.BEAN_ZOHO_CONFIGURATION) @PropertySource(value = "classpath:zoho_import.properties", ignoreResourceNotFound = true) -public class ZohoAccessConfiguration { +public class ZohoConfiguration { @Value("${zoho.email:#{null}}") private String zohoEmail; @@ -29,6 +29,9 @@ public class ZohoAccessConfiguration { @Value("${zoho.base.url:#{null}}") private String zohoBaseUrl; + + @Value("${zoho.country.mapping.file:#{null}}") + private String zohoCountryMappingFile; private volatile ZohoAccessClient zohoAccessClient; @@ -57,4 +60,9 @@ public String getZohoBaseUrl() { public void setZohoBaseUrl(String zohoBaseUrl) { this.zohoBaseUrl = zohoBaseUrl; } + + public String getZohoCountryMappingFile() { + return zohoCountryMappingFile; + } + } diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java index 4a2b37178..583582ca7 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java @@ -1,5 +1,6 @@ package eu.europeana.entitymanagement.zoho.organization; +import java.io.IOException; import java.util.Optional; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.lang.NonNull; @@ -11,25 +12,28 @@ @Service public class ZohoDereferenceService implements Dereferencer { - private final ZohoAccessConfiguration zohoAccessConfiguration; + private final ZohoOrganizationConverter zohoOrgConverter; + private final ZohoConfiguration zohoConfiguration; @Autowired - public ZohoDereferenceService(ZohoAccessConfiguration zohoAccessConfiguration) { - this.zohoAccessConfiguration = zohoAccessConfiguration; + public ZohoDereferenceService(ZohoConfiguration zohoConfiguration, ZohoOrganizationConverter zohoOrgConverter) throws IOException { + this.zohoConfiguration = zohoConfiguration; + this.zohoOrgConverter=zohoOrgConverter; } @Override public Optional dereferenceEntityById(@NonNull String id) throws Exception { Optional zohoOrganization = - zohoAccessConfiguration.getZohoAccessClient().getZohoRecordOrganizationById(id); + zohoConfiguration.getZohoAccessClient().getZohoRecordOrganizationById(id); // Gson resp = new Gson(); // System.out.println(resp.toJson(zohoOrganization.get().getKeyValues())); // if(zohoOrganization.isPresent()) { - return Optional.of(ZohoOrganizationConverter.convertToOrganizationEntity(zohoOrganization.get(), zohoAccessConfiguration.getZohoBaseUrl())); + return Optional.of(zohoOrgConverter.convertToOrganizationEntity(zohoOrganization.get())); } else { return Optional.empty(); } - } + } + } diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java index 2b7752d79..1bd8f11e3 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java @@ -1,35 +1,73 @@ package eu.europeana.entitymanagement.zoho.organization; import static eu.europeana.entitymanagement.zoho.utils.ZohoUtils.toIsoLanguage; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONObject; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; import com.zoho.crm.api.record.Record; import com.zoho.crm.api.users.User; import eu.europeana.entitymanagement.definitions.model.Address; +import eu.europeana.entitymanagement.definitions.model.Country; +import eu.europeana.entitymanagement.definitions.model.EntityRecord; import eu.europeana.entitymanagement.definitions.model.Organization; import eu.europeana.entitymanagement.definitions.model.WebResource; +import eu.europeana.entitymanagement.mongo.repository.EntityRecordRepository; import eu.europeana.entitymanagement.utils.EntityUtils; import eu.europeana.entitymanagement.zoho.utils.ZohoConstants; import eu.europeana.entitymanagement.zoho.utils.ZohoUtils; +@Component public class ZohoOrganizationConverter { + + private final EntityRecordRepository entityRecordRepository; + private final ZohoConfiguration zohoConfiguration; + private Map countryMapping; + + static final Logger logger = LogManager.getLogger(ZohoOrganizationConverter.class); private static final String POSITION_SEPARATOR = "_"; - private ZohoOrganizationConverter() { - // private constructor to prevent instantiation + @Autowired + public ZohoOrganizationConverter(EntityRecordRepository entityRecordRepository, ZohoConfiguration zohoConfiguration) throws IOException { + this.entityRecordRepository=entityRecordRepository; + this.zohoConfiguration=zohoConfiguration; + //reading the country mapping file + countryMapping = new HashMap<>(); + String countryMappingFile = this.zohoConfiguration.getZohoCountryMappingFile(); + try (InputStream inputStream = getClass().getResourceAsStream(countryMappingFile)) { + assert inputStream != null; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { + String contents = reader.lines().collect(Collectors.joining(System.lineSeparator())); + if(! StringUtils.isBlank(contents)) { + JSONObject contentsJson = new JSONObject(contents); + contentsJson.keys().forEachRemaining(key -> { + countryMapping.put(key, contentsJson.getString(key)); + }); + } + } + } + } - public static Organization convertToOrganizationEntity(Record zohoRecord, String zohoBaseUrl) { + public Organization convertToOrganizationEntity(Record zohoRecord) { Organization org = new Organization(); Long zohoId = zohoRecord.getId(); - org.setAbout(ZohoUtils.buildZohoOrganizationId(zohoBaseUrl, zohoRecord.getId())); + org.setAbout(ZohoUtils.buildZohoOrganizationId(zohoConfiguration.getZohoBaseUrl(), zohoRecord.getId())); org.setIdentifier(List.of(Long.toString(zohoId))); // extract language maps @@ -57,21 +95,36 @@ public static Organization convertToOrganizationEntity(Record zohoRecord, String Locale.ENGLISH.getLanguage(), organizationRoleStringList)); } - String organizationCountry = - toEdmCountry(getStringFieldValue(zohoRecord, ZohoConstants.ORGANIZATION_COUNTRY_FIELD)); - org.setCountry(organizationCountry); - org.setSameReferenceLinks(getAllSameAs(zohoRecord)); - Address address = new Address(); address.setVcardStreetAddress( ZohoUtils.stringFieldSupplier(zohoRecord.getKeyValue(ZohoConstants.STREET_FIELD))); address.setVcardLocality( ZohoUtils.stringFieldSupplier(zohoRecord.getKeyValue(ZohoConstants.CITY_FIELD))); - address.setVcardCountryName( - ZohoUtils.stringFieldSupplier(zohoRecord.getKeyValue(ZohoConstants.COUNTRY_FIELD))); + String vcardCountryName = ZohoUtils.stringFieldSupplier(zohoRecord.getKeyValue(ZohoConstants.COUNTRY_FIELD)); + address.setVcardCountryName(vcardCountryName); address.setVcardPostalCode( ZohoUtils.stringFieldSupplier(zohoRecord.getKeyValue(ZohoConstants.ZIP_CODE_FIELD))); + //set country + String orgCountryUri = countryMapping.get(vcardCountryName); + if(StringUtils.isBlank(orgCountryUri)) { + logger.info("The mapping for the zoho country: {}, to the uri does not exist.", vcardCountryName); + } + else { + Country orgCountry = new Country(); + orgCountry.setId(orgCountryUri); + EntityRecord orgEntityRecord = entityRecordRepository.findByEntityId(orgCountryUri); + if(orgEntityRecord==null) { + logger.info("The entity record to be dereferenced for the zoho country with the id: {}, does not exist in the db.", orgCountryUri); + } + else { + orgCountry.setPrefLabel(orgEntityRecord.getEntity().getPrefLabel()); + } + org.setCountry(orgCountry); + } + + org.setSameReferenceLinks(getAllSameAs(zohoRecord)); + // only set address if it contains metadata properties. if (address.hasMetadataProperties()) { address.setAbout(org.getAbout() + ZohoConstants.ADDRESS_ABOUT); @@ -105,7 +158,7 @@ public static Organization convertToOrganizationEntity(Record zohoRecord, String return org; } - private static WebResource buildWebResource(Record zohoRecord, String logoFieldName) { + private WebResource buildWebResource(Record zohoRecord, String logoFieldName) { String id = getStringFieldValue(zohoRecord, logoFieldName); if (id == null) { return null; @@ -116,14 +169,14 @@ private static WebResource buildWebResource(Record zohoRecord, String logoFieldN return resource; } - private static Map getPrefLabel(Map> allLabels) { + private Map getPrefLabel(Map> allLabels) { Map prefLabel = new LinkedHashMap<>(allLabels.size()); // first label for each language goes to prefLabel map allLabels.forEach((key, value) -> prefLabel.put(key, value.get(0))); return prefLabel; } - private static Map> getAltLabel(Map> allLabels) { + private Map> getAltLabel(Map> allLabels) { Map> altLabel = new LinkedHashMap<>(); for (Map.Entry> entry : allLabels.entrySet()) { int size = entry.getValue().size(); @@ -135,7 +188,7 @@ private static Map> getAltLabel(Map> a return altLabel; } - private static Map> getAllRecordLabels(Record zohoRecord) { + private Map> getAllRecordLabels(Record zohoRecord) { Map> allLabels = new LinkedHashMap<>(); // read account name first @@ -155,7 +208,7 @@ private static Map> getAllRecordLabels(Record zohoRecord) { return allLabels; } - static void addLabel( + void addLabel( Record zohoRecord, Map> allLabels, String langFieldName, @@ -167,15 +220,15 @@ static void addLabel( } } - public static String getStringFieldValue(Record zohoRecord, String zohoFieldName) { + public String getStringFieldValue(Record zohoRecord, String zohoFieldName) { return ZohoUtils.stringFieldSupplier(zohoRecord.getKeyValue(zohoFieldName)); } - public static String getEuropeanaIdFieldValue(Record zohoRecord) { + public String getEuropeanaIdFieldValue(Record zohoRecord) { return getStringFieldValue(zohoRecord, ZohoConstants.EUROPEANA_ID_FIELD); } - static List getTextAreaFieldValues(Record zohoRecord, String zohoFieldName) { + List getTextAreaFieldValues(Record zohoRecord, String zohoFieldName) { String textArea = ZohoUtils.stringFieldSupplier(zohoRecord.getKeyValue(zohoFieldName)); if (StringUtils.isBlank(textArea)) { return Collections.emptyList(); @@ -184,22 +237,22 @@ static List getTextAreaFieldValues(Record zohoRecord, String zohoFieldNa return List.of(StringUtils.split(textArea, "\n")); } - static String getIsoLanguage(Record zohoRecord, String zohoLangFieldName) { + String getIsoLanguage(Record zohoRecord, String zohoLangFieldName) { return toIsoLanguage(getStringFieldValue(zohoRecord, zohoLangFieldName)); } - static void addValueToList(String value, List list) { + void addValueToList(String value, List list) { if (value != null) { list.add(value); } } - static void addLabel(Map> allLabels, String isoLanguage, String label) { + void addLabel(Map> allLabels, String isoLanguage, String label) { allLabels.computeIfAbsent(isoLanguage, k -> new ArrayList<>()); allLabels.get(isoLanguage).add(label); } - private static List getAllSameAs(Record zohoRecord) { + private List getAllSameAs(Record zohoRecord) { List sameAsList = new ArrayList<>(); for (int i = 1; i <= ZohoConstants.SAME_AS_CODE_LENGTH; i++) { String sameAs = getStringFieldValue(zohoRecord, ZohoConstants.SAME_AS_FIELD + "_" + i); @@ -210,22 +263,22 @@ private static List getAllSameAs(Record zohoRecord) { return sameAsList; } - private static String toEdmCountry(String organizationCountry) { - if (StringUtils.isBlank(organizationCountry)) { - return null; - } else { - String isoCode = null; - int commaSeparatorPos = organizationCountry.indexOf(44); - int bracketSeparatorPos = organizationCountry.indexOf(40); - if (commaSeparatorPos > 0) { - isoCode = organizationCountry.substring(commaSeparatorPos + 1).trim(); - } else if (bracketSeparatorPos > 0) { - isoCode = organizationCountry.substring(0, bracketSeparatorPos).trim(); - } - - return isoCode; - } - } +// private static String toEdmCountry(String organizationCountry) { +// if (StringUtils.isBlank(organizationCountry)) { +// return null; +// } else { +// String isoCode = null; +// int commaSeparatorPos = organizationCountry.indexOf(44); +// int bracketSeparatorPos = organizationCountry.indexOf(40); +// if (commaSeparatorPos > 0) { +// isoCode = organizationCountry.substring(commaSeparatorPos + 1).trim(); +// } else if (bracketSeparatorPos > 0) { +// isoCode = organizationCountry.substring(0, bracketSeparatorPos).trim(); +// } +// +// return isoCode; +// } +// } /** * The method is to process the ZOHO_OWNER_FIELD name value @@ -233,11 +286,11 @@ private static String toEdmCountry(String organizationCountry) { * @param recordOrganization * @return */ - public static String getOwnerName(Record recordOrganization) { + public String getOwnerName(Record recordOrganization) { return ((User) recordOrganization.getKeyValue(ZohoConstants.ZOHO_OWNER_FIELD)).getName(); } - public static boolean isMarkedForDeletion(Record recordOrganization) { + public boolean isMarkedForDeletion(Record recordOrganization) { Object scheduledDeletion = recordOrganization.getKeyValue(ZohoConstants.ZOHO_SCHEDULED_DELETION); if (scheduledDeletion == null) { diff --git a/entity-management-zoho/src/main/resources/zoho_country_mapping.json b/entity-management-zoho/src/main/resources/zoho_country_mapping.json new file mode 100644 index 000000000..320d2e1c1 --- /dev/null +++ b/entity-management-zoho/src/main/resources/zoho_country_mapping.json @@ -0,0 +1,3 @@ +{ + "Sweden":"http://data.europeana.eu/place/4" +} \ No newline at end of file diff --git a/entity-management-zoho/src/main/resources/zoho_import.properties.template b/entity-management-zoho/src/main/resources/zoho_import.properties.template index 366f519a1..de29939d0 100644 --- a/entity-management-zoho/src/main/resources/zoho_import.properties.template +++ b/entity-management-zoho/src/main/resources/zoho_import.properties.template @@ -3,4 +3,5 @@ zoho.client.id= zoho.client.secret= zoho.refresh.token= zoho.redirect.url= -zoho.base.url= \ No newline at end of file +zoho.base.url= +zoho.country.mapping.file= \ No newline at end of file From b924abecb02580af8d7da2e9bee6c4a85431772e Mon Sep 17 00:00:00 2001 From: Srdjan Stevanetic Date: Tue, 5 Dec 2023 17:44:52 +0100 Subject: [PATCH 04/22] fixed entity fields validator test --- .../entitymanagement/validation/EntityFieldsValidatorTest.java | 2 +- .../src/test/resources/validation/organization-validation.json | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/entity-management-web/src/test/java/eu/europeana/entitymanagement/validation/EntityFieldsValidatorTest.java b/entity-management-web/src/test/java/eu/europeana/entitymanagement/validation/EntityFieldsValidatorTest.java index 8de61514e..7d579b8e6 100644 --- a/entity-management-web/src/test/java/eu/europeana/entitymanagement/validation/EntityFieldsValidatorTest.java +++ b/entity-management-web/src/test/java/eu/europeana/entitymanagement/validation/EntityFieldsValidatorTest.java @@ -57,7 +57,7 @@ public void completeValidationEntityFieldsOrganization() throws IOException { for (ConstraintViolation violation : violations) { System.out.println(violation.getMessageTemplate()); } - Assertions.assertEquals(13, violations.size()); + Assertions.assertEquals(12, violations.size()); } @Test diff --git a/entity-management-web/src/test/resources/validation/organization-validation.json b/entity-management-web/src/test/resources/validation/organization-validation.json index 1cb5eb566..42997001b 100644 --- a/entity-management-web/src/test/resources/validation/organization-validation.json +++ b/entity-management-web/src/test/resources/validation/organization-validation.json @@ -83,7 +83,6 @@ "GLAM" ] }, - "country": "http://NL", "homepage": "wrong-for-testing-4", "phone": [ "http://+31-71-751-9600" From 24723952c620b4c88bcadd552a1ff6c75b9ebbdd Mon Sep 17 00:00:00 2001 From: Srdjan Stevanetic Date: Tue, 5 Dec 2023 18:26:51 +0100 Subject: [PATCH 05/22] sonar bugs improved --- .../solr/model/SolrOrganization.java | 7 +++++-- .../service/DereferenceServiceIT.java | 1 - .../web/service/BaseZohoAccess.java | 13 ++++--------- .../web/service/EntityRecordService.java | 5 ++--- .../zoho/organization/ZohoDereferenceService.java | 3 +-- 5 files changed, 12 insertions(+), 17 deletions(-) diff --git a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java index 7792360b4..0a382c203 100644 --- a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java +++ b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java @@ -85,8 +85,11 @@ public SolrOrganization(Organization organization) { this.phone = organization.getPhone(); if (organization.getMbox() != null) this.mbox = new ArrayList<>(organization.getMbox()); setEuropeanaRole(organization.getEuropeanaRole()); - this.countryId = organization.getCountry().getId(); - setCountryPrefLabel(organization.getCountry().getPrefLabel()); + + if(organization.getCountry()!=null) { + this.countryId = organization.getCountry().getId(); + this.setCountryPrefLabel(organization.getCountry().getPrefLabel()); + } if (organization.getSameReferenceLinks() != null) { this.sameAs = new ArrayList<>(organization.getSameReferenceLinks()); diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java index a56338230..7a17488c6 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java @@ -79,7 +79,6 @@ public void zohoOrganizationDereferenceTest() throws Exception { assertEquals(1, org.getAcronym().size()); assertEquals(1, org.getEuropeanaRole().size()); assertEquals(1, org.getAcronym().size()); - assertEquals("FR", org.getCountry()); Assertions.assertNotNull(org.getHomepage()); Assertions.assertNotNull(org.getLogo()); Assertions.assertNotNull(org.getAddress().getVcardStreetAddress()); diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java index f12a102c1..b298c2334 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java @@ -62,7 +62,7 @@ public class BaseZohoAccess { final ZohoSyncRepository zohoSyncRepo; - static final Logger logger = LogManager.getLogger(ZohoSyncService.class); + static final Logger logger = LogManager.getLogger(BaseZohoAccess.class); public BaseZohoAccess( EntityRecordService entityRecordService, @@ -95,17 +95,12 @@ protected DataSource initZohoDataSource() { return zohoDatasource.get(); } - OffsetDateTime generateFixDate() { + OffsetDateTime generateFixDate() throws ParseException { //hardcoded date, just for manual testing SimpleDateFormat formatter = new SimpleDateFormat("dd-MMM-yyyy HH:mm:ss", Locale.ENGLISH); String dateInString = "23-Oct-2023 14:38:00"; - try { - Date date = formatter.parse(dateInString); - return DateUtils.toOffsetDateTime(date); - } catch (ParseException e) { - e.printStackTrace(); - } - return null; + Date date = formatter.parse(dateInString); + return DateUtils.toOffsetDateTime(date); } protected String buildErrorMessage(String message, List ids) { diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java index f414e4177..9e7c67417 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java @@ -496,8 +496,7 @@ String generateEntityId(Entity datasourceResponse) throws UnsupportedEntityTypeE // } else { // entityId = generateEntityId(type, null); // } - entityId = generateEntityId(type, null); - return entityId; + return generateEntityId(type, null); } List buildSameAsReferenceLinks(String externalProxyId, Entity datasourceResponse, @@ -1302,7 +1301,7 @@ void updateEuropeanaIDFieldInZoho(String zohoOrganizationUrl, String europeanaI String message = "Updating EuropeanaID field in Zoho faild for Organization: " + zohoOrganizationUrl; - throw new EntityCreationException(message); + throw new EntityCreationException(message, e); } } } diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java index 583582ca7..3e2340c6a 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java @@ -1,6 +1,5 @@ package eu.europeana.entitymanagement.zoho.organization; -import java.io.IOException; import java.util.Optional; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.lang.NonNull; @@ -16,7 +15,7 @@ public class ZohoDereferenceService implements Dereferencer { private final ZohoConfiguration zohoConfiguration; @Autowired - public ZohoDereferenceService(ZohoConfiguration zohoConfiguration, ZohoOrganizationConverter zohoOrgConverter) throws IOException { + public ZohoDereferenceService(ZohoConfiguration zohoConfiguration, ZohoOrganizationConverter zohoOrgConverter) { this.zohoConfiguration = zohoConfiguration; this.zohoOrgConverter=zohoOrgConverter; } From 250165fb09ca159b4525ccd3f29f812a1ea59122 Mon Sep 17 00:00:00 2001 From: SrdjanStevanetic Date: Thu, 7 Dec 2023 12:57:15 +0100 Subject: [PATCH 06/22] improved combineEntities to do a deep copy of objects --- development_tips.txt | 3 +- .../definitions/model/Address.java | 45 ++- .../definitions/model/Country.java | 20 ++ .../definitions/model/WebResource.java | 9 +- .../vocabulary/EntityFieldsTypes.java | 3 +- .../web/service/EntityRecordService.java | 313 ++++++++++-------- 6 files changed, 252 insertions(+), 141 deletions(-) diff --git a/development_tips.txt b/development_tips.txt index 9b9bc00ef..209abcb95 100644 --- a/development_tips.txt +++ b/development_tips.txt @@ -16,5 +16,6 @@ public ZohoDereferenceService(ZohoAccessConfiguration zohoAccessConfiguration, The printed response can then be saved to a file in test/resources. Also make sure to adjust the fields to be either string or list type (no map). -2. Whenever introducing new field types for any of the model definition classes, e.g. Organization, make sure to properly hanlde the merging of the entities in the method combineEntities. +2. Whenever introducing new types for a field for any of the model definition classes, e.g. Organization, make sure to properly hanlde the merging of the entities in the method combineEntities. +Please see the java doc on that method. Also override the equals() method for the new field type and add it to the EntityFieldsTypes class. diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Address.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Address.java index a153578bf..6f5ebb984 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Address.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Address.java @@ -1,13 +1,22 @@ package eu.europeana.entitymanagement.definitions.model; -import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.*; - +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.CONTEXT; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.COUNTRY_NAME; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.HAS_GEO; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ID; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.LOCALITY; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.POSTAL_CODE; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.POST_OFFICE_BOX; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.REGION; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.STREET_ADDRESS; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.TYPE; +import java.util.Objects; +import org.apache.commons.lang3.StringUtils; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonSetter; import dev.morphia.annotations.Embedded; -import org.apache.commons.lang3.StringUtils; @Embedded @JsonInclude(value = JsonInclude.Include.NON_EMPTY) @@ -127,4 +136,34 @@ public boolean hasMetadataProperties() { || StringUtils.isNotEmpty(countryName) || StringUtils.isNotEmpty(hasGeo); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Address that = (Address) o; + + if (!Objects.equals(about, that.getAbout())) return false; + if (!Objects.equals(streetAddress, that.getVcardStreetAddress())) return false; + if (!Objects.equals(postalCode, that.getVcardPostalCode())) return false; + if (!Objects.equals(postBox, that.getVcardPostOfficeBox())) return false; + if (!Objects.equals(locality, that.getVcardLocality())) return false; + if (!Objects.equals(countryName, that.getVcardCountryName())) return false; + return Objects.equals(hasGeo, that.getVcardHasGeo()); + } + + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((about == null) ? 0 : about.hashCode()); + result = prime * result + ((streetAddress == null) ? 0 : streetAddress.hashCode()); + result = prime * result + ((postalCode == null) ? 0 : postalCode.hashCode()); + result = prime * result + ((postBox == null) ? 0 : postBox.hashCode()); + result = prime * result + ((locality == null) ? 0 : locality.hashCode()); + result = prime * result + ((countryName == null) ? 0 : countryName.hashCode()); + result = prime * result + ((hasGeo == null) ? 0 : hasGeo.hashCode()); + return result; + } + } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Country.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Country.java index 29ec37cab..f77d48133 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Country.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Country.java @@ -5,6 +5,7 @@ import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.PREF_LABEL; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonPropertyOrder; @@ -56,4 +57,23 @@ public void setPrefLabel(Map prefLabel) { this.prefLabel = prefLabel; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Country that = (Country) o; + + if (!Objects.equals(id, that.getId())) return false; + return Objects.equals(prefLabel, that.getPrefLabel()); + } + + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((id == null) ? 0 : id.hashCode()); + result = prime * result + ((prefLabel == null) ? 0 : prefLabel.hashCode()); + return result; + } + } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/WebResource.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/WebResource.java index 5688fb9c2..d71e7703b 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/WebResource.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/WebResource.java @@ -3,7 +3,7 @@ import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ID; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.SOURCE; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.THUMBNAIL; - +import java.util.Objects; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; @@ -11,7 +11,6 @@ import com.fasterxml.jackson.annotation.JsonSetter; import dev.morphia.annotations.Embedded; import eu.europeana.entitymanagement.vocabulary.WebEntityFields; -import java.util.Objects; @Embedded @JsonInclude(value = JsonInclude.Include.NON_EMPTY) @@ -82,9 +81,9 @@ public boolean equals(Object o) { WebResource that = (WebResource) o; - if (!Objects.equals(source, that.source)) return false; - if (!id.equals(that.id)) return false; - return Objects.equals(thumbnail, that.thumbnail); + if (!Objects.equals(source, that.getSource())) return false; + if (!id.equals(that.getId())) return false; + return Objects.equals(thumbnail, that.getThumbnail()); } public int hashCode() { diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java index 9ca78fd8c..c07678e92 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java @@ -101,7 +101,7 @@ public enum EntityFieldsTypes { logo(EntityFieldsTypes.FIELD_TYPE_WEB_RESOURCE, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), europeanaRole( EntityFieldsTypes.FIELD_TYPE_KEYWORD, true, EntityFieldsTypes.FIELD_CARDINALITY_0_INFINITE), - country(EntityFieldsTypes.FIELD_TYPE_TEXT, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), + country(EntityFieldsTypes.FIELD_TYPE_COUNTRY, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), homepage(EntityFieldsTypes.FIELD_TYPE_URI, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), phone( EntityFieldsTypes.FIELD_TYPE_KEYWORD, false, EntityFieldsTypes.FIELD_CARDINALITY_0_INFINITE), @@ -137,6 +137,7 @@ public enum EntityFieldsTypes { public static final String FIELD_TYPE_DATE_OR_URI = "Date or URI"; public static final String FIELD_TYPE_WEB_RESOURCE = "WebResource"; public static final String FIELD_TYPE_ADDRESS = "Address"; + public static final String FIELD_TYPE_COUNTRY = "Country"; public static final String FIELD_CARDINALITY_1_1 = "1..1"; public static final String FIELD_CARDINALITY_0_1 = "0..1"; diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java index 9e7c67417..81fdc2c1e 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java @@ -753,8 +753,9 @@ public void replaceEuropeanaProxy(final Entity updateRequestEntity, EntityRecord } /** - * Reconciles metadata between two entities. - * + * Merges metadata between two entities. This method performs a deep copy of the objects, + * for the mutable (custom) filed types. + * * @param primary Primary entity. Metadata from this entity takes precedence * @param secondary Secondary entity. Metadata from this entity is only used if no matching field * is contained within the primary entity. @@ -781,18 +782,7 @@ private Entity combineEntities(Entity primary, Entity secondary, List fie Class fieldType = field.getType(); String fieldName = field.getName(); - if (fieldType.isArray()) { - Object[] mergedArray = mergeArrays(primary, secondary, field, accumulate); - consolidatedEntity.setFieldValue(field, mergedArray); - - } else if (List.class.isAssignableFrom(fieldType)) { - List fieldValuePrimaryObjectList = (List) primary.getFieldValue(field); - List fieldValueSecondaryObjectList = - (List) secondary.getFieldValue(field); - mergeList(consolidatedEntity, fieldValuePrimaryObjectList, fieldValueSecondaryObjectList, - field, accumulate); - - } else if (isStringOrPrimitive(fieldType)) { + if (isStringOrPrimitive(fieldType)) { Object fieldValuePrimaryObjectPrimitiveOrString = primary.getFieldValue(field); Object fieldValueSecondaryObjectPrimitiveOrString = secondary.getFieldValue(field); @@ -814,16 +804,20 @@ private Entity combineEntities(Entity primary, Entity secondary, List fie consolidatedEntity.setFieldValue(field, new Date(((Date) fieldValuePrimaryObjectDate).getTime())); } - + } else if (fieldType.isArray()) { + Object[] mergedArray = mergeArrays(primary, secondary, field, accumulate); + consolidatedEntity.setFieldValue(field, mergedArray); + } else if (List.class.isAssignableFrom(fieldType)) { + List fieldValuePrimaryObjectList = (List) primary.getFieldValue(field); + List fieldValueSecondaryObjectList = + (List) secondary.getFieldValue(field); + mergeList(consolidatedEntity, fieldValuePrimaryObjectList, fieldValueSecondaryObjectList, + field, accumulate); } else if (Map.class.isAssignableFrom(fieldType)) { combineEntities(consolidatedEntity, primary, secondary, prefLabelsForAltLabels, field, fieldName, accumulate); - } else if (WebResource.class.isAssignableFrom(fieldType)) { - mergeWebResources(primary, secondary, field, consolidatedEntity); - } else if (Address.class.isAssignableFrom(fieldType)) { - mergeAddress(primary, secondary, field, consolidatedEntity); - } else if (Country.class.isAssignableFrom(fieldType)) { - mergeCountry(primary, secondary, field, consolidatedEntity); + } else { + mergeCustomObjects(primary, secondary, field, consolidatedEntity); } } @@ -837,75 +831,39 @@ private Entity combineEntities(Entity primary, Entity secondary, List fie return consolidatedEntity; } - /** - * Will merge the Web Resources - * - * @param primary - * @param secondary - * @param field - * @param consolidatedEntity - * @throws IllegalAccessException - */ - private void mergeWebResources(Entity primary, Entity secondary, Field field, - Entity consolidatedEntity) throws IllegalAccessException { - WebResource primaryWebResource = (WebResource) primary.getFieldValue(field); - WebResource secondaryWebResource = (WebResource) secondary.getFieldValue(field); - if (primaryWebResource == null && secondaryWebResource != null) { - consolidatedEntity.setFieldValue(field, new WebResource(secondaryWebResource)); - } else if (primaryWebResource != null) { - consolidatedEntity.setFieldValue(field, new WebResource(primaryWebResource)); - } - } - - private void mergeCountry(Entity primary, Entity secondary, Field field, - Entity consolidatedEntity) throws IllegalAccessException { - Country primaryCountry = (Country) primary.getFieldValue(field); - Country secondaryCountry = (Country) secondary.getFieldValue(field); - if (primaryCountry == null && secondaryCountry != null) { - consolidatedEntity.setFieldValue(field, new Country(secondaryCountry)); - } else if (primaryCountry != null) { - consolidatedEntity.setFieldValue(field, new Country(primaryCountry)); - } - } - - /** - * Will combine the address - * - * @param primary - * @param secondary - * @param field - * @param consolidatedEntity - * @throws IllegalAccessException - */ - private void mergeAddress(Entity primary, Entity secondary, Field field, - Entity consolidatedEntity) throws IllegalAccessException { - Address primaryAddress = (Address) primary.getFieldValue(field); - Address secondaryAddress = (Address) secondary.getFieldValue(field); - if (primaryAddress == null && secondaryAddress != null) { - consolidatedEntity.setFieldValue(field, new Address(secondaryAddress)); - } else if (primaryAddress != null) { - consolidatedEntity.setFieldValue(field, new Address(primaryAddress)); + private void mergeCustomObjects(Entity primary, Entity secondary, Field field, + Entity consolidatedEntity) throws IllegalAccessException, IllegalArgumentException, EntityUpdateException { + Object primaryObj = primary.getFieldValue(field); + Object secondaryObj = secondary.getFieldValue(field); + if (primaryObj == null && secondaryObj != null) { + consolidatedEntity.setFieldValue(field, deepCopyOfObject(secondaryObj)); + } else if (primaryObj != null) { + consolidatedEntity.setFieldValue(field, deepCopyOfObject(primaryObj)); } } boolean isStringOrPrimitive(Class fieldType) { - return String.class.isAssignableFrom(fieldType) || fieldType.isPrimitive() - || Float.class.isAssignableFrom(fieldType) || Integer.class.isAssignableFrom(fieldType); + return + String.class.isAssignableFrom(fieldType) + || fieldType.isPrimitive() + || Float.class.isAssignableFrom(fieldType) || Integer.class.isAssignableFrom(fieldType) + || Double.class.isAssignableFrom(fieldType) || Short.class.isAssignableFrom(fieldType) + || Byte.class.isAssignableFrom(fieldType) || Boolean.class.isAssignableFrom(fieldType) + || Long.class.isAssignableFrom(fieldType); } @SuppressWarnings({"unchecked", "rawtypes"}) void combineEntities(Entity consolidatedEntity, Entity primary, Entity secondary, Map prefLabelsForAltLabels, Field field, String fieldName, boolean accumulate) - throws IllegalAccessException { + throws IllegalAccessException, EntityUpdateException { // TODO: refactor implemetation Map fieldValuePrimaryObjectMap = (Map) primary.getFieldValue(field); Map fieldValueSecondaryObjectMap = (Map) secondary.getFieldValue(field); - Map fieldValuePrimaryObject = initialiseObjectMap(fieldValuePrimaryObjectMap); - Map fieldValueSecondaryObject = - initialiseObjectMap(fieldValueSecondaryObjectMap); + Map fieldValuePrimaryObject = deepCopyOfMap(fieldValuePrimaryObjectMap); + Map fieldValueSecondaryObject = deepCopyOfMap(fieldValueSecondaryObjectMap); if (CollectionUtils.isEmpty(fieldValuePrimaryObject) && !CollectionUtils.isEmpty(fieldValueSecondaryObject)) { @@ -936,16 +894,16 @@ void combineEntities(Entity consolidatedEntity, Entity primary, Entity secondary * @param elemSecondary * @param fieldName * @param prefLabelsForAltLabels + * @throws EntityUpdateException */ @SuppressWarnings({"rawtypes", "unchecked"}) private void mergePrimarySecondaryListWitoutDuplicates( Map fieldValuePrimaryObject, Object key, Map.Entry elemSecondary, - String fieldName, Map prefLabelsForAltLabels) { + String fieldName, Map prefLabelsForAltLabels) throws EntityUpdateException { if (fieldValuePrimaryObject.containsKey(key) - && List.class.isAssignableFrom(elemSecondary.getValue().getClass())) { + && List.class.isAssignableFrom(elemSecondary.getValue().getClass())) { List listSecondaryObject = (List) elemSecondary.getValue(); - List listPrimaryObject = - new ArrayList<>((List) fieldValuePrimaryObject.get(key)); + List listPrimaryObject = deepCopyOfList((List) fieldValuePrimaryObject.get(key)); boolean listPrimaryObjectChanged = false; for (Object elemSecondaryList : listSecondaryObject) { // check if value already exists in the primary list. @@ -975,16 +933,9 @@ else if (fieldValuePrimaryObject.containsKey(key) && fieldName.toLowerCase().con } } - private Map initialiseObjectMap(Map fieldValueObjectMap) { - if (fieldValueObjectMap != null) { - return new HashMap<>(fieldValueObjectMap); - } - return new HashMap<>(); - } - @SuppressWarnings("unchecked") void mergeSkippedPrefLabels(Entity consilidatedEntity, Map prefLabelsForAltLabels, - List allEntityFields) throws IllegalAccessException { + List allEntityFields) throws IllegalAccessException, EntityUpdateException { /* * adding the preferred labels from the secondary object to the alternative labels of * consolidated object @@ -995,8 +946,7 @@ void mergeSkippedPrefLabels(Entity consilidatedEntity, Map prefL if (isFieldAltLabel(fieldName)) { Map altLabelConsolidatedMap = (Map) consilidatedEntity.getFieldValue(field); - Map altLabelPrimaryObject = - initialiseAltLabelMap(altLabelConsolidatedMap); + Map altLabelPrimaryObject = deepCopyOfMap(altLabelConsolidatedMap); boolean altLabelPrimaryValueChanged = false; altLabelPrimaryValueChanged = addValuesToAltLabel(prefLabelsForAltLabels, altLabelPrimaryObject, altLabelPrimaryValueChanged); @@ -1011,12 +961,12 @@ void mergeSkippedPrefLabels(Entity consilidatedEntity, Map prefL @SuppressWarnings("unchecked") private boolean addValuesToAltLabel(Map prefLabelsForAltLabels, - Map altLabelPrimaryObject, boolean altLabelPrimaryValueChanged) { + Map altLabelPrimaryObject, boolean altLabelPrimaryValueChanged) throws EntityUpdateException { for (Map.Entry prefLabel : prefLabelsForAltLabels.entrySet()) { String keyPrefLabel = (String) prefLabel.getKey(); List altLabelPrimaryObjectList = (List) altLabelPrimaryObject.get(keyPrefLabel); - List altLabelPrimaryValue = initialiseAltLabelList(altLabelPrimaryObjectList); + List altLabelPrimaryValue = deepCopyOfList(altLabelPrimaryObjectList); if (shouldValuesBeAddedToAltLabel(altLabelPrimaryValue, prefLabel)) { altLabelPrimaryValue.add(prefLabel.getValue()); if (altLabelPrimaryValueChanged == false) { @@ -1038,33 +988,13 @@ private boolean shouldValuesBeAddedToAltLabel(List altLabelPrimaryValue, .ifValueAlreadyExistsInList(altLabelPrimaryValue, prefLabel.getValue(), true)); } - private Map initialiseAltLabelMap(Map altLabelConsolidatedMap) { - if (altLabelConsolidatedMap != null) { - return new HashMap<>(altLabelConsolidatedMap); - } - return new HashMap<>(); - } - - private List initialiseAltLabelList(List altLabelPrimaryObjectList) { - if (altLabelPrimaryObjectList != null) { - return new ArrayList<>(altLabelPrimaryObjectList); - } - return new ArrayList<>(); - } - void mergeList(Entity consolidatedEntity, List fieldValuePrimaryObjectList, List fieldValueSecondaryObjectList, Field field, boolean accumulate) - throws IllegalAccessException { - List fieldValuePrimaryObject = null; - List fieldValueSecondaryObject = null; - if (fieldValuePrimaryObjectList != null) { - fieldValuePrimaryObject = new ArrayList(fieldValuePrimaryObjectList); - } - if (fieldValueSecondaryObjectList != null) { - fieldValueSecondaryObject = new ArrayList(fieldValueSecondaryObjectList); - } + throws IllegalAccessException, EntityUpdateException { + List fieldValuePrimaryObject = deepCopyOfList(fieldValuePrimaryObjectList); + List fieldValueSecondaryObject = deepCopyOfList(fieldValueSecondaryObjectList); - if (fieldValuePrimaryObject != null && fieldValueSecondaryObject != null) { + if (!CollectionUtils.isEmpty(fieldValuePrimaryObject) && !CollectionUtils.isEmpty(fieldValueSecondaryObject)) { if (accumulate) { for (Object secondaryObjectListObject : fieldValueSecondaryObject) { addToPrimaryList(field, fieldValuePrimaryObject, secondaryObjectListObject); @@ -1074,10 +1004,10 @@ void mergeList(Entity consolidatedEntity, List fieldValuePrimaryObjectLi consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); } return; - } else if (fieldValuePrimaryObject == null && fieldValueSecondaryObject != null) { + } else if (!CollectionUtils.isEmpty(fieldValueSecondaryObject)) { consolidatedEntity.setFieldValue(field, fieldValueSecondaryObject); return; - } else if (fieldValuePrimaryObject != null && fieldValueSecondaryObject == null) { + } else if (!CollectionUtils.isEmpty(fieldValuePrimaryObject)) { consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); return; } @@ -1100,30 +1030,151 @@ secondaryObjectListObject, doSloppyMatch(field.getName()))) { } Object[] mergeArrays(Entity primary, Entity secondary, Field field, boolean append) - throws IllegalAccessException { + throws IllegalAccessException, EntityUpdateException { Object[] primaryArray = (Object[]) primary.getFieldValue(field); Object[] secondaryArray = (Object[]) secondary.getFieldValue(field); - - if (primaryArray == null && secondaryArray == null) { - return null; - } else if (primaryArray == null) { - // return a clone of the secondary - return secondaryArray.clone(); - } else if (secondaryArray == null || !append) { - // return a clone of the primary if we're not appending - return primaryArray.clone(); + + Object[] deepCopyPrimaryArray = deepCopyOfArray(primaryArray); + Object[] deepCopySecondaryArray = deepCopyOfArray(secondaryArray); + + if (deepCopyPrimaryArray.length==0 && deepCopySecondaryArray.length==0) { + return deepCopyPrimaryArray; + } else if (deepCopyPrimaryArray.length==0) { + return deepCopySecondaryArray; + } else if (secondaryArray.length==0 || !append) { + return deepCopyPrimaryArray; } // merge arrays - Set mergedAndOrdered = new TreeSet<>(Arrays.asList(primaryArray)); - for (Object second : secondaryArray) { - if (!EMCollectionUtils.ifValueAlreadyExistsInList(Arrays.asList(primaryArray), second, + Set mergedAndOrdered = new TreeSet<>(Arrays.asList(deepCopyPrimaryArray)); + for (Object second : deepCopySecondaryArray) { + if (!EMCollectionUtils.ifValueAlreadyExistsInList(Arrays.asList(deepCopyPrimaryArray), second, doSloppyMatch(field.getName()))) { mergedAndOrdered.add(second); } } - return mergedAndOrdered.toArray(Arrays.copyOf(primaryArray, 0)); + return mergedAndOrdered.toArray(Arrays.copyOf(deepCopyPrimaryArray, 0)); + } + + private Object deepCopyOfObject(Object obj) throws EntityUpdateException { + if(obj==null) { + return obj; + } + + if(isStringOrPrimitive(obj.getClass())) { + return obj; + } else if(obj instanceof WebResource) { + return new WebResource((WebResource) obj); + } else if (obj instanceof Address) { + return new Address((Address) obj); + } else if (obj instanceof Country) { + return new Country((Country) obj); + } else { + throw new EntityUpdateException("Metadata consolidation failed due to unknown object type!"); + } } + private Object[] deepCopyOfArray(Object[] input) throws EntityUpdateException { + if(input==null || input.length==0) { + return new Object[0]; + } + + Object[] copy; + if(isStringOrPrimitive(input[0].getClass())) { + copy=input.clone(); + } + else { + copy = new Object [input.length]; + if(input[0] instanceof WebResource) { + for(int i=0;i deepCopyOfList(List input) throws EntityUpdateException { + if(input==null || input.size()==0) { + return new ArrayList<>(); + } + + List copy; + if(isStringOrPrimitive(input.get(0).getClass())) { + copy=new ArrayList(input); + } + else { + copy = new ArrayList<>(input.size()); + if(input.get(0) instanceof WebResource) { + for(int i=0;i deepCopyOfMap(Map input) throws EntityUpdateException { + if(input==null || input.size()==0) { + return new HashMap<>(); + } + + Map copy; + Object mapFirstKey = input.keySet().stream().findFirst().get(); + Object mapFirstValue = input.values().stream().findFirst().get(); + //if both keys and values are of primitive type, no need for deep copy + if(isStringOrPrimitive(mapFirstKey.getClass()) && isStringOrPrimitive(mapFirstValue.getClass())) { + copy=new HashMap<>(input); + } + else { + copy=new HashMap<>(input.size()); + for(Map.Entry entry : input.entrySet()) { + Object keyDeepCopy = null; + Object valueDeepCopy = null; + if(List.class.isAssignableFrom(mapFirstKey.getClass())) { + keyDeepCopy = deepCopyOfList((List) entry.getKey()); + } else { + keyDeepCopy = deepCopyOfObject(entry.getKey()); + } + + if(List.class.isAssignableFrom(mapFirstValue.getClass())) { + valueDeepCopy = deepCopyOfList((List) entry.getValue()); + } else { + valueDeepCopy = deepCopyOfObject(entry.getValue()); + } + copy.put(keyDeepCopy, valueDeepCopy); + } + } + + return copy; + + } + public void dropRepository() { this.entityRecordRepository.dropCollection(); } From 240d7086e39722c34370503513326547e21198c9 Mon Sep 17 00:00:00 2001 From: SrdjanStevanetic Date: Thu, 7 Dec 2023 13:42:42 +0100 Subject: [PATCH 07/22] method combineEntities logic more clear (primary entity has precedence over secondary entity) --- .../web/service/EntityRecordService.java | 51 ++++++++----------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java index 81fdc2c1e..bd7117457 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java @@ -786,23 +786,22 @@ private Entity combineEntities(Entity primary, Entity secondary, List fie Object fieldValuePrimaryObjectPrimitiveOrString = primary.getFieldValue(field); Object fieldValueSecondaryObjectPrimitiveOrString = secondary.getFieldValue(field); - if (fieldValuePrimaryObjectPrimitiveOrString == null - && fieldValueSecondaryObjectPrimitiveOrString != null) { - consolidatedEntity.setFieldValue(field, fieldValueSecondaryObjectPrimitiveOrString); - } else if (fieldValuePrimaryObjectPrimitiveOrString != null) { + if (fieldValuePrimaryObjectPrimitiveOrString != null) { consolidatedEntity.setFieldValue(field, fieldValuePrimaryObjectPrimitiveOrString); + } else if (fieldValueSecondaryObjectPrimitiveOrString != null) { + consolidatedEntity.setFieldValue(field, fieldValueSecondaryObjectPrimitiveOrString); } } else if (Date.class.isAssignableFrom(fieldType)) { Object fieldValuePrimaryObjectDate = primary.getFieldValue(field); Object fieldValueSecondaryObjectDate = secondary.getFieldValue(field); - if (fieldValuePrimaryObjectDate == null && fieldValueSecondaryObjectDate != null) { - consolidatedEntity.setFieldValue(field, - new Date(((Date) fieldValueSecondaryObjectDate).getTime())); - } else if (fieldValuePrimaryObjectDate != null) { + if (fieldValuePrimaryObjectDate != null) { consolidatedEntity.setFieldValue(field, new Date(((Date) fieldValuePrimaryObjectDate).getTime())); + } else if (fieldValueSecondaryObjectDate != null) { + consolidatedEntity.setFieldValue(field, + new Date(((Date) fieldValueSecondaryObjectDate).getTime())); } } else if (fieldType.isArray()) { Object[] mergedArray = mergeArrays(primary, secondary, field, accumulate); @@ -835,11 +834,11 @@ private void mergeCustomObjects(Entity primary, Entity secondary, Field field, Entity consolidatedEntity) throws IllegalAccessException, IllegalArgumentException, EntityUpdateException { Object primaryObj = primary.getFieldValue(field); Object secondaryObj = secondary.getFieldValue(field); - if (primaryObj == null && secondaryObj != null) { - consolidatedEntity.setFieldValue(field, deepCopyOfObject(secondaryObj)); - } else if (primaryObj != null) { + if (primaryObj != null) { consolidatedEntity.setFieldValue(field, deepCopyOfObject(primaryObj)); - } + } else if (secondaryObj != null) { + consolidatedEntity.setFieldValue(field, deepCopyOfObject(secondaryObj)); + } } boolean isStringOrPrimitive(Class fieldType) { @@ -865,11 +864,7 @@ void combineEntities(Entity consolidatedEntity, Entity primary, Entity secondary Map fieldValuePrimaryObject = deepCopyOfMap(fieldValuePrimaryObjectMap); Map fieldValueSecondaryObject = deepCopyOfMap(fieldValueSecondaryObjectMap); - if (CollectionUtils.isEmpty(fieldValuePrimaryObject) - && !CollectionUtils.isEmpty(fieldValueSecondaryObject)) { - fieldValuePrimaryObject.putAll(fieldValueSecondaryObject); - - } else if (!CollectionUtils.isEmpty(fieldValuePrimaryObject) + if (!CollectionUtils.isEmpty(fieldValuePrimaryObject) && !CollectionUtils.isEmpty(fieldValueSecondaryObject) && accumulate) { for (Map.Entry elemSecondary : fieldValueSecondaryObject.entrySet()) { Object key = elemSecondary.getKey(); @@ -880,10 +875,12 @@ void combineEntities(Entity consolidatedEntity, Entity primary, Entity secondary mergePrimarySecondaryListWitoutDuplicates(fieldValuePrimaryObject, key, elemSecondary, fieldName, prefLabelsForAltLabels); } - } - if (!CollectionUtils.isEmpty(fieldValuePrimaryObject)) { consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); - } + } else if (!CollectionUtils.isEmpty(fieldValuePrimaryObject)) { + consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); + } else if (!CollectionUtils.isEmpty(fieldValueSecondaryObject)) { + consolidatedEntity.setFieldValue(field, fieldValueSecondaryObject); + } } /** @@ -994,22 +991,16 @@ void mergeList(Entity consolidatedEntity, List fieldValuePrimaryObjectLi List fieldValuePrimaryObject = deepCopyOfList(fieldValuePrimaryObjectList); List fieldValueSecondaryObject = deepCopyOfList(fieldValueSecondaryObjectList); - if (!CollectionUtils.isEmpty(fieldValuePrimaryObject) && !CollectionUtils.isEmpty(fieldValueSecondaryObject)) { - if (accumulate) { + if (!CollectionUtils.isEmpty(fieldValuePrimaryObject) && !CollectionUtils.isEmpty(fieldValueSecondaryObject) + && accumulate) { for (Object secondaryObjectListObject : fieldValueSecondaryObject) { addToPrimaryList(field, fieldValuePrimaryObject, secondaryObjectListObject); } consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); - } else { - consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); - } - return; - } else if (!CollectionUtils.isEmpty(fieldValueSecondaryObject)) { - consolidatedEntity.setFieldValue(field, fieldValueSecondaryObject); - return; } else if (!CollectionUtils.isEmpty(fieldValuePrimaryObject)) { consolidatedEntity.setFieldValue(field, fieldValuePrimaryObject); - return; + } else if (!CollectionUtils.isEmpty(fieldValueSecondaryObject)) { + consolidatedEntity.setFieldValue(field, fieldValueSecondaryObject); } } From c5736b86c1f91d48e128947e99c40284057c1754 Mon Sep 17 00:00:00 2001 From: SrdjanStevanetic Date: Sun, 10 Dec 2023 23:52:26 +0100 Subject: [PATCH 08/22] small code comments --- development_tips.txt | 4 ++-- .../web/service/EntityRecordService.java | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/development_tips.txt b/development_tips.txt index 209abcb95..d2ecca280 100644 --- a/development_tips.txt +++ b/development_tips.txt @@ -16,6 +16,6 @@ public ZohoDereferenceService(ZohoAccessConfiguration zohoAccessConfiguration, The printed response can then be saved to a file in test/resources. Also make sure to adjust the fields to be either string or list type (no map). -2. Whenever introducing new types for a field for any of the model definition classes, e.g. Organization, make sure to properly hanlde the merging of the entities in the method combineEntities. -Please see the java doc on that method. Also override the equals() method for the new field type and add it to the EntityFieldsTypes class. +2. Whenever introducing new custom types for a field for any of the model definition classes, e.g. Organization, make sure to properly hanlde the merging of the entities in the method combineEntities(). +Please also override the equals() method for the new field type and add it to the EntityFieldsTypes class. diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java index bd7117457..c6052554b 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java @@ -754,7 +754,7 @@ public void replaceEuropeanaProxy(final Entity updateRequestEntity, EntityRecord /** * Merges metadata between two entities. This method performs a deep copy of the objects, - * for the mutable (custom) filed types. + * for the mutable (custom) field types. * * @param primary Primary entity. Metadata from this entity takes precedence * @param secondary Secondary entity. Metadata from this entity is only used if no matching field @@ -1060,7 +1060,8 @@ private Object deepCopyOfObject(Object obj) throws EntityUpdateException { } else if (obj instanceof Country) { return new Country((Country) obj); } else { - throw new EntityUpdateException("Metadata consolidation failed due to unknown object type!"); + throw new EntityUpdateException("Metadata consolidation failed due to unknown object type! When defining" + + "a new custom field type, please also override the equals() method."); } } @@ -1091,7 +1092,8 @@ else if(input[0] instanceof Country) { } } else { - throw new EntityUpdateException("Metadata consolidation failed due to unknown object type in array!"); + throw new EntityUpdateException("Metadata consolidation failed due to unknown object type in array! " + + "When defining a new custom field type, please also override the equals() method."); } } return copy; @@ -1124,7 +1126,8 @@ else if(input.get(0) instanceof Country) { } } else { - throw new EntityUpdateException("Metadata consolidation failed due to unknown object type in list!"); + throw new EntityUpdateException("Metadata consolidation failed due to unknown object type in list! " + + "When defining a new custom field type, please also override the equals() method."); } } return copy; From 441d68162107865dbf45f05c45829e79c1bd9414 Mon Sep 17 00:00:00 2001 From: SrdjanStevanetic Date: Thu, 21 Dec 2023 14:04:35 +0100 Subject: [PATCH 09/22] 1) deep copy of objects during entity merging; 2) mapping of zoho country to europeana place; 3) lazy retrieval of the country reference --- .../config/EntityManagementConfiguration.java | 7 + .../definitions/model/Country.java | 79 ---- .../definitions/model/CountryMapping.java | 64 ++++ .../definitions/model/Entity.java | 18 +- .../definitions/model/EntityProxy.java | 5 +- .../definitions/model/EntityRecord.java | 20 +- .../definitions/model/Organization.java | 76 +++- .../definitions/model/Place.java | 23 ++ .../definitions/model/WebResource.java | 1 + .../vocabulary/EntityFieldsTypes.java | 8 +- .../vocabulary/EntityProfile.java | 3 +- .../vocabulary/OrganizationSolrFields.java | 2 + .../vocabulary/WebEntityFields.java | 7 + .../web/xml/model/XmlConstants.java | 2 + .../web/xml/model/XmlOrganizationImpl.java | 41 ++- .../solr/model/SolrOrganization.java | 23 +- .../AbstractIntegrationTest.java | 45 ++- .../service/DereferenceServiceIT.java | 3 +- .../testutils/TestConfig.java | 1 - .../web/BaseWebControllerTest.java | 3 +- .../web/EntityRegistrationIT.java | 11 +- .../web/EntityRetrievalIT.java | 39 +- .../resources/content/organization.json | 5 +- .../resources/solr-docker/conf/schema.xml | 4 + .../resources/zoho_country_mapping_test.json | 10 +- .../EntityConsolidationProcessor.java | 18 +- .../entitymanagement/config/AppConfig.java | 2 +- .../entitymanagement/web/BaseRest.java | 4 + .../entitymanagement/web/EMController.java | 68 ++-- .../web/service/BaseZohoAccess.java | 10 +- .../web/service/EntityRecordService.java | 134 ++++--- .../web/service/ZohoSyncService.java | 7 +- .../resources/entitymanagement.properties | 4 + .../entitymanagement.user.properties.template | 5 +- .../main/resources/zoho_country_mapping.json | 337 ++++++++++++++++++ .../resources/zoho_country_mapping_local.json | 7 + .../zoho/organization/ZohoConfiguration.java | 7 - .../organization/ZohoDereferenceService.java | 6 +- .../ZohoOrganizationConverter.java | 87 +---- .../main/resources/zoho_country_mapping.json | 3 - .../resources/zoho_import.properties.template | 1 - 41 files changed, 822 insertions(+), 378 deletions(-) delete mode 100644 entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Country.java create mode 100644 entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/CountryMapping.java create mode 100644 entity-management-web/src/main/resources/zoho_country_mapping.json create mode 100644 entity-management-web/src/main/resources/zoho_country_mapping_local.json delete mode 100644 entity-management-zoho/src/main/resources/zoho_country_mapping.json diff --git a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java index d0ebcaf80..1bc057764 100644 --- a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java +++ b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java @@ -148,6 +148,9 @@ public class EntityManagementConfiguration implements InitializingBean { @Value("${spring.profiles.active:}") private String activeProfileString; + + @Value("${zoho.country.mapping.file:#{null}}") + private String zohoCountryMappingFile; public EntityManagementConfiguration() { LOG.info("Initializing EntityManagementConfiguration bean as: configuration"); @@ -357,4 +360,8 @@ public boolean isGenerateOrganizationEuropeanaId() { return generateOrganizationEuropeanaId; } + public String getZohoCountryMappingFile() { + return zohoCountryMappingFile; + } + } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Country.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Country.java deleted file mode 100644 index f77d48133..000000000 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Country.java +++ /dev/null @@ -1,79 +0,0 @@ -package eu.europeana.entitymanagement.definitions.model; - -import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.CONTEXT; -import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ID; -import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.PREF_LABEL; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import com.fasterxml.jackson.annotation.JsonGetter; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import com.fasterxml.jackson.annotation.JsonSetter; -import dev.morphia.annotations.Embedded; -import eu.europeana.entitymanagement.vocabulary.WebEntityFields; - -@Embedded -@JsonInclude(value = JsonInclude.Include.NON_EMPTY) -@JsonPropertyOrder({ - CONTEXT, - ID, - PREF_LABEL -}) -public class Country { - - public Country() { - super(); - } - - public Country(Country copy) { - super(); - this.id = copy.getId(); - if(copy.getPrefLabel()!=null) { - this.prefLabel=new HashMap<>(copy.getPrefLabel()); - } - } - - private String id; - private Map prefLabel; - - @JsonSetter(ID) - public void setId(String id) { - this.id = id; - } - - @JsonGetter(ID) - public String getId() { - return id; - } - - @JsonGetter(WebEntityFields.PREF_LABEL) - public Map getPrefLabel() { - return prefLabel; - } - - @JsonSetter(WebEntityFields.PREF_LABEL) - public void setPrefLabel(Map prefLabel) { - this.prefLabel = prefLabel; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Country that = (Country) o; - - if (!Objects.equals(id, that.getId())) return false; - return Objects.equals(prefLabel, that.getPrefLabel()); - } - - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((id == null) ? 0 : id.hashCode()); - result = prime * result + ((prefLabel == null) ? 0 : prefLabel.hashCode()); - return result; - } - -} diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/CountryMapping.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/CountryMapping.java new file mode 100644 index 000000000..4f12b0b66 --- /dev/null +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/CountryMapping.java @@ -0,0 +1,64 @@ +package eu.europeana.entitymanagement.definitions.model; + +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ENTITY_URI; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.WIKIDATA_URI; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ZOHO_LABEL; +import java.util.Arrays; +import java.util.List; +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonSetter; + +@JsonInclude(value = JsonInclude.Include.NON_EMPTY) +@JsonPropertyOrder({ + ZOHO_LABEL, + ENTITY_URI, + WIKIDATA_URI +}) +public class CountryMapping { + + private String zohoLabel; + private String entityUri; + private String wikidataUri; + + @JsonGetter(ZOHO_LABEL) + public String getZohoLabel() { + return zohoLabel; + } + + @JsonSetter(ZOHO_LABEL) + public void setZohoLabel(String zohoLabel) { + this.zohoLabel = zohoLabel; + } + + @JsonGetter(ENTITY_URI) + public String getEntityUri() { + return entityUri; + } + + @JsonSetter(ENTITY_URI) + public void setEntityUri(String entityUri) { + this.entityUri = entityUri; + } + + @JsonGetter(WIKIDATA_URI) + public String getWikidataUri() { + return wikidataUri; + } + + @JsonSetter(WIKIDATA_URI) + public void setWikidataUri(String wikidataUri) { + this.wikidataUri = wikidataUri; + } + + public static String getEntityUriFromName(List list ,String name) { + for(CountryMapping cm : list) { + List splittedAndTrimmed = Arrays.stream(cm.getZohoLabel().split(",")).map(String::trim).toList(); + if(splittedAndTrimmed.contains(name)) { + return cm.getEntityUri(); + } + } + return null; + } +} diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java index f85de2c0f..0163311cc 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java @@ -14,7 +14,13 @@ import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.IS_SHOWN_BY; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.NOTE; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.TYPE; - +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -31,13 +37,6 @@ import eu.europeana.entitymanagement.normalization.EntityFieldsEuropeanaProxyValidationInterface; import eu.europeana.entitymanagement.vocabulary.ValidationObject; import eu.europeana.entitymanagement.vocabulary.WebEntityFields; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; @dev.morphia.annotations.Embedded @JsonIgnoreProperties(ignoreUnknown = true) @@ -299,4 +298,7 @@ public void addSameReferenceLink(String uri) { getSameReferenceLinks().add(uri); } } + + public void dereference() { + } } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/EntityProxy.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/EntityProxy.java index 8c3631c33..4d389d886 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/EntityProxy.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/EntityProxy.java @@ -6,13 +6,13 @@ import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.PROXY_FOR; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.PROXY_IN; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.TYPE; - import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonSetter; import dev.morphia.annotations.Embedded; +import eu.europeana.entitymanagement.utils.EntityObjectFactory; @Embedded @JsonInclude(value = JsonInclude.Include.NON_EMPTY) @@ -24,6 +24,9 @@ public class EntityProxy { String proxyFor; Aggregation proxyIn; String type; + + public EntityProxy() { + } @JsonGetter(TYPE) public String getType() { diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/EntityRecord.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/EntityRecord.java index ab113a0f7..8c8ebf207 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/EntityRecord.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/EntityRecord.java @@ -4,7 +4,11 @@ import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_SAME_AS; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.BASE_DATA_EUROPEANA_URI; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ID; - +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; +import org.bson.types.ObjectId; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; @@ -18,11 +22,6 @@ import dev.morphia.annotations.Indexes; import eu.europeana.entitymanagement.utils.EntityRecordWatcher; import eu.europeana.entitymanagement.vocabulary.WebEntityFields; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.stream.Collectors; -import org.bson.types.ObjectId; @JsonInclude(value = JsonInclude.Include.NON_EMPTY) @dev.morphia.annotations.Entity("EntityRecord") @@ -44,6 +43,9 @@ public class EntityRecord { @JsonIgnore private Date created; @JsonIgnore private Date modified; + + public EntityRecord() { + } @JsonGetter public Entity getEntity() { @@ -69,7 +71,7 @@ public void setEntityId(String entityId) { public List getProxies() { return proxies; } - + @JsonSetter public void addProxy(EntityProxy proxy) { this.proxies.add(proxy); @@ -87,6 +89,10 @@ public boolean isDisabled() { return disabled != null; } + public Date getDisabled() { + return disabled; + } + public void setDisabled(Date disabledParam) { this.disabled = disabledParam; } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java index 2c4a17268..f3e971152 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java @@ -4,6 +4,8 @@ import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ALT_LABEL; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.CONTEXT; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.COUNTRY; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.COUNTRY_ID; +import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.COUNTRY_PLACE; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.DEPICTION; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.DESCRIPTION; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.EUROPEANA_ROLE; @@ -25,9 +27,11 @@ import java.util.List; import java.util.Map; import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonPropertyOrder; import com.fasterxml.jackson.annotation.JsonSetter; +import dev.morphia.annotations.Reference; import eu.europeana.entitymanagement.vocabulary.EntityTypes; /** This class defines base organization type of an entity. */ @@ -45,6 +49,8 @@ FOAF_LOGO, EUROPEANA_ROLE, COUNTRY, + COUNTRY_ID, + COUNTRY_PLACE, LANGUAGE, FOAF_HOMEPAGE, FOAF_PHONE, @@ -63,7 +69,13 @@ public class Organization extends Entity { private List phone; private List mbox; private Map> europeanaRole; - private Country country; + + @Reference(lazy = true) + private EntityRecord countryRef; + private Place countryPlace; + private String country; + private String countryId; + private Address hasAddress; private List sameAs; private List language; @@ -82,7 +94,10 @@ public Organization(Organization copy) { if (copy.getMbox() != null) this.mbox = new ArrayList<>(copy.getMbox()); if (copy.getEuropeanaRole() != null) this.europeanaRole = new HashMap<>(copy.getEuropeanaRole()); - if (copy.getCountry() != null) this.country = new Country(copy.getCountry()); + //because the countryRef is a reference to the object we keep it the same + this.countryRef=copy.getCountryRef(); + + if (copy.getAddress() != null) this.hasAddress = new Address(copy.getAddress()); if (copy.sameAs != null) this.sameAs = (new ArrayList<>(copy.sameAs)); if (copy.language != null) this.language = (new ArrayList<>(copy.language)); @@ -138,16 +153,6 @@ public void setMbox(List mbox) { this.mbox = mbox; } - @JsonGetter(COUNTRY) - public Country getCountry() { - return country; - } - - @JsonSetter(COUNTRY) - public void setCountry(Country country) { - this.country = country; - } - @JsonGetter(HAS_ADDRESS) public Address getAddress() { return hasAddress; @@ -215,4 +220,51 @@ public List getLanguage() { public void setLanguage(List edmLanguage) { this.language = edmLanguage; } + + @JsonIgnore + public EntityRecord getCountryRef() { + return countryRef; + } + + public void setCountryRef(EntityRecord countryRef) { + this.countryRef=countryRef; + } + + @JsonGetter(COUNTRY_PLACE) + public Place getCountryPlace() { + return countryPlace; + } + + @JsonSetter(COUNTRY_PLACE) + public void setCountryPlace(Place countryPlace) { + this.countryPlace = countryPlace; + } + + @JsonGetter(COUNTRY) + public String getCountry() { + return country; + } + + @JsonSetter(COUNTRY) + public void setCountry(String country) { + this.country = country; + } + + @JsonGetter(COUNTRY_ID) + public String getCountryId() { + return countryId; + } + + @JsonSetter(COUNTRY_ID) + public void setCountryId(String countryId) { + this.countryId = countryId; + } + + @Override + public void dereference() { + if(this.getCountryRef()!=null) { + this.countryPlace=(Place) this.getCountryRef().getEntity(); + } + } + } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Place.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Place.java index 98aaffa44..ad2d09da2 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Place.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Place.java @@ -29,6 +29,7 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; +import java.util.Objects; @JsonInclude(value = JsonInclude.Include.NON_EMPTY) @JsonPropertyOrder({ @@ -146,4 +147,26 @@ public void setFieldValue(Field field, Object value) // method to call the setter for each field individually field.set(this, value); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Place that = (Place) o; + + if (!Objects.equals(latitude, that.getLatitude())) return false; + if (!Objects.equals(longitude, that.getLongitude())) return false; + return Objects.equals(altitude, that.getAltitude()); + } + + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((latitude == null) ? 0 : latitude.hashCode()); + result = prime * result + ((longitude == null) ? 0 : longitude.hashCode()); + result = prime * result + ((altitude == null) ? 0 : altitude.hashCode()); + return result; + } + } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/WebResource.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/WebResource.java index d71e7703b..f59d12b99 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/WebResource.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/WebResource.java @@ -94,4 +94,5 @@ public int hashCode() { result = prime * result + ((source == null) ? 0 : source.hashCode()); return result; } + } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java index c07678e92..15f59620c 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityFieldsTypes.java @@ -101,7 +101,10 @@ public enum EntityFieldsTypes { logo(EntityFieldsTypes.FIELD_TYPE_WEB_RESOURCE, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), europeanaRole( EntityFieldsTypes.FIELD_TYPE_KEYWORD, true, EntityFieldsTypes.FIELD_CARDINALITY_0_INFINITE), - country(EntityFieldsTypes.FIELD_TYPE_COUNTRY, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), + country(EntityFieldsTypes.FIELD_TYPE_TEXT, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), + countryId(EntityFieldsTypes.FIELD_TYPE_URI, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), + countryPlace(EntityFieldsTypes.FIELD_TYPE_PLACE, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), + countryRef(EntityFieldsTypes.FIELD_TYPE_ENTITY_RECORD, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), homepage(EntityFieldsTypes.FIELD_TYPE_URI, false, EntityFieldsTypes.FIELD_CARDINALITY_0_1), phone( EntityFieldsTypes.FIELD_TYPE_KEYWORD, false, EntityFieldsTypes.FIELD_CARDINALITY_0_INFINITE), @@ -137,7 +140,8 @@ public enum EntityFieldsTypes { public static final String FIELD_TYPE_DATE_OR_URI = "Date or URI"; public static final String FIELD_TYPE_WEB_RESOURCE = "WebResource"; public static final String FIELD_TYPE_ADDRESS = "Address"; - public static final String FIELD_TYPE_COUNTRY = "Country"; + public static final String FIELD_TYPE_PLACE = "Place"; + public static final String FIELD_TYPE_ENTITY_RECORD = "EntityRecord"; public static final String FIELD_CARDINALITY_1_1 = "1..1"; public static final String FIELD_CARDINALITY_0_1 = "0..1"; diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityProfile.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityProfile.java index a1b973418..4e495153b 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityProfile.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/EntityProfile.java @@ -3,5 +3,6 @@ public enum EntityProfile { internal, external, - debug; + debug, + dereference; } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java index cb58c3b46..79d1848c2 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java @@ -19,6 +19,8 @@ public interface OrganizationSolrFields extends EntitySolrFields { public static final String COUNTRY_ID = "countryId"; public static final String COUNTRY_PREF_LABEL_ALL = "countryPrefLabel.*"; public static final String COUNTRY_PREF_LABEL = "countryPrefLabel"; + public static final String COUNTRY_LAT = "countryLatitude"; + public static final String COUNTRY_LONG = "countryLongitude"; public static final String VCARD_HAS_ADDRESS = "vcard_hasAddress.1"; public static final String VCARD_STREET_ADDRESS = "vcard_streetAddress.1"; public static final String VCARD_LOCALITY = "vcard_locality.1"; diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/WebEntityFields.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/WebEntityFields.java index 7c290e3b1..079ef84b9 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/WebEntityFields.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/WebEntityFields.java @@ -89,6 +89,8 @@ public interface WebEntityFields { // Organization fields public static final String DESCRIPTION = "description"; public static final String ACRONYM = "acronym"; + public static final String COUNTRY_ID = "countryId"; + public static final String COUNTRY_PLACE = "countryPlace"; public static final String COUNTRY = "country"; public static final String EUROPEANA_ROLE = "europeanaRole"; public static final String FOAF_LOGO = "logo"; @@ -124,4 +126,9 @@ public interface WebEntityFields { public static final String PROXY_FOR = "proxyFor"; public static final String PROXY_IN = "proxyIn"; public static final String PROXY = "Proxy"; + + //CountryMapping fields + public static final String ZOHO_LABEL="Zoho Label"; + public static final String ENTITY_URI="Entity URI"; + public static final String WIKIDATA_URI="Wikidata URI"; } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlConstants.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlConstants.java index b2d15fafd..93702733d 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlConstants.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlConstants.java @@ -83,6 +83,8 @@ public final class XmlConstants extends XmlFields { public static final String XML_LOGO = "logo"; public static final String XML_EUROPEANA_ROLE = "europeanaRole"; public static final String XML_COUNTRY = "country"; + public static final String XML_COUNTRY_ID = "countryId"; + public static final String XML_COUNTRY_PLACE = "countryPlace"; public static final String XML_HOMEPAGE = "homepage"; public static final String XML_PHONE = "phone"; public static final String XML_MBOX = "mbox"; diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java index b1950cc83..b4a82072f 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java @@ -13,6 +13,8 @@ import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.PREF_LABEL; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_ACRONYM; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_COUNTRY; +import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_COUNTRY_ID; +import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_COUNTRY_PLACE; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_DESCRIPTION; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_EUROPEANA_ROLE; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_HAS_ADDRESS; @@ -25,9 +27,7 @@ import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_PHONE; import static eu.europeana.entitymanagement.web.xml.model.XmlConstants.XML_SAME_AS; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; @@ -35,7 +35,6 @@ import javax.xml.bind.annotation.XmlType; import org.apache.commons.collections.CollectionUtils; import eu.europeana.entitymanagement.definitions.exceptions.EntityModelCreationException; -import eu.europeana.entitymanagement.definitions.model.Country; import eu.europeana.entitymanagement.definitions.model.Organization; import eu.europeana.entitymanagement.vocabulary.EntityTypes; @@ -54,6 +53,8 @@ XML_LOGO, XML_EUROPEANA_ROLE, XML_COUNTRY, + XML_COUNTRY_ID, + XML_COUNTRY_PLACE, XML_LANGUAGE, XML_HOMEPAGE, XML_PHONE, @@ -82,7 +83,13 @@ public class XmlOrganizationImpl extends XmlBaseEntityImpl { @XmlElement(namespace = NAMESPACE_EDM, name = XML_COUNTRY) private String country; - + + @XmlElement(namespace = NAMESPACE_EDM, name = XML_COUNTRY_ID) + private String countryId; + + @XmlElement(namespace = NAMESPACE_EDM, name = XML_COUNTRY_PLACE) + private XmlPlaceImpl countryPlace; + @XmlElement(namespace = NAMESPACE_FOAF, name = XML_HOMEPAGE) private LabelledResource homepage; @@ -112,9 +119,11 @@ public XmlOrganizationImpl(Organization organization) { this.europeanaRole = RdfXmlUtils.convertToXmlMultilingualString(organization.getEuropeanaRole()); - if(organization.getCountry()!=null && organization.getCountry().getPrefLabel()!=null) { - this.country = organization.getCountry().getPrefLabel().get("en"); - } + this.country = organization.getCountry(); + this.countryId = organization.getCountryId(); + if(organization.getCountryPlace()!=null) { + this.countryPlace=new XmlPlaceImpl(organization.getCountryPlace()); + } if (organization.getHomepage() != null) { this.homepage = new LabelledResource(organization.getHomepage()); @@ -144,12 +153,10 @@ public Organization toEntityModel() throws EntityModelCreationException { entity.setLogo(XmlWebResourceWrapper.toWebResource(getLogo())); entity.setEuropeanaRole(RdfXmlUtils.toLanguageMapList(getEuropeanaRole())); - if(country!=null) { - Country orgCountry = new Country(); - Map countryPrefLabel = new HashMap<>(); - countryPrefLabel.put("en", country); - orgCountry.setPrefLabel(countryPrefLabel); - entity.setCountry(orgCountry); + entity.setCountry(getCountry()); + entity.setCountryId(getCountryId()); + if(getCountryPlace()!=null) { + entity.setCountryPlace(getCountryPlace().toEntityModel()); } if (getHomepage() != null) { @@ -229,4 +236,12 @@ public void setSameReferenceLinks(List uris) { public List getLanguage() { return language; } + + public String getCountryId() { + return countryId; + } + + public XmlPlaceImpl getCountryPlace() { + return countryPlace; + } } diff --git a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java index 0a382c203..513dff32b 100644 --- a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java +++ b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java @@ -8,6 +8,7 @@ import org.apache.solr.client.solrj.beans.Field; import eu.europeana.entitymanagement.definitions.model.Address; import eu.europeana.entitymanagement.definitions.model.Organization; +import eu.europeana.entitymanagement.definitions.model.Place; import eu.europeana.entitymanagement.solr.SolrUtils; import eu.europeana.entitymanagement.utils.EntityUtils; import eu.europeana.entitymanagement.vocabulary.EntitySolrFields; @@ -45,6 +46,12 @@ public class SolrOrganization extends SolrEntity { @Field(OrganizationSolrFields.COUNTRY_PREF_LABEL_ALL) private Map countryPrefLabel; + @Field(OrganizationSolrFields.COUNTRY_LAT) + private Float countryLatitude; + + @Field(OrganizationSolrFields.COUNTRY_LONG) + private Float countryLongitude; + @Field(OrganizationSolrFields.VCARD_HAS_ADDRESS) private String hasAddress; @@ -86,9 +93,11 @@ public SolrOrganization(Organization organization) { if (organization.getMbox() != null) this.mbox = new ArrayList<>(organization.getMbox()); setEuropeanaRole(organization.getEuropeanaRole()); - if(organization.getCountry()!=null) { - this.countryId = organization.getCountry().getId(); - this.setCountryPrefLabel(organization.getCountry().getPrefLabel()); + if(organization.getCountryRef()!=null) { + this.countryId = organization.getCountryRef().getEntity().getEntityId(); + this.setCountryPrefLabel(organization.getCountryRef().getEntity().getPrefLabel()); + this.countryLatitude = ((Place) organization.getCountryRef().getEntity()).getLatitude(); + this.countryLongitude = ((Place) organization.getCountryRef().getEntity()).getLongitude(); } if (organization.getSameReferenceLinks() != null) { @@ -218,4 +227,12 @@ public String getCountryId() { public Map getCountryPrefLabel() { return countryPrefLabel; } + + public Float getCountryLatitude() { + return countryLatitude; + } + + public Float getCountryLongitude() { + return countryLongitude; + } } diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java index 7017e1486..7d24b1784 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java @@ -1,5 +1,26 @@ package eu.europeana.entitymanagement; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Objects; +import java.util.Optional; +import javax.xml.bind.JAXBContext; +import org.apache.commons.io.IOUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.testcontainers.containers.output.ToStringConsumer; +import org.testcontainers.containers.output.WaitingConsumer; import com.fasterxml.jackson.databind.ObjectMapper; import eu.europeana.entitymanagement.batch.service.EntityUpdateService; import eu.europeana.entitymanagement.common.config.DataSource; @@ -19,32 +40,10 @@ import eu.europeana.entitymanagement.web.service.EntityRecordService; import eu.europeana.entitymanagement.web.xml.model.XmlBaseEntityImpl; import eu.europeana.entitymanagement.zoho.organization.ZohoConfiguration; -import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.Objects; -import java.util.Optional; -import javax.xml.bind.JAXBContext; import okhttp3.mockwebserver.Dispatcher; import okhttp3.mockwebserver.MockResponse; import okhttp3.mockwebserver.MockWebServer; import okhttp3.mockwebserver.RecordedRequest; -import org.apache.commons.io.IOUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.testcontainers.containers.output.ToStringConsumer; -import org.testcontainers.containers.output.WaitingConsumer; @ComponentScan(basePackageClasses = EntityManagementBasePackageMapper.class) @AutoConfigureMockMvc @@ -66,7 +65,6 @@ public abstract class AbstractIntegrationTest { @Autowired protected ConceptSchemeService emConceptSchemeService; @Autowired protected EntityManagementConfiguration emConfig; @Autowired protected ZohoConfiguration zohoConfiguration; - @Autowired protected ZohoOrganizationConverter zohoOrgConverter; static { MONGO_CONTAINER = @@ -149,6 +147,7 @@ static void setProperties(DynamicPropertyRegistry registry) { //tests must not register organizations as this is updating the zoho //generate Europeana ID can be set to true when using the mock service, see TextConfig class registry.add("zoho.generate.organization.europeanaid", () -> true); + registry.add("zoho.country.mapping.file", () -> "/zoho_country_mapping_test.json"); // could be used to fix eclipse issues registry.add("scmBranch", () -> "dev"); diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java index 7a17488c6..9e40f1bf2 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java @@ -20,6 +20,7 @@ import eu.europeana.entitymanagement.testutils.IntegrationTestUtils; import eu.europeana.entitymanagement.testutils.TestConfig; import eu.europeana.entitymanagement.web.service.DereferenceServiceLocator; +import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter; import eu.europeana.entitymanagement.zoho.utils.ZohoConstants; import eu.europeana.entitymanagement.zoho.utils.ZohoException; @@ -130,7 +131,7 @@ public void zohoOrganizationDereferenceLabelsTest() throws Exception { // choice = new Choice("EN"); // record.addKeyValue(ZohoConstants.LANG_ALTERNATIVE_FIELD + "_4", choice); - Organization org = zohoOrgConverter.convertToOrganizationEntity(record); + Organization org = ZohoOrganizationConverter.convertToOrganizationEntity(record, zohoConfiguration.getZohoBaseUrl()); Assertions.assertEquals(2, org.getPrefLabel().size()); Assertions.assertEquals(1, org.getAltLabel().size()); diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java index cdc11eb55..145a45dff 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java @@ -29,7 +29,6 @@ public ZohoConfiguration configureZoho() throws Exception { ZohoAccessClient zohoClient = Mockito.mock(ZohoAccessClient.class); Mockito.when(zohoConfiguration.getZohoAccessClient()).thenReturn(zohoClient); Mockito.when(zohoConfiguration.getZohoBaseUrl()).thenReturn(MOCK_ZOHO_BASE_URL); - Mockito.when(zohoConfiguration.getZohoCountryMappingFile()).thenReturn(MOCK_ZOHO_COUNTRY_MAPPING_FILE); // find matching JSON file based on zohoId argument, then create a Record object for it Mockito.doAnswer( diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java index 8c9950fc9..3e124ac6b 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java @@ -34,6 +34,7 @@ import eu.europeana.entitymanagement.testutils.IntegrationTestUtils; import eu.europeana.entitymanagement.testutils.TestConfig; import eu.europeana.entitymanagement.web.xml.model.XmlBaseEntityImpl; +import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter; @Import(TestConfig.class) abstract class BaseWebControllerTest extends AbstractIntegrationTest { @@ -171,7 +172,7 @@ protected EntityRecord createOrganization(String europeanaProxyEntityStr, Record Entity europeanaProxyEntity = objectMapper.readValue(europeanaProxyEntityStr, Entity.class); DataSource dataSource = datasources.verifyDataSource(europeanaProxyEntity.getEntityId(), false); Organization zohoOrganization = - zohoOrgConverter.convertToOrganizationEntity(zohoRecord); + ZohoOrganizationConverter.convertToOrganizationEntity(zohoRecord, zohoConfiguration.getZohoBaseUrl()); EntityRecord savedRecord = entityRecordService.createEntityFromRequest( europeanaProxyEntity, zohoOrganization, dataSource); diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java index a383f5964..30aafb182 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java @@ -226,14 +226,6 @@ public void registerZohoOrganizationShouldBeSuccessful() throws Exception { @Test public void registerZohoOrganizationGFMShouldBeSuccessful() throws Exception { - //1. create a place "Sweden" to be used to dereference zoho country for the zoho GFM org - mockMvc - .perform( - MockMvcRequestBuilders.post(IntegrationTestUtils.BASE_SERVICE_URL) - .content(loadFile(IntegrationTestUtils.PLACE_REGISTER_SWEDEN_JSON)) - .contentType(MediaType.APPLICATION_JSON_VALUE)); - - //2. register zoho GFM org ResultActions response = mockMvc.perform( MockMvcRequestBuilders.post(IntegrationTestUtils.BASE_SERVICE_URL) @@ -256,8 +248,7 @@ public void registerZohoOrganizationGFMShouldBeSuccessful() throws Exception { IntegrationTestUtils.ORGANIZATION_GFM_URI_WIKIDATA_URI))) .andExpect(jsonPath("$.prefLabel[*]", hasSize(2))) // should have Europeana, Zoho and Wikidata proxies - .andExpect(jsonPath("$.proxies", hasSize(3))) - .andExpect(jsonPath("$.country.prefLabel").isNotEmpty()); + .andExpect(jsonPath("$.proxies", hasSize(3))); } @Test diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java index ec1969b92..bdb0e1440 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java @@ -14,17 +14,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.xpath; - -import com.zoho.crm.api.record.Record; -import eu.europeana.entitymanagement.batch.service.FailedTaskService; -import eu.europeana.entitymanagement.definitions.batch.model.ScheduledUpdateType; -import eu.europeana.entitymanagement.definitions.model.EntityRecord; -import eu.europeana.entitymanagement.testutils.IntegrationTestUtils; -import eu.europeana.entitymanagement.vocabulary.EntityTypes; -import eu.europeana.entitymanagement.vocabulary.FailedTaskJsonFields; -import eu.europeana.entitymanagement.vocabulary.WebEntityConstants; -import eu.europeana.entitymanagement.vocabulary.WebEntityFields; -import eu.europeana.entitymanagement.web.xml.model.XmlConstants; import java.util.Map; import java.util.Optional; import org.hamcrest.Matchers; @@ -35,6 +24,16 @@ import org.springframework.http.MediaType; import org.springframework.test.web.servlet.ResultActions; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import com.zoho.crm.api.record.Record; +import eu.europeana.entitymanagement.batch.service.FailedTaskService; +import eu.europeana.entitymanagement.definitions.batch.model.ScheduledUpdateType; +import eu.europeana.entitymanagement.definitions.model.EntityRecord; +import eu.europeana.entitymanagement.testutils.IntegrationTestUtils; +import eu.europeana.entitymanagement.vocabulary.EntityTypes; +import eu.europeana.entitymanagement.vocabulary.FailedTaskJsonFields; +import eu.europeana.entitymanagement.vocabulary.WebEntityConstants; +import eu.europeana.entitymanagement.vocabulary.WebEntityFields; +import eu.europeana.entitymanagement.web.xml.model.XmlConstants; @SpringBootTest @AutoConfigureMockMvc @@ -461,8 +460,13 @@ public void retrievePlaceExternalSchemaOrgShouldBeSuccessful() throws Exception @Test public void retrieveOrganizationJsonExternalShouldBeSuccessful() throws Exception { - // id in JSON matches ORGANIZATION_BNF_URI_ZOHO value - String europeanaMetadata = loadFile(IntegrationTestUtils.ORGANIZATION_REGISTER_GFM_ZOHO_JSON); + //1. create a place "Sweden" to be used to dereference zoho country for the zoho GFM org + String europeanaMetadata = loadFile(IntegrationTestUtils.PLACE_REGISTER_SWEDEN_JSON); + String metisResponse = loadFile(IntegrationTestUtils.PLACE_SWEDEN_XML); + createEntity(europeanaMetadata, metisResponse, IntegrationTestUtils.PLACE_SWEDEN_URI); + + //2. register zoho GFM org + europeanaMetadata = loadFile(IntegrationTestUtils.ORGANIZATION_REGISTER_GFM_ZOHO_JSON); Optional zohoRecord = IntegrationTestUtils.getZohoOrganizationRecord( IntegrationTestUtils.ORGANIZATION_GFM_URI_ZOHO); @@ -474,12 +478,14 @@ public void retrieveOrganizationJsonExternalShouldBeSuccessful() throws Exceptio mockMvc .perform( get(IntegrationTestUtils.BASE_SERVICE_URL + "/" + requestPath + ".jsonld") - .param(WebEntityConstants.QUERY_PARAM_PROFILE, "external") + .param(WebEntityConstants.QUERY_PARAM_PROFILE, "external, dereference") .accept(MediaType.APPLICATION_JSON)) .andExpect(status().isOk()) .andExpect(jsonPath("$.id", is(entityId))) .andExpect(jsonPath("$.type", is(EntityTypes.Organization.getEntityType()))) - .andExpect(jsonPath("$.sameAs").isNotEmpty()); + .andExpect(jsonPath("$.sameAs").isNotEmpty()) + .andExpect(jsonPath("$.countryId").isNotEmpty()) + .andExpect(jsonPath("$.countryPlace").isNotEmpty()); } @Test @@ -533,7 +539,8 @@ public void retrieveOrganizationXmlExternalShouldBeSuccessful() throws Exception .andExpect(xpath(entityBaseXpath + "/@rdf:about", xmlNamespaces).string(entityId)) .andExpect(xpath(entityBaseXpath + "/@rdf:about", xmlNamespaces).string(entityId)) .andExpect( - xpath(entityBaseXpath + "/skos:prefLabel", xmlNamespaces).nodeCount(greaterThan(0))); + xpath(entityBaseXpath + "/skos:prefLabel", xmlNamespaces).nodeCount(greaterThan(0))) + .andExpect(xpath(entityBaseXpath + "/edm:countryPlace", xmlNamespaces).doesNotExist()); } @Test diff --git a/entity-management-tests/src/integration-test/resources/content/organization.json b/entity-management-tests/src/integration-test/resources/content/organization.json index 1ecd4fd6c..f58de26d7 100644 --- a/entity-management-tests/src/integration-test/resources/content/organization.json +++ b/entity-management-tests/src/integration-test/resources/content/organization.json @@ -81,10 +81,7 @@ "GLAM" ] }, - "country" : { - "_t" : "Country", - "id" : "http://data.europeana.eu/place/4" - }, + "country" : "Netherlands", "homepage": "https://www.naturalis.nl/nl/", "phone": [ "+31-71-751-9600" diff --git a/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml b/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml index 709bb9fd7..9bca1f7d1 100644 --- a/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml +++ b/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml @@ -513,6 +513,10 @@ multiValued="false" /> + + + + + + + + From c3561f94d3dd248cd753fda1868bbfff11e47549 Mon Sep 17 00:00:00 2001 From: GordeaS Date: Thu, 25 Jan 2024 10:46:55 +0100 Subject: [PATCH 14/22] fix to allow access to private fields of the parent class #EA-3641 --- .../europeana/entitymanagement/definitions/model/Entity.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java index 740b52e9a..c9db854ab 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java @@ -231,6 +231,9 @@ public void setIsShownBy(WebResource resource) { } public Object getFieldValue(Field field) throws IllegalAccessException { + if(!field.canAccess(this)) { + field.setAccessible(true); + } return field.get(this); } From fa8271448bfaf25c4c079739479bdc2d304cfe2d Mon Sep 17 00:00:00 2001 From: GordeaS Date: Wed, 31 Jan 2024 15:14:46 +0100 Subject: [PATCH 15/22] make fields accessible for refactoring, added draft implementation for zoho data generator #EA-3641 --- .../common/config/DataSource.java | 2 +- .../definitions/model/Entity.java | 4 ++ .../service/DereferenceServiceIT.java | 4 +- .../testutils/IntegrationTestUtils.java | 2 +- .../testutils/TestConfig.java | 6 +- .../testutils/ZohoRecordTestDeserializer.java | 8 ++- .../organization/ZohoTestDataGenerator.java | 59 +++++++++++++++++++ entity-management-web/.gitignore | 1 + .../entitymanagement/web/EMController.java | 4 +- .../service/DereferenceServiceLocator.java | 2 + .../src/main/resources/datasources.xml | 2 +- entity-management-zoho/pom.xml | 2 + .../zoho/ZohoAccessClient.java | 4 +- .../organization/ZohoDereferenceService.java | 11 ++-- 14 files changed, 96 insertions(+), 15 deletions(-) create mode 100644 entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java diff --git a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/DataSource.java b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/DataSource.java index bb859e6be..776cfb7dd 100644 --- a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/DataSource.java +++ b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/DataSource.java @@ -9,7 +9,7 @@ public class DataSource { public static final String FREQ_STATIC = "static"; public static final String EUROPEANA_ID = "europeana"; public static final String ZOHO_ID = "zoho-crm"; - public static final String ZOHO_HOST = "crm.zoho.com"; + public static final String ZOHO_HOST = "crm.zoho.eu"; @JacksonXmlProperty(isAttribute = true) private String url; diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java index c9db854ab..79177bba4 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Entity.java @@ -239,6 +239,10 @@ public Object getFieldValue(Field field) throws IllegalAccessException { public void setFieldValue(Field field, Object value) throws IllegalAccessException { + if(TYPE.equals(field.getName())) { + //type is immutable must not be overwritten + return; + } field.set(this, value); } diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java index 1582c9d62..cc363b6fc 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java @@ -28,7 +28,7 @@ // enable test config to use zoho mocking @Import(TestConfig.class) // enable tests only on local machine -// @Disabled +//@Disabled @SpringBootTest public class DereferenceServiceIT extends AbstractIntegrationTest { @@ -65,7 +65,7 @@ public void dereferenceConceptById() throws Exception { assertEquals(8, entity.getNote().size()); } - // @Test + @Test public void zohoOrganizationDereferenceTest() throws Exception { String organizationId = IntegrationTestUtils.ORGANIZATION_BNF_URI_ZOHO; Dereferencer dereferencer = diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java index 11c682fce..9ed6e4003 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java @@ -169,7 +169,7 @@ public class IntegrationTestUtils { public static final String INVALID_MIGRATION_ID = "http://www.testing.org/entity/testing"; public static final String ORGANIZATION_BNF_URI_ZOHO = - "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000002112001"; + "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000938399"; public static final String ORGANIZATION_NATURALIS_URI_ZOHO = "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000000370517"; public static final String ORGANIZATION_PCCE_URI_ZOHO = diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java index 145a45dff..4de3f5e62 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java @@ -4,6 +4,7 @@ import org.mockito.ArgumentMatchers; import org.mockito.Mockito; import org.mockito.stubbing.Answer; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Primary; @@ -18,6 +19,9 @@ public class TestConfig { public static final String MOCK_ZOHO_BASE_URL = "https://crm.zoho.com/crm/org51823723/tab/Accounts/"; public static final String MOCK_ZOHO_COUNTRY_MAPPING_FILE = "/zoho_country_mapping_test.json"; + @Autowired + ZohoConfiguration zohoConfiguration; + /** * Since requests to Zoho are done via its SDK, and require authentication first, we mock out the * entire flow with Mockito. @@ -28,7 +32,7 @@ public ZohoConfiguration configureZoho() throws Exception { ZohoConfiguration zohoConfiguration = Mockito.mock(ZohoConfiguration.class); ZohoAccessClient zohoClient = Mockito.mock(ZohoAccessClient.class); Mockito.when(zohoConfiguration.getZohoAccessClient()).thenReturn(zohoClient); - Mockito.when(zohoConfiguration.getZohoBaseUrl()).thenReturn(MOCK_ZOHO_BASE_URL); + Mockito.when(zohoConfiguration.getZohoBaseUrl()).thenReturn(zohoConfiguration.getZohoBaseUrl()); // find matching JSON file based on zohoId argument, then create a Record object for it Mockito.doAnswer( diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/ZohoRecordTestDeserializer.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/ZohoRecordTestDeserializer.java index ff9018298..d5c0975f8 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/ZohoRecordTestDeserializer.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/ZohoRecordTestDeserializer.java @@ -99,7 +99,13 @@ public Record deserialize(JsonParser p, DeserializationContext ctxt) throws IOEx List> values = new ArrayList>(); currentNode.elements().forEachRemaining(v -> values.add(new Choice(v.asText()))); record.addKeyValue(key, values); - } + }else if (currentNode.isContainerNode()){ + System.out.println("container node: " + key); + }else if(currentNode.isPojo()) { + System.out.println("pojo node: " + key); + }else if(currentNode.isObject()) { + System.out.println("object node: " + key); + } } // add fields with numeric suffixes diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java new file mode 100644 index 000000000..cbccaa050 --- /dev/null +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java @@ -0,0 +1,59 @@ +package eu.europeana.entitymanagement.zoho.organization; + +import static org.junit.Assert.assertNotNull; +import java.util.Optional; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Bean; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.zoho.crm.api.record.Record; +import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration; +import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants; +import eu.europeana.entitymanagement.config.SerializationConfig; +import eu.europeana.entitymanagement.testutils.IntegrationTestUtils; +import eu.europeana.entitymanagement.testutils.ZohoRecordTestDeserializer; + +/** JUnit test to check if DataSources are properly deserialized from XML */ +@SpringBootTest(classes = {SerializationConfig.class, ZohoConfiguration.class, EntityManagementConfiguration.class}) +public class ZohoTestDataGenerator { + + @Autowired + @Qualifier(AppConfigConstants.BEAN_JSON_MAPPER) + private ObjectMapper jsonMapper; + + @Autowired + ZohoConfiguration zohoConfiguration; + EntityManagementConfiguration emConfig; + + @Bean + ZohoDereferenceService getZohoDereferenceService() { + return new ZohoDereferenceService(zohoConfiguration, emConfig); + } + +// @Test + //manually enable the test when data needs to be generated + public void generateBNFJson() throws Exception { + //get original zoho record + Optional zohoOrganization = + zohoConfiguration.getZohoAccessClient().getZohoRecordOrganizationById(IntegrationTestUtils.ORGANIZATION_BNF_URI_ZOHO); + String zohoRecord = getZohoDereferenceService().serialize(zohoOrganization.get()); + //if you need to see original enable : System.out.println(zohoRecord); + System.out.println(zohoRecord); + + //deserialize data with the test deserializer + ZohoRecordTestDeserializer zohoRecordDeserializer = new ZohoRecordTestDeserializer(); + //TODO: many fields are missing or need to be renamed in the deserializer, need to include all fields from ZohoMaping except for the sensitive ones (like email addresses) + Record zohoTestRecord = zohoRecordDeserializer.deserialize(jsonMapper.getFactory().createParser(zohoRecord), jsonMapper.getDeserializationContext()); + + //serialize test data + String zohoTestRecordJson = getZohoDereferenceService().serialize(zohoTestRecord); + System.out.println(zohoTestRecordJson); + + Record zohoTestRecordFiltered = zohoRecordDeserializer.deserialize(jsonMapper.getFactory().createParser(zohoTestRecordJson), jsonMapper.getDeserializationContext()); + assertNotNull(zohoTestRecordFiltered.getId()); + assertNotNull(zohoTestRecordFiltered.getKeyValue("Account_Name")); + + } +} \ No newline at end of file diff --git a/entity-management-web/.gitignore b/entity-management-web/.gitignore index 154abc922..658853ec2 100644 --- a/entity-management-web/.gitignore +++ b/entity-management-web/.gitignore @@ -1,2 +1,3 @@ /SDKLogs.log* **/log4j2.xml +/sdk_tokens.txt diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/EMController.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/EMController.java index 5b53035b8..b1c875869 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/EMController.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/EMController.java @@ -72,6 +72,7 @@ import eu.europeana.entitymanagement.vocabulary.WebEntityConstants; import eu.europeana.entitymanagement.web.service.DereferenceServiceLocator; import eu.europeana.entitymanagement.web.service.EntityRecordService; +import eu.europeana.entitymanagement.zoho.utils.ZohoUtils; import io.swagger.annotations.ApiOperation; @RestController @@ -521,8 +522,7 @@ public ResponseEntity registerEntity(@RequestBody Entity europeanaProxyE // in case of Organization it must be the zoho Organization String creationRequestType = europeanaProxyEntity.getType(); - if (EntityTypes.isOrganization(creationRequestType) - && !creationRequestId.contains(DataSource.ZOHO_HOST)) { + if (!ZohoUtils.isZohoOrganization(creationRequestId, creationRequestType)) { throw new HttpBadRequestException(String.format( "The Organization entity should come from Zoho and have the corresponding id format containing: %s", DataSource.ZOHO_HOST)); diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/DereferenceServiceLocator.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/DereferenceServiceLocator.java index 0a3106203..08689236f 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/DereferenceServiceLocator.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/DereferenceServiceLocator.java @@ -35,6 +35,8 @@ public Dereferencer getDereferencer(String id, String entityType) { if (ZohoUtils.isZohoOrganization(id, entityType)) { return zohoDereferenceService; } + + System.out.println(); return metisDereferenceService; } diff --git a/entity-management-web/src/main/resources/datasources.xml b/entity-management-web/src/main/resources/datasources.xml index c100b6c16..5df2515b8 100644 --- a/entity-management-web/src/main/resources/datasources.xml +++ b/entity-management-web/src/main/resources/datasources.xml @@ -10,7 +10,7 @@ - + \ No newline at end of file diff --git a/entity-management-zoho/pom.xml b/entity-management-zoho/pom.xml index 08058a1b2..68c211e0b 100644 --- a/entity-management-zoho/pom.xml +++ b/entity-management-zoho/pom.xml @@ -23,6 +23,8 @@ com.zoho.crm java-sdk ${zoho-sdk.version} + + org.springframework diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/ZohoAccessClient.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/ZohoAccessClient.java index f45d7a1a6..c5291e9a0 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/ZohoAccessClient.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/ZohoAccessClient.java @@ -23,6 +23,7 @@ import com.zoho.crm.api.SDKConfig; import com.zoho.crm.api.UserSignature; import com.zoho.crm.api.dc.DataCenter.Environment; +import com.zoho.crm.api.dc.EUDataCenter; import com.zoho.crm.api.dc.USDataCenter; import com.zoho.crm.api.exception.SDKException; import com.zoho.crm.api.record.APIException; @@ -84,7 +85,8 @@ public ZohoAccessClient( new OAuthToken(clientId, clientSecret, refreshToken, TokenType.REFRESH, redirectUrl); SDKConfig sdkConfig = new SDKConfig.Builder().setAutoRefreshFields(false).setPickListValidation(true).build(); - Environment environment = USDataCenter.PRODUCTION; + //Environment environment = USDataCenter.PRODUCTION; + Environment environment = EUDataCenter.PRODUCTION; String resourcePath = SystemUtils.getUserHome().getAbsolutePath(); // Does not generate any tokens, we'll need to execute a command to do so Initializer.initialize( diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java index 122dd9d96..9cb680f0b 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java @@ -1,7 +1,6 @@ package eu.europeana.entitymanagement.zoho.organization; import java.text.SimpleDateFormat; -import java.util.Map; import java.util.Optional; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; @@ -9,11 +8,12 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; import com.zoho.crm.api.record.Record; import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration; -import eu.europeana.entitymanagement.definitions.model.CountryMapping; import eu.europeana.entitymanagement.definitions.model.Entity; import eu.europeana.entitymanagement.dereference.Dereferencer; @@ -34,8 +34,9 @@ public Optional dereferenceEntityById(@NonNull String id) throws Excepti Optional zohoOrganization = zohoConfiguration.getZohoAccessClient().getZohoRecordOrganizationById(id); - // Gson resp = new Gson(); - // System.out.println(resp.toJson(zohoOrganization.get().getKeyValues())); + + + System.out.println(serialize(zohoOrganization.get())); if(zohoOrganization.isPresent()) { return Optional.of( @@ -57,7 +58,7 @@ public String serialize(Record zohoRecord) throws JsonProcessingException { .serializationInclusion(JsonInclude.Include.NON_NULL) .build(); mapper.findAndRegisterModules(); - return mapper.writeValueAsString(zohoRecord.getKeyValues()); + return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(zohoRecord.getKeyValues()); } } From d4222ac7b9830f7497ed7ce3aa139d8bff1c7de1 Mon Sep 17 00:00:00 2001 From: GordeaS Date: Wed, 31 Jan 2024 15:39:28 +0100 Subject: [PATCH 16/22] remove sysouts --- .../zoho/organization/ZohoTestDataGenerator.java | 2 +- .../zoho/organization/ZohoDereferenceService.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java index cbccaa050..5eb2ba8ba 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java @@ -40,7 +40,7 @@ public void generateBNFJson() throws Exception { zohoConfiguration.getZohoAccessClient().getZohoRecordOrganizationById(IntegrationTestUtils.ORGANIZATION_BNF_URI_ZOHO); String zohoRecord = getZohoDereferenceService().serialize(zohoOrganization.get()); //if you need to see original enable : System.out.println(zohoRecord); - System.out.println(zohoRecord); +// System.out.println(zohoRecord); //deserialize data with the test deserializer ZohoRecordTestDeserializer zohoRecordDeserializer = new ZohoRecordTestDeserializer(); diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java index 9cb680f0b..2208d30e2 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java @@ -35,8 +35,7 @@ public Optional dereferenceEntityById(@NonNull String id) throws Excepti Optional zohoOrganization = zohoConfiguration.getZohoAccessClient().getZohoRecordOrganizationById(id); - - System.out.println(serialize(zohoOrganization.get())); + //enable when you need to print the data for debuging purposes System.out.println(serialize(zohoOrganization.get())); if(zohoOrganization.isPresent()) { return Optional.of( From 1fb2901a4fefec5ec670cc63093d2c558521d4b0 Mon Sep 17 00:00:00 2001 From: Srdjan Stevanetic Date: Thu, 1 Feb 2024 19:40:50 +0100 Subject: [PATCH 17/22] zoho fields changed to comply with zoho eu --- .../common/config/DataSource.java | 1 - .../vocabulary/OrganizationSolrFields.java | 6 +- .../vocabulary/WebEntityFields.java | 2 +- .../solr/model/SolrOrganization.java | 47 ++----------- .../AbstractIntegrationTest.java | 5 +- .../service/DereferenceServiceIT.java | 4 +- .../testutils/IntegrationTestUtils.java | 9 +-- .../testutils/TestConfig.java | 4 +- .../testutils/ZohoRecordTestDeserializer.java | 4 +- .../web/EntityRegistrationIT.java | 2 +- .../web/EntityRetrievalIT.java | 5 +- .../organization/ZohoTestDataGenerator.java | 4 +- .../resources/content/organization.json | 2 +- ...anization_register_zoho_berger_museum.json | 2 +- .../organization_register_zoho_bnf.json | 2 +- .../organization_register_zoho_gfm.json | 2 +- .../organization_register_zoho_naturalis.json | 2 +- .../organization_register_zoho_pcce.json | 2 +- .../resources/solr-docker/conf/schema.xml | 8 +-- ...anization_zoho_berger_museum_response.json | 36 +++++----- .../organization_zoho_bnf_response.json | 69 ++++++++----------- .../organization_zoho_gfm_response.json | 33 ++++----- .../organization_zoho_naturalis_response.json | 38 +++++----- .../organization_zoho_pcce_response.json | 47 +++++-------- .../entitymanagement/web/EMController.java | 7 +- .../web/service/EntityRecordService.java | 1 - .../ZohoOrganizationConverter.java | 18 ++--- .../zoho/utils/ZohoConstants.java | 14 ++-- .../zoho/utils/ZohoUtils.java | 10 +-- 29 files changed, 154 insertions(+), 232 deletions(-) diff --git a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/DataSource.java b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/DataSource.java index 776cfb7dd..c296e3adb 100644 --- a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/DataSource.java +++ b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/DataSource.java @@ -9,7 +9,6 @@ public class DataSource { public static final String FREQ_STATIC = "static"; public static final String EUROPEANA_ID = "europeana"; public static final String ZOHO_ID = "zoho-crm"; - public static final String ZOHO_HOST = "crm.zoho.eu"; @JacksonXmlProperty(isAttribute = true) private String url; diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java index 79d1848c2..f286ec96c 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/OrganizationSolrFields.java @@ -16,11 +16,7 @@ public interface OrganizationSolrFields extends EntitySolrFields { public static final String FOAF_MBOX = "foaf_mbox"; public static final String EUROPEANA_ROLE = "europeanaRole"; public static final String EUROPEANA_ROLE_ALL = EUROPEANA_ROLE + EXTENSION_ALL; - public static final String COUNTRY_ID = "countryId"; - public static final String COUNTRY_PREF_LABEL_ALL = "countryPrefLabel.*"; - public static final String COUNTRY_PREF_LABEL = "countryPrefLabel"; - public static final String COUNTRY_LAT = "countryLatitude"; - public static final String COUNTRY_LONG = "countryLongitude"; + public static final String COUNTRY = "country"; public static final String VCARD_HAS_ADDRESS = "vcard_hasAddress.1"; public static final String VCARD_STREET_ADDRESS = "vcard_streetAddress.1"; public static final String VCARD_LOCALITY = "vcard_locality.1"; diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/WebEntityFields.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/WebEntityFields.java index 99eeb85ac..2b5b51e55 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/WebEntityFields.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/WebEntityFields.java @@ -9,7 +9,7 @@ public interface WebEntityFields { public static final String LANGUAGE_EN = "en"; public static final String BASE_DATA_EUROPEANA_URI = "http://data.europeana.eu/"; public static final String WIKIDATA_HOST = "www.wikidata.org"; - public static final String ZOHO_CRM_HOST = "crm.zoho.com"; + public static final String ZOHO_CRM_HOST = "crm.zoho.eu"; // common fields public static final String ID = "id"; diff --git a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java index 92e5a9e98..1e98d8a06 100644 --- a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java +++ b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java @@ -8,7 +8,6 @@ import org.apache.solr.client.solrj.beans.Field; import eu.europeana.entitymanagement.definitions.model.Address; import eu.europeana.entitymanagement.definitions.model.Organization; -import eu.europeana.entitymanagement.definitions.model.Place; import eu.europeana.entitymanagement.solr.SolrUtils; import eu.europeana.entitymanagement.utils.EntityUtils; import eu.europeana.entitymanagement.vocabulary.EntitySolrFields; @@ -40,17 +39,8 @@ public class SolrOrganization extends SolrEntity { @Field(OrganizationSolrFields.EUROPEANA_ROLE_ALL) private Map> europeanaRole; - @Field(OrganizationSolrFields.COUNTRY_ID) - private String countryId; - - @Field(OrganizationSolrFields.COUNTRY_PREF_LABEL_ALL) - private Map countryPrefLabel; - - @Field(OrganizationSolrFields.COUNTRY_LAT) - private Float countryLatitude; - - @Field(OrganizationSolrFields.COUNTRY_LONG) - private Float countryLongitude; + @Field(OrganizationSolrFields.COUNTRY) + private String country; @Field(OrganizationSolrFields.VCARD_HAS_ADDRESS) private String hasAddress; @@ -93,12 +83,7 @@ public SolrOrganization(Organization organization) { if (organization.getMbox() != null) this.mbox = new ArrayList<>(organization.getMbox()); setEuropeanaRole(organization.getEuropeanaRole()); - if(organization.getCountry()!=null) { - this.countryId = organization.getCountry().getEntityId(); - this.setCountryPrefLabel(organization.getCountry().getPrefLabel()); - this.countryLatitude = ((Place) organization.getCountry()).getLatitude(); - this.countryLongitude = ((Place) organization.getCountry()).getLongitude(); - } + this.country=organization.getCountryId(); if (organization.getSameReferenceLinks() != null) { this.sameAs = new ArrayList<>(organization.getSameReferenceLinks()); @@ -145,16 +130,6 @@ private void setEuropeanaRole(Map> europeanaRole) { } } - private void setCountryPrefLabel(Map prefLabel) { - if (MapUtils.isNotEmpty(prefLabel)) { - this.countryPrefLabel = - new HashMap<>( - SolrUtils.normalizeStringMapByAddingPrefix( - OrganizationSolrFields.COUNTRY_PREF_LABEL + EntitySolrFields.DYNAMIC_FIELD_SEPARATOR, - prefLabel)); - } - } - public Map getDescription() { return description; } @@ -220,19 +195,7 @@ protected void setSameReferenceLinks(ArrayList uris) { this.sameAs = uris; } - public String getCountryId() { - return countryId; - } - - public Map getCountryPrefLabel() { - return countryPrefLabel; - } - - public Float getCountryLatitude() { - return countryLatitude; - } - - public Float getCountryLongitude() { - return countryLongitude; + public String getCountry() { + return country; } } diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java index 1ed56cb7e..97d192d17 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java @@ -65,7 +65,7 @@ public abstract class AbstractIntegrationTest { @Autowired protected ConceptSchemeService emConceptSchemeService; @Autowired protected EntityManagementConfiguration emConfig; @Autowired protected ZohoConfiguration zohoConfiguration; - + static { MONGO_CONTAINER = new MongoContainer("entity-management", "job-repository", "enrichment") @@ -78,6 +78,8 @@ public abstract class AbstractIntegrationTest { .withLogConsumer(new WaitingConsumer().andThen(new ToStringConsumer())); SOLR_CONTAINER.start(); + + } /** MockWebServer needs to be static, so we can inject its port into the Spring context. */ @@ -101,6 +103,7 @@ public static void setupAll() throws IOException { mockSearchAndRecord = new MockWebServer(); mockSearchAndRecord.setDispatcher(setupSearchAndRecordDispatcher()); mockSearchAndRecord.start(); + } @AfterAll diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java index cc363b6fc..25e532607 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java @@ -84,9 +84,9 @@ public void zohoOrganizationDereferenceTest() throws Exception { Assertions.assertNotNull(org.getLogo()); Assertions.assertNotNull(org.getAddress().getVcardStreetAddress()); Assertions.assertNotNull(org.getAddress().getVcardCountryName()); + Assertions.assertTrue(org.getHiddenLabel().contains("BN Paris (blíže neurčeno)")); Assertions.assertTrue(org.getHiddenLabel().contains("Bibliothèque nationale")); - Assertions.assertTrue(org.getHiddenLabel().contains("Bibliothèque nationale Francaise")); - Assertions.assertTrue(org.getHiddenLabel().contains("French National Library")); + Assertions.assertTrue(org.getHiddenLabel().contains("Bibliothèque nationale Francaise French National Library")); } // @Test diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java index 9ed6e4003..3d5442041 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java @@ -171,13 +171,14 @@ public class IntegrationTestUtils { public static final String ORGANIZATION_BNF_URI_ZOHO = "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000938399"; public static final String ORGANIZATION_NATURALIS_URI_ZOHO = - "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000000370517"; + "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000923816"; public static final String ORGANIZATION_PCCE_URI_ZOHO = - "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000000338555"; + "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000923271"; public static final String ORGANIZATION_GFM_URI_ZOHO = - "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000004503618"; + "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000940433"; public static final String ORGANIZATION_BERGER_MUSEUM_URI_ZOHO = - "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000004477407"; + "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000939337"; + public static final String ORGANIZATION_NATURALIS_URI_WIKIDATA_PATH_SUFFIX = "/entity/Q641676"; public static final String ORGANIZATION_NATURALIS_URI_WIKIDATA_URI = WIKIDATA_BASE_URI + ORGANIZATION_NATURALIS_URI_WIKIDATA_PATH_SUFFIX; diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java index 4de3f5e62..6e310b8bc 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java @@ -16,7 +16,7 @@ public class TestConfig { - public static final String MOCK_ZOHO_BASE_URL = "https://crm.zoho.com/crm/org51823723/tab/Accounts/"; + public static final String MOCK_ZOHO_BASE_URL = "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/"; public static final String MOCK_ZOHO_COUNTRY_MAPPING_FILE = "/zoho_country_mapping_test.json"; @Autowired @@ -32,7 +32,7 @@ public ZohoConfiguration configureZoho() throws Exception { ZohoConfiguration zohoConfiguration = Mockito.mock(ZohoConfiguration.class); ZohoAccessClient zohoClient = Mockito.mock(ZohoAccessClient.class); Mockito.when(zohoConfiguration.getZohoAccessClient()).thenReturn(zohoClient); - Mockito.when(zohoConfiguration.getZohoBaseUrl()).thenReturn(zohoConfiguration.getZohoBaseUrl()); + Mockito.when(zohoConfiguration.getZohoBaseUrl()).thenReturn(MOCK_ZOHO_BASE_URL); // find matching JSON file based on zohoId argument, then create a Record object for it Mockito.doAnswer( diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/ZohoRecordTestDeserializer.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/ZohoRecordTestDeserializer.java index d5c0975f8..427edfe1c 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/ZohoRecordTestDeserializer.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/ZohoRecordTestDeserializer.java @@ -20,11 +20,11 @@ import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.LOGO_LINK_TO_WIKIMEDIACOMMONS_FIELD; import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.LONGITUDE_FIELD; import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.OFFICIAL_LANGUAGE_FIELD; -import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.ORGANIZATION_COUNTRY_FIELD; import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.ORGANIZATION_ROLE_FIELD; import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.PO_BOX_FIELD; import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.SAME_AS_CODE_LENGTH; import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.SAME_AS_FIELD; +import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.STREET_FIELD; import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.WEBSITE_FIELD; import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.ZIP_CODE_FIELD; import java.io.IOException; @@ -57,7 +57,7 @@ public class ZohoRecordTestDeserializer extends StdDeserializer { ACRONYM_FIELD, LOGO_LINK_TO_WIKIMEDIACOMMONS_FIELD, WEBSITE_FIELD, - ORGANIZATION_COUNTRY_FIELD, + STREET_FIELD, CITY_FIELD, COUNTRY_FIELD, ZIP_CODE_FIELD, diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java index 30aafb182..591c77ff9 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java @@ -273,7 +273,7 @@ public void registerZohoOrganizationBergerShouldBeSuccessful() throws Exception Matchers.hasItems( IntegrationTestUtils.ORGANIZATION_BERGER_MUSEUM_WIKIDATA_URI, IntegrationTestUtils.ORGANIZATION_BERGER_MUSEUM_URI_ZOHO))) - .andExpect(jsonPath("$.prefLabel[*]", hasSize(1))) + .andExpect(jsonPath("$.prefLabel[*]", hasSize(2))) // should have Europeana, Zoho and Wikidata proxies .andExpect(jsonPath("$.proxies", hasSize(3))); diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java index f1e1429e0..ccf69eb6a 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java @@ -523,8 +523,7 @@ public void retrieveOrganizationJsonExternalWithCountryDereference() throws Exce .andExpect(jsonPath("$.id", is(entityId))) .andExpect(jsonPath("$.type", is(EntityTypes.Organization.getEntityType()))) .andExpect(jsonPath("$.sameAs").isNotEmpty()) - .andExpect(jsonPath("$.countryId", is("http://data.europeana.eu/place/1"))) - .andExpect(jsonPath("$.countryPlace.prefLabel.en", is("Sweden"))); + .andExpect(jsonPath("$.country.prefLabel.en", is("Sweden"))); } @Test @@ -547,7 +546,7 @@ public void retrieveOrganizationJsonExternal() throws Exception { .andExpect(jsonPath("$.id", is(entityId))) .andExpect(jsonPath("$.type", is(EntityTypes.Organization.getEntityType()))) .andExpect(jsonPath("$.sameAs").isNotEmpty()) - .andExpect(jsonPath("$.countryId").isNotEmpty()) + .andExpect(jsonPath("$.hasAddress.countryName").isNotEmpty()) .andExpect(jsonPath("$.countryPlace").doesNotExist()); } diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java index 5eb2ba8ba..7fa028cf2 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/zoho/organization/ZohoTestDataGenerator.java @@ -37,10 +37,10 @@ ZohoDereferenceService getZohoDereferenceService() { public void generateBNFJson() throws Exception { //get original zoho record Optional zohoOrganization = - zohoConfiguration.getZohoAccessClient().getZohoRecordOrganizationById(IntegrationTestUtils.ORGANIZATION_BNF_URI_ZOHO); + zohoConfiguration.getZohoAccessClient().getZohoRecordOrganizationById(IntegrationTestUtils.ORGANIZATION_PCCE_URI_ZOHO); String zohoRecord = getZohoDereferenceService().serialize(zohoOrganization.get()); //if you need to see original enable : System.out.println(zohoRecord); -// System.out.println(zohoRecord); + System.out.println(zohoRecord); //deserialize data with the test deserializer ZohoRecordTestDeserializer zohoRecordDeserializer = new ZohoRecordTestDeserializer(); diff --git a/entity-management-tests/src/integration-test/resources/content/organization.json b/entity-management-tests/src/integration-test/resources/content/organization.json index ec60d2f89..41462810a 100644 --- a/entity-management-tests/src/integration-test/resources/content/organization.json +++ b/entity-management-tests/src/integration-test/resources/content/organization.json @@ -1,6 +1,6 @@ { "@context": "http://www.europeana.eu/schemas/context/entity.jsonld", - "id": "http://data.europeana.eu/organization/1482250000000370517", + "id": "http://data.europeana.eu/organization/1", "type": "Organization", "depiction": { "id": "http://commons.wikimedia.org/wiki/Special:FilePath/WLANL%20-%20thedogg%20-%20Mammoet%20%282%29.jpg", diff --git a/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_berger_museum.json b/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_berger_museum.json index 09b532f7a..86d580897 100644 --- a/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_berger_museum.json +++ b/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_berger_museum.json @@ -1,4 +1,4 @@ { "type": "Organization", - "id": "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000004477407" + "id": "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000939337" } \ No newline at end of file diff --git a/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_bnf.json b/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_bnf.json index 89ecf2d47..32c78b332 100644 --- a/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_bnf.json +++ b/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_bnf.json @@ -1,6 +1,6 @@ { "type": "Organization", - "id": "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000002112001", + "id": "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000938399", "prefLabel": { "fr": "Bibliothèque nationale de France" }, diff --git a/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_gfm.json b/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_gfm.json index 626752a50..ba5e846a1 100644 --- a/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_gfm.json +++ b/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_gfm.json @@ -1,4 +1,4 @@ { "type": "Organization", - "id": "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000004503618" + "id": "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000940433" } \ No newline at end of file diff --git a/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_naturalis.json b/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_naturalis.json index 29ed51e0e..887a5a7c1 100644 --- a/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_naturalis.json +++ b/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_naturalis.json @@ -1,6 +1,6 @@ { "type": "Organization", - "id": "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000000370517", + "id": "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000923816", "prefLabel": { "fr": "Nauralis" }, diff --git a/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_pcce.json b/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_pcce.json index 41c3f9149..fe7cee3f7 100644 --- a/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_pcce.json +++ b/entity-management-tests/src/integration-test/resources/content/organization_register_zoho_pcce.json @@ -1,4 +1,4 @@ { "type": "Organization", - "id": "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000000338555" + "id": "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000923271" } \ No newline at end of file diff --git a/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml b/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml index 9bca1f7d1..e7b4243a6 100644 --- a/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml +++ b/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml @@ -509,14 +509,8 @@ - - - - - + + @@ -571,7 +573,7 @@ dest="text.*" /> - + @@ -585,7 +587,7 @@ dest="text" /> - + diff --git a/entity-management-tests/src/integration-test/resources/zoho_role_mapping_test.json b/entity-management-tests/src/integration-test/resources/zoho_role_mapping_test.json new file mode 100644 index 000000000..10a5602c7 --- /dev/null +++ b/entity-management-tests/src/integration-test/resources/zoho_role_mapping_test.json @@ -0,0 +1,6 @@ +[ + { + "Zoho Label": "providing institution", + "Entity URI": "http://data.europeana.eu/vocabulary/role/ProvidingInstitution" + } +] diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/MetisDereferenceUtils.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/MetisDereferenceUtils.java index b04307ad2..60b0a8c98 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/MetisDereferenceUtils.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/MetisDereferenceUtils.java @@ -1,13 +1,14 @@ package eu.europeana.entitymanagement.web; -import eu.europeana.api.commons.error.EuropeanaApiException; -import eu.europeana.entitymanagement.exception.DatasourceDereferenceException; -import eu.europeana.entitymanagement.web.xml.model.XmlBaseEntityImpl; -import eu.europeana.entitymanagement.web.xml.model.metis.EnrichmentResultList; import java.io.StringReader; +import java.util.List; import java.util.Optional; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; +import eu.europeana.api.commons.error.EuropeanaApiException; +import eu.europeana.entitymanagement.exception.DatasourceDereferenceException; +import eu.europeana.entitymanagement.web.xml.model.XmlBaseEntityImpl; +import eu.europeana.entitymanagement.web.xml.model.metis.EnrichmentResultList; public class MetisDereferenceUtils { @@ -59,4 +60,34 @@ public static XmlBaseEntityImpl parseMetisResponse( return entityOptional.get(); } + + public static List> parseMetisResponseMany( + Unmarshaller unmarshaller, String metisResponseBody) throws EuropeanaApiException { + EnrichmentResultList derefResult; + + try { + derefResult = + (EnrichmentResultList) unmarshaller.unmarshal(new StringReader(metisResponseBody)); + } catch (JAXBException | RuntimeException e) { + throw new DatasourceDereferenceException( + String.format( + "Error while deserializing metis dereference response %s: ", metisResponseBody), + e); + } + + if (derefResult == null + || derefResult.getEnrichmentBaseResultWrapperList().isEmpty() + || derefResult + .getEnrichmentBaseResultWrapperList() + .get(0) + .getEnrichmentBaseList() + .isEmpty()) { + // Metis returns an empty XML response if de-referencing is unsuccessful, + // instead of throwing an error + return null; + } + + return derefResult.getEnrichmentBaseResultWrapperList().get(0).getEnrichmentBaseList(); + + } } diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java index ba6783a4f..8dd9aa028 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java @@ -326,7 +326,7 @@ List performEntityRegistration(SortedSet createOperations, Zo */ private void performEntityRegistration(Operation operation, ZohoSyncReport zohoSyncReport, List entitiesToUpdate) { Organization zohoOrganization = - ZohoOrganizationConverter.convertToOrganizationEntity(operation.getZohoRecord(), zohoConfiguration.getZohoBaseUrl(), emConfiguration.getCountryMappings()); + ZohoOrganizationConverter.convertToOrganizationEntity(operation.getZohoRecord(), zohoConfiguration.getZohoBaseUrl(), emConfiguration.getCountryMappings(), emConfiguration.getRoleMappings()); try { List existingEntities = diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java index e21191c3e..c0f1d9f66 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java @@ -47,6 +47,7 @@ import eu.europeana.entitymanagement.definitions.model.Organization; import eu.europeana.entitymanagement.definitions.model.Place; import eu.europeana.entitymanagement.definitions.model.TimeSpan; +import eu.europeana.entitymanagement.definitions.model.Vocabulary; import eu.europeana.entitymanagement.definitions.web.EntityIdDisabledStatus; import eu.europeana.entitymanagement.exception.EntityAlreadyExistsException; import eu.europeana.entitymanagement.exception.EntityCreationException; @@ -57,6 +58,7 @@ import eu.europeana.entitymanagement.exception.MultipleChoicesException; import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException; import eu.europeana.entitymanagement.mongo.repository.EntityRecordRepository; +import eu.europeana.entitymanagement.mongo.repository.VocabularyRepository; import eu.europeana.entitymanagement.solr.exception.SolrServiceException; import eu.europeana.entitymanagement.solr.service.SolrService; import eu.europeana.entitymanagement.utils.EMCollectionUtils; @@ -79,6 +81,8 @@ public class EntityRecordService { private final EntityRecordRepository entityRecordRepository; + + private final VocabularyRepository vocabRepository; final EntityManagementConfiguration emConfiguration; @@ -94,10 +98,11 @@ public class EntityRecordService { private static final Set ignoredMergeFields = Set.of(WebEntityFields.TYPE); @Autowired - public EntityRecordService(EntityRecordRepository entityRecordRepository, + public EntityRecordService(EntityRecordRepository entityRecordRepository, VocabularyRepository vocabRepository, EntityManagementConfiguration emConfiguration, ZohoConfiguration zohoConfiguration, DataSources datasources, SolrService solrService) throws IOException { this.entityRecordRepository = entityRecordRepository; + this.vocabRepository=vocabRepository; this.emConfiguration = emConfiguration; this.zohoConfiguration = zohoConfiguration; this.datasources = datasources; @@ -160,7 +165,7 @@ void dereferenceLinkedEntities(EntityRecord entityRecord) { } private void dereferenceLinkedEntities(Organization org) { - // in case of dereference profile + //dereference country if (org.getCountryId() != null) { EntityRecord countryRecord = entityRecordRepository.findByEntityId(org.getCountryId(), true); if (countryRecord != null) { @@ -169,6 +174,10 @@ private void dereferenceLinkedEntities(Organization org) { org.setCountry(country); } } + //dereference role + if(org.getEuropeanaRoleIds()!=null && !org.getEuropeanaRoleIds().isEmpty()) { + org.setEuropeanaRole(vocabRepository.findByVocabularyUris(org.getEuropeanaRoleIds())); + } } /** @@ -1279,6 +1288,7 @@ && isStringOrPrimitive(mapFirstValue.getClass())) { public void dropRepository() { this.entityRecordRepository.dropCollection(); + this.vocabRepository.dropCollection(); } /** @@ -1462,7 +1472,7 @@ void updateEuropeanaIDFieldInZoho(String zohoOrganizationUrl, String europeanaId public void processReferenceFields(Entity entity) { if (EntityTypes.isOrganization(entity.getType())) { Organization org = (Organization) entity; - + //country reference if (StringUtils.isNotEmpty(org.getCountryId())) { EntityRecord orgCountry = entityRecordRepository.findByEntityId(org.getCountryId()); if (orgCountry == null) { @@ -1473,6 +1483,17 @@ public void processReferenceFields(Entity entity) { org.setCountryRef(orgCountry); } } + //role reference + if(org.getEuropeanaRoleIds()!=null && !org.getEuropeanaRoleIds().isEmpty()) { + List vocabs=vocabRepository.findByVocabularyUris(org.getEuropeanaRoleIds()); + if (vocabs.isEmpty()) { + logger.info( + "No vocabularies with the uris: {} were found in the database. Cannot assign role reference to organization with id {}", + org.getEuropeanaRoleIds(), org.getEntityId()); + } else { + org.setEuropeanaRoleRefs(vocabs); + } + } } } } diff --git a/entity-management-web/src/main/resources/entitymanagement.properties b/entity-management-web/src/main/resources/entitymanagement.properties index 7938774ae..a21f91cfe 100644 --- a/entity-management-web/src/main/resources/entitymanagement.properties +++ b/entity-management-web/src/main/resources/entitymanagement.properties @@ -76,4 +76,5 @@ zoho.sync.register.deprecated = false #disable generation of entity ids #zoho.generate.organization.europeanaid=: false -zoho.country.mapping=/zoho_country_mapping.json \ No newline at end of file +zoho.country.mapping=/zoho_country_mapping.json +zoho.role.mapping=/zoho_role_mapping.json \ No newline at end of file diff --git a/entity-management-web/src/main/resources/entitymanagement.user.properties.template b/entity-management-web/src/main/resources/entitymanagement.user.properties.template index 4b2dbaacd..15c8a4fe3 100644 --- a/entity-management-web/src/main/resources/entitymanagement.user.properties.template +++ b/entity-management-web/src/main/resources/entitymanagement.user.properties.template @@ -48,4 +48,5 @@ europeana.searchapi.urlPrefix=https:///record/v2/search.json?wskey #enable/disable generation of entity ids for organizations (should be enabled only in productive environment) #zoho.generate.organization.europeanaid=false -zoho.country.mapping= \ No newline at end of file +zoho.country.mapping= +zoho.role.mapping= \ No newline at end of file diff --git a/entity-management-web/src/main/resources/zoho_role_mapping.json b/entity-management-web/src/main/resources/zoho_role_mapping.json new file mode 100644 index 000000000..2e88e0d62 --- /dev/null +++ b/entity-management-web/src/main/resources/zoho_role_mapping.json @@ -0,0 +1,18 @@ +[ + { + "Zoho Label": "providing institution", + "Entity URI": "http://data.europeana.eu/vocabulary/role/ProvidingInstitution" + }, + { + "Zoho Label": "aggregator", + "Entity URI": "http://data.europeana.eu/vocabulary/role/Aggregator" + }, + { + "Zoho Label": "accredited aggregator", + "Entity URI": "http://data.europeana.eu/vocabulary/role/AccreditedAggregator" + }, + { + "Zoho Label": "potential providing institution", + "Entity URI": "http://data.europeana.eu/vocabulary/role/ProvidingInstitution" + } +] diff --git a/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java b/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java index 6a56a16ff..893be8f31 100644 --- a/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java +++ b/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java @@ -2,16 +2,23 @@ import static eu.europeana.entitymanagement.solr.SolrUtils.createSolrEntity; import java.util.List; +import javax.xml.bind.JAXBContext; import org.junit.jupiter.api.Disabled; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.context.SpringBootTest; import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.definitions.model.Concept; import eu.europeana.entitymanagement.definitions.model.EntityRecord; +import eu.europeana.entitymanagement.definitions.model.Vocabulary; import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException; import eu.europeana.entitymanagement.mongo.repository.EntityRecordRepository; +import eu.europeana.entitymanagement.mongo.repository.VocabularyRepository; import eu.europeana.entitymanagement.solr.exception.SolrServiceException; import eu.europeana.entitymanagement.solr.service.SolrService; +import eu.europeana.entitymanagement.testutils.UnitTestUtils; +import eu.europeana.entitymanagement.web.xml.model.XmlBaseEntityImpl; +import eu.europeana.entitymanagement.web.xml.model.XmlConceptImpl; @SpringBootTest @Disabled("Excluded from automated runs") @@ -23,6 +30,11 @@ public class UtilityTests { @Autowired private EntityRecordRepository entityRecordRepository; + + @Autowired + VocabularyRepository vocabularyRepo; + + @Autowired protected JAXBContext jaxbContext; /** * This test can be used to reindex the local/other solr from mongo, when the schema fields change. @@ -40,5 +52,20 @@ public void reindexSolrFromMongo() throws EntityUpdateException { } } } + + //@Test + public void saveVocabulariesToMongo() throws Exception { + List> xmlEntities = MetisDereferenceUtils.parseMetisResponseMany( + jaxbContext.createUnmarshaller(), UnitTestUtils.loadFile("/metis-deref-unittest/roles.xml")); + for(XmlBaseEntityImpl xmlEntity : xmlEntities) { + XmlConceptImpl xmlConcept = (XmlConceptImpl) xmlEntity; + Concept concept = xmlConcept.toEntityModel(); + Vocabulary vocab = new Vocabulary(); + vocab.setVocabularyUri(concept.getEntityId()); + vocab.setInScheme(concept.getInScheme()); + vocab.setPrefLabel(concept.getPrefLabel()); + vocabularyRepo.save(vocab); + } + } } diff --git a/entity-management-web/src/test/resources/metis-deref-unittest/roles.xml b/entity-management-web/src/test/resources/metis-deref-unittest/roles.xml new file mode 100644 index 000000000..509bab698 --- /dev/null +++ b/entity-management-web/src/test/resources/metis-deref-unittest/roles.xml @@ -0,0 +1,30 @@ + + + + + Providing Institution + + + + + Aggregator + + + + + Accredited Aggregator + + + + \ No newline at end of file diff --git a/entity-management-web/src/test/resources/zoho_role_mapping.json b/entity-management-web/src/test/resources/zoho_role_mapping.json new file mode 100644 index 000000000..da1a058a9 --- /dev/null +++ b/entity-management-web/src/test/resources/zoho_role_mapping.json @@ -0,0 +1,4 @@ +[ + { + } +] diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java index 2208d30e2..fa926f4ac 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java @@ -8,10 +8,8 @@ import org.springframework.stereotype.Service; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.Gson; import com.zoho.crm.api.record.Record; import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration; import eu.europeana.entitymanagement.definitions.model.Entity; @@ -42,7 +40,8 @@ public Optional dereferenceEntityById(@NonNull String id) throws Excepti ZohoOrganizationConverter.convertToOrganizationEntity( zohoOrganization.get(), zohoConfiguration.getZohoBaseUrl(), - emConfig.getCountryMappings())); + emConfig.getCountryMappings(), + emConfig.getRoleMappings())); } else { return Optional.empty(); } diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java index d2c535d74..b16c99c3f 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java @@ -6,7 +6,6 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; @@ -15,9 +14,9 @@ import com.zoho.crm.api.record.Record; import com.zoho.crm.api.users.User; import eu.europeana.entitymanagement.definitions.model.Address; -import eu.europeana.entitymanagement.definitions.model.CountryMapping; import eu.europeana.entitymanagement.definitions.model.Organization; import eu.europeana.entitymanagement.definitions.model.WebResource; +import eu.europeana.entitymanagement.definitions.model.ZohoLabelUriMapping; import eu.europeana.entitymanagement.utils.EntityUtils; import eu.europeana.entitymanagement.zoho.utils.ZohoConstants; import eu.europeana.entitymanagement.zoho.utils.ZohoUtils; @@ -28,7 +27,8 @@ public class ZohoOrganizationConverter { private static final String POSITION_SEPARATOR = "_"; - public static Organization convertToOrganizationEntity(Record zohoRecord, String zohoBaseUrl, @NonNull final Map countryMappings) { + public static Organization convertToOrganizationEntity(Record zohoRecord, String zohoBaseUrl, @NonNull final Map countryMappings, + @NonNull final Map roleMappings) { Organization org = new Organization(); Long zohoId = zohoRecord.getId(); org.setAbout(ZohoUtils.buildZohoOrganizationId(zohoBaseUrl, zohoRecord.getId())); @@ -51,12 +51,19 @@ public static Organization convertToOrganizationEntity(Record zohoRecord, String String logoFieldName = ZohoConstants.LOGO_LINK_TO_WIKIMEDIACOMMONS_FIELD; org.setLogo(buildWebResource(zohoRecord, logoFieldName)); org.setHomepage(getStringFieldValue(zohoRecord, ZohoConstants.WEBSITE_FIELD)); - List organizationRoleStringList = + + List orgRoleLabels = ZohoUtils.stringListSupplier(zohoRecord.getKeyValue(ZohoConstants.ORGANIZATION_ROLE_FIELD)); - if (!organizationRoleStringList.isEmpty()) { - org.setEuropeanaRole( - ZohoUtils.createLanguageMapOfStringList( - Locale.ENGLISH.getLanguage(), organizationRoleStringList)); + if (!orgRoleLabels.isEmpty()) { + List orgRoleIds = new ArrayList<>(); + for(String roleLabel : orgRoleLabels) { + if(roleMappings.containsKey(roleLabel.toLowerCase())) { + orgRoleIds.add(roleMappings.get(roleLabel.toLowerCase())); + } + } + if(! orgRoleIds.isEmpty()) { + org.setEuropeanaRoleIds(orgRoleIds); + } } //build address object From deba06c2ebade8554b900598461cbe3eec305f38 Mon Sep 17 00:00:00 2001 From: GordeaS Date: Tue, 6 Feb 2024 17:38:59 +0100 Subject: [PATCH 22/22] load vocabulary at startup, revert solr schema, #EA-3641 #EA-3702 --- .../config/EntityManagementConfiguration.java | 33 ++++++++-- .../definitions/VocabularyFields.java | 2 +- .../definitions/model/Organization.java | 3 +- .../definitions/model/Vocabulary.java | 18 ++++-- .../web/xml/model/XmlOrganizationImpl.java | 20 ++---- .../repository/VocabularyRepository.java | 32 +++++++++- .../solr/model/SolrOrganization.java | 3 +- .../AbstractIntegrationTest.java | 3 +- .../service/ScheduledTaskServiceIT.java | 4 +- .../service/ScoringServiceIT.java | 4 +- .../entitymanagement/solr/SolrServiceIT.java | 4 +- .../web/BaseWebControllerTest.java | 4 +- .../web/EntityRegistrationIT.java | 2 + .../web/EntityRetrievalIT.java | 15 ++--- .../resources/solr-docker/conf/schema.xml | 16 +++-- .../entitymanagement/EntityManagementApp.java | 62 +++++++++++-------- .../{AppConfig.java => AppAutoconfig.java} | 51 +++++++++++++-- .../ApplicationInitializationException.java | 12 ++++ .../web/service/ConceptSchemeService.java | 4 +- .../web/service/EntityRecordService.java | 9 ++- .../web/service/MetisDereferenceService.java | 4 +- .../web/service/ZohoSyncService.java | 4 +- .../entitymanagement.user.properties.template | 6 +- .../resources/role_vocabulary.xml} | 0 .../entitymanagement/web/UtilityTests.java | 7 ++- .../test/resources/zoho_country_mapping.json | 4 -- .../src/test/resources/zoho_role_mapping.json | 4 -- 27 files changed, 217 insertions(+), 113 deletions(-) rename entity-management-web/src/main/java/eu/europeana/entitymanagement/config/{AppConfig.java => AppAutoconfig.java} (63%) create mode 100644 entity-management-web/src/main/java/eu/europeana/entitymanagement/exception/ApplicationInitializationException.java rename entity-management-web/src/{test/resources/metis-deref-unittest/roles.xml => main/resources/role_vocabulary.xml} (100%) delete mode 100644 entity-management-web/src/test/resources/zoho_country_mapping.json delete mode 100644 entity-management-web/src/test/resources/zoho_role_mapping.json diff --git a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java index 5489cf1cd..7af3f875c 100644 --- a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java +++ b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -21,6 +22,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.PropertySources; +import org.springframework.core.io.ClassPathResource; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import eu.europeana.entitymanagement.definitions.model.ZohoLabelUriMapping; @@ -165,11 +167,14 @@ public class EntityManagementConfiguration implements InitializingBean { @Value("${spring.profiles.active:}") private String activeProfileString; - @Value("${zoho.country.mapping:null}") + @Value("${zoho.country.mapping:zoho_country_mapping.json}") private String zohoCountryMappingFilename; - @Value("${zoho.role.mapping:null}") + @Value("${zoho.role.mapping:zoho_role_mapping.json}") private String zohoRoleMappingFilename; + + @Value("${europeana.role.vocabulary:role_vocabulary.xml}") + private String roleVocabularyFilename; Map countryMappings = new HashMap<>(); Map roleMappings = new HashMap<>(); @@ -224,7 +229,9 @@ private void verifyRequiredProperties() { } private void initCountryMappings() throws IOException { - try (InputStream inputStream = getClass().getResourceAsStream(getZohoCountryMappingFilename())) { + ClassPathResource resource = new ClassPathResource(getZohoCountryMappingFilename()); + + try (InputStream inputStream = resource.getInputStream()) { assert inputStream != null; try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { String contents = reader.lines().collect(Collectors.joining(System.lineSeparator())); @@ -237,7 +244,10 @@ private void initCountryMappings() throws IOException { } private void initRoleMappings() throws IOException { - try (InputStream inputStream = getClass().getResourceAsStream(getZohoRoleMappingFilename())) { + + ClassPathResource resource = new ClassPathResource(getZohoRoleMappingFilename()); + + try (InputStream inputStream = resource.getInputStream()) { assert inputStream != null; try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { String contents = reader.lines().collect(Collectors.joining(System.lineSeparator())); @@ -249,6 +259,17 @@ private void initRoleMappings() throws IOException { } } + public String loadRoleVocabulary() throws IOException { + ClassPathResource resource = new ClassPathResource(getRoleVocabularyFilename()); + + try (InputStream inputStream = resource.getInputStream()) { + assert inputStream != null; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { + return reader.lines().collect(Collectors.joining(System.lineSeparator())); + } + } + } + public String getPrSolrUrl() { return prSolrUrl; } @@ -436,4 +457,8 @@ public String getZohoRoleMappingFilename() { public Map getRoleMappings() { return roleMappings; } + + public String getRoleVocabularyFilename() { + return roleVocabularyFilename; + } } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/VocabularyFields.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/VocabularyFields.java index 17820921e..4fb92e997 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/VocabularyFields.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/VocabularyFields.java @@ -7,6 +7,6 @@ private VocabularyFields() { // private constructor to prevent instantiation } - public static final String VOCABULARY_URI = "vocabularyUri"; + public static final String URI = "uri"; } \ No newline at end of file diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java index 2b39e2e1b..335f267f3 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Organization.java @@ -95,7 +95,6 @@ public Organization(Organization copy) { if (copy.getPhone() != null) this.phone = new ArrayList<>(copy.getPhone()); if (copy.getMbox() != null) this.mbox = new ArrayList<>(copy.getMbox()); //because the europeanaRoleRef is a reference to the object we keep it the same (therefore also for europeanaRole) - this.europeanaRole = copy.getEuropeanaRole(); this.europeanaRoleRefs = copy.getEuropeanaRoleRefs(); if(copy.getEuropeanaRoleIds()!=null) this.europeanaRoleIds=new ArrayList<>(copy.getEuropeanaRoleIds()); //because the countryRef is a reference to the object we keep it the same (therefore also for country) @@ -272,7 +271,7 @@ public List getEuropeanaRole() { europeanaRole=new ArrayList<>(); for(String roleId : europeanaRoleIds) { Vocabulary vocab = new Vocabulary(); - vocab.setVocabularyUri(roleId); + vocab.setUri(roleId); europeanaRole.add(vocab); } } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Vocabulary.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Vocabulary.java index ee32500cc..63de01610 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Vocabulary.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/Vocabulary.java @@ -21,6 +21,9 @@ @JsonInclude(value = JsonInclude.Include.NON_EMPTY) @dev.morphia.annotations.Entity("Vocabulary") @EntityListeners(VocabularyWatcher.class) +/** + * class used for storing static vocabularies (e.g. europeana roles) + */ public class Vocabulary { private String type = "Vocabulary"; @@ -28,7 +31,7 @@ public class Vocabulary { @Id @JsonIgnore private ObjectId dbId; @Indexed(options = @IndexOptions(unique = true)) - private String vocabularyUri; + private String uri; protected List inScheme; @@ -40,6 +43,9 @@ public class Vocabulary { @JsonIgnore private Date modified; + /** + * default constructor + */ public Vocabulary() { } @@ -69,17 +75,17 @@ public Date getModified() { @Override public String toString() { - return String.format("Vocabulary.vocabularyUri: %s", getVocabularyUri()); + return String.format("Vocabulary.uri: %s", getUri()); } @JsonGetter - public String getVocabularyUri() { - return vocabularyUri; + public String getUri() { + return uri; } @JsonSetter - public void setVocabularyUri(String vocabularyUri) { - this.vocabularyUri = vocabularyUri; + public void setUri(String vocabularyUri) { + this.uri = vocabularyUri; } @JsonGetter(IN_SCHEME) diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java index 8bef05ad7..045f40a64 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/web/xml/model/XmlOrganizationImpl.java @@ -113,7 +113,7 @@ public XmlOrganizationImpl(Organization organization) { List orgXmlRole= new ArrayList<>(); for(Vocabulary vocab : orgRole) { XmlConceptImpl xmlConcept = new XmlConceptImpl(); - xmlConcept.setAbout(vocab.getVocabularyUri()); + xmlConcept.setAbout(vocab.getUri()); xmlConcept.setPrefLabel(RdfXmlUtils.convertMapToXmlMultilingualString(vocab.getPrefLabel())); xmlConcept.setInScheme(RdfXmlUtils.convertToRdfResource(vocab.getInScheme())); orgXmlRole.add(xmlConcept); @@ -149,30 +149,18 @@ public Organization toEntityModel() throws EntityModelCreationException { entity.setAcronym(RdfXmlUtils.toLanguageMapList(getAcronym())); entity.setDescription(RdfXmlUtils.toLanguageMap(getDescription())); entity.setLogo(XmlWebResourceWrapper.toWebResource(getLogo())); - //set europeanaRole + //set europeanaRole id (external dereferencers deliver only the ids, not transitive data) if(getEuropeanaRole()!=null && !getEuropeanaRole().isEmpty()) { - List roleIds=new ArrayList<>(); - List role=new ArrayList<>(); - for(XmlConceptImpl xmlConcept : getEuropeanaRole()) { - Vocabulary vocab=new Vocabulary(); - vocab.setVocabularyUri(xmlConcept.getAbout()); - vocab.setPrefLabel(RdfXmlUtils.toLanguageMap(xmlConcept.getPrefLabel())); - vocab.setInScheme(RdfXmlUtils.toStringList(xmlConcept.getInScheme())); - role.add(vocab); - roleIds.add(xmlConcept.getAbout()); - } - entity.setEuropeanaRole(role); + List roleIds = getEuropeanaRole().stream().map(e -> e.getAbout()).toList(); entity.setEuropeanaRoleIds(roleIds); } + //set country id (external dereferencers deliver only the ids, not transitive data) if(getCountry() != null) { - //the country is not saved in the database - entity.setCountry(getCountry().toEntityModel()); //we need to extract the countryID as well (xml about holds the entityId) entity.setCountryId(getCountry().getAbout()); } - if (getHomepage() != null) { entity.setHomepage(getHomepage().getResource()); } diff --git a/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/VocabularyRepository.java b/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/VocabularyRepository.java index 0e35213c5..a49537a3c 100644 --- a/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/VocabularyRepository.java +++ b/entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/VocabularyRepository.java @@ -1,7 +1,7 @@ package eu.europeana.entitymanagement.mongo.repository; import static dev.morphia.query.experimental.filters.Filters.in; -import static eu.europeana.entitymanagement.definitions.VocabularyFields.VOCABULARY_URI; +import static eu.europeana.entitymanagement.definitions.VocabularyFields.URI; import java.util.ArrayList; import java.util.List; import javax.annotation.Resource; @@ -18,24 +18,50 @@ public class VocabularyRepository { @Resource(name = AppConfigConstants.BEAN_EM_DATA_STORE) Datastore datastore; - public List findByVocabularyUris(List vocabularyUris) { + /** + * retrieve records by their URI/id + * @param vocabularyUris + * @return + */ + public List findByUri(List vocabularyUris) { List filters = new ArrayList<>(); - filters.add(in(VOCABULARY_URI, vocabularyUris)); + filters.add(in(URI, vocabularyUris)); return datastore.find(Vocabulary.class) .filter(filters.toArray(Filter[]::new)) .iterator() .toList(); } + /** + * save to database + * @param vocab record to save + * @return saved record + */ public Vocabulary save(Vocabulary vocab) { return datastore.save(vocab); } + /** + * save list of records to database + * @param vocabs list of records to save + * @return saved records + */ public List saveBulk(List vocabs) { return datastore.save(vocabs); } + /** + * clear database collection + */ public void dropCollection() { datastore.getMapper().getCollection(Vocabulary.class).drop(); } + + /** + * count the records available in the database + * @return number of database records + */ + public long countRecords() { + return datastore.find(Vocabulary.class).count(); + } } \ No newline at end of file diff --git a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java index 97b6e1def..669f1bb51 100644 --- a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java +++ b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java @@ -36,7 +36,8 @@ public class SolrOrganization extends SolrEntity { @Field(OrganizationSolrFields.FOAF_MBOX) private List mbox; - @Field(OrganizationSolrFields.EUROPEANA_ROLE) + //MAPPING TO BE ENABLED BACK IN THE NEXT VERSION - requires schema and ENtity API UPDATE + //@Field(OrganizationSolrFields.EUROPEANA_ROLE) private List europeanaRole; @Field(OrganizationSolrFields.COUNTRY) diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java index f33ed6119..ec5ea9068 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/AbstractIntegrationTest.java @@ -151,8 +151,7 @@ static void setProperties(DynamicPropertyRegistry registry) { //generate Europeana ID can be set to true when using the mock service, see TextConfig class registry.add("zoho.generate.organization.europeanaid", () -> true); - registry.add("zoho.country.mapping", () -> "/zoho_country_mapping_test.json"); - registry.add("zoho.role.mapping", () -> "/zoho_role_mapping_test.json"); + registry.add("zoho.country.mapping", () -> "zoho_country_mapping_test.json"); // could be used to fix eclipse issues registry.add("scmBranch", () -> "dev"); diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScheduledTaskServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScheduledTaskServiceIT.java index 76b510463..eb975acd7 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScheduledTaskServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScheduledTaskServiceIT.java @@ -14,7 +14,7 @@ import eu.europeana.entitymanagement.batch.repository.FailedTaskRepository; import eu.europeana.entitymanagement.batch.service.ScheduledTaskService; import eu.europeana.entitymanagement.batch.utils.BatchUtils; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord; import eu.europeana.entitymanagement.definitions.batch.model.FailedTask; import eu.europeana.entitymanagement.definitions.batch.model.FailedTask.Builder; @@ -63,7 +63,7 @@ class ScheduledTaskServiceIT extends AbstractIntegrationTest { @Autowired EntityUpdateJobConfig updateJobConfig; - @Qualifier(AppConfig.BEAN_EM_SOLR_SERVICE) + @Qualifier(AppAutoconfig.BEAN_EM_SOLR_SERVICE) @Autowired private SolrService emSolrService; diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScoringServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScoringServiceIT.java index a78c01c61..4ebb4d700 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScoringServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/ScoringServiceIT.java @@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import eu.europeana.entitymanagement.AbstractIntegrationTest; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.definitions.model.Agent; import eu.europeana.entitymanagement.definitions.model.Place; import eu.europeana.entitymanagement.definitions.model.TimeSpan; @@ -26,7 +26,7 @@ @AutoConfigureMockMvc class ScoringServiceIT extends AbstractIntegrationTest { - @Resource(name = AppConfig.BEAN_EM_SCORING_SERVICE) + @Resource(name = AppAutoconfig.BEAN_EM_SCORING_SERVICE) ScoringService scoringService; // @Test diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/solr/SolrServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/solr/SolrServiceIT.java index d4fb9d432..a9b93ef9d 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/solr/SolrServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/solr/SolrServiceIT.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.europeana.entitymanagement.AbstractIntegrationTest; import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.definitions.model.Agent; import eu.europeana.entitymanagement.definitions.model.Aggregation; import eu.europeana.entitymanagement.definitions.model.Concept; @@ -48,7 +48,7 @@ public class SolrServiceIT extends AbstractIntegrationTest { // private static final String RIGHTS_PD = "https://creativecommons.org/publicdomain/zero/1.0/"; - @Qualifier(AppConfig.BEAN_EM_SOLR_SERVICE) + @Qualifier(AppAutoconfig.BEAN_EM_SOLR_SERVICE) @Autowired private SolrService emSolrService; diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java index 0476b51d4..aa99ec23e 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java @@ -23,7 +23,7 @@ import eu.europeana.entitymanagement.batch.service.EntityUpdateService; import eu.europeana.entitymanagement.batch.service.ScheduledTaskService; import eu.europeana.entitymanagement.common.config.DataSource; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTask; import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTaskType; import eu.europeana.entitymanagement.definitions.model.Entity; @@ -49,7 +49,7 @@ abstract class BaseWebControllerTest extends AbstractIntegrationTest { @Autowired private EntityUpdateService entityUpdateService; - @Qualifier(AppConfig.BEAN_EM_SOLR_SERVICE) + @Qualifier(AppAutoconfig.BEAN_EM_SOLR_SERVICE) @Autowired protected SolrService emSolrService; diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java index 591c77ff9..df9896450 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java @@ -330,6 +330,8 @@ void registerZohoOrganizationBnfWithNewFieldsShouldBeSuccessful() throws Excepti .andExpect(jsonPath("$.language", everyItem(matchesRegex("[a-z]+")))) .andExpect(jsonPath("$.hiddenLabel", hasSize(3))) .andExpect(jsonPath("$.id", any(String.class))); + // excluded as the wikidata hasGeo is not consolidated + //.andExpect(jsonPath("$.hasAddress.hasGeo").isNotEmpty()) } @Test diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java index 411df4cd7..d547cc3ad 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java @@ -498,13 +498,6 @@ public void retrieveOrganizationJsonExternalWithCountryAndRoleDereference() thro String metisResponse = loadFile(IntegrationTestUtils.PLACE_SWEDEN_XML); createEntity(europeanaMetadata, metisResponse, IntegrationTestUtils.PLACE_SWEDEN_URI); - //2. create a vocabulary for the europeanaRole - Vocabulary vocab = new Vocabulary(); - vocab.setVocabularyUri("http://data.europeana.eu/vocabulary/role/ProvidingInstitution"); - vocab.setInScheme(List.of("http://data.europeana.eu/vocabulary/role")); - vocab.setPrefLabel(Map.of("en", "Providing Institution")); - vocabRepository.save(vocab); - // //forcefully change the country mapping uri to the right one // List countryMap= entityRecordService.getCountryMapping(); // for(CountryMapping cm : countryMap) { @@ -513,7 +506,7 @@ public void retrieveOrganizationJsonExternalWithCountryAndRoleDereference() thro // } // } - //3. register zoho GFM org + //2. register zoho GFM org europeanaMetadata = loadFile(IntegrationTestUtils.ORGANIZATION_REGISTER_GFM_ZOHO_JSON); Optional zohoRecord = IntegrationTestUtils.getZohoOrganizationRecord( @@ -523,12 +516,12 @@ public void retrieveOrganizationJsonExternalWithCountryAndRoleDereference() thro String entityId = createOrganization(europeanaMetadata, zohoRecord.get()).getEntityId(); String requestPath = getEntityRequestPath(entityId); - mockMvc + ResultActions resultActions = mockMvc .perform( get(IntegrationTestUtils.BASE_SERVICE_URL + "/" + requestPath + ".jsonld") .param(WebEntityConstants.QUERY_PARAM_PROFILE, "external, dereference") - .accept(MediaType.APPLICATION_JSON)) - .andExpect(status().isOk()) + .accept(MediaType.APPLICATION_JSON)); + resultActions.andExpect(status().isOk()) .andExpect(jsonPath("$.id", is(entityId))) .andExpect(jsonPath("$.type", is(EntityTypes.Organization.getEntityType()))) .andExpect(jsonPath("$.sameAs").isNotEmpty()) diff --git a/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml b/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml index 8d83e5388..720ed19cb 100644 --- a/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml +++ b/entity-management-tests/src/integration-test/resources/solr-docker/conf/schema.xml @@ -485,11 +485,15 @@ - + + + + @@ -573,7 +577,8 @@ dest="text.*" /> - + + @@ -587,7 +592,8 @@ dest="text" /> - + + diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/EntityManagementApp.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/EntityManagementApp.java index 4a27f9140..4ba75eeb8 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/EntityManagementApp.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/EntityManagementApp.java @@ -19,7 +19,9 @@ import eu.europeana.entitymanagement.batch.service.BatchEntityUpdateExecutor; import eu.europeana.entitymanagement.batch.service.ScheduledTaskService; import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants; +import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException; import eu.europeana.entitymanagement.web.model.ZohoSyncReport; +import eu.europeana.entitymanagement.web.service.EntityRecordService; import eu.europeana.entitymanagement.web.service.ZohoSyncService; /** @@ -39,6 +41,8 @@ public class EntityManagementApp implements CommandLineRunner { private BatchEntityUpdateExecutor batchUpdateExecutor; @Autowired private ZohoSyncService zohoSyncService; + @Autowired + private EntityRecordService entityRecordService; /** * Main entry point of this application @@ -51,7 +55,7 @@ public class EntityManagementApp implements CommandLineRunner { */ public static void main(String[] args) { // jobType = args.length > 0 ? args[0] : ""; - if (hasCmdLineParams(args)) { + if (isScheduledTask(args)) { if (LOG.isInfoEnabled()) { LOG.info("Starting batch updates execution with args: {}", Arrays.toString(args)); } @@ -92,6 +96,10 @@ public static void main(String[] args) { } } + static boolean isScheduledTask(String[] args) { + return hasCmdLineParams(args); + } + static ScheduledTaskService getScheduledTasksService(ConfigurableApplicationContext context) { return (ScheduledTaskService) context .getBean(AppConfigConstants.BEAN_BATCH_SCHEDULED_TASK_SERVICE); @@ -108,34 +116,38 @@ static boolean hasCmdLineParams(String[] args) { @Override public void run(String... args) throws Exception { - if (hasCmdLineParams(args)) { - Set tasks = Set.of(args); - - // first zoho sync as it runs synchronuous operations - if (tasks.contains(JobType.ZOHO_SYNC.value())) { - LOG.info("Executing zoho sync"); - ZohoSyncReport zohoSyncReport = zohoSyncService.synchronizeModifiedZohoOrganizations(); - LOG.info("Synchronization Report: {}", zohoSyncReport.toString()); - } + if (isScheduledTask(args)) { + runScheduledTasks(args); + } + // if no arguments then web server should be started + return; + } - if (tasks.contains(JobType.SCHEDULE_DELETION.value())) { - // run also the deletions called through the API directly - LOG.info("Executing scheduled deletions"); - batchUpdateExecutor.runScheduledDeprecationsAndDeletions(); - // TODO: should read the number of scheduled deletions and deprecations from the database - // and write it to the logs - } - if (tasks.contains(JobType.SCHEDULE_UPDATE.value())) { - LOG.info("Executing scheduled updates"); - batchUpdateExecutor.runScheduledUpdate(); - // TODO: should read the number of scheduled deletions and deprecations from the database - // and write it to the logs - } + void runScheduledTasks(String... args) throws EntityUpdateException { + Set tasks = Set.of(args); + // first zoho sync as it runs synchronuous operations + if (tasks.contains(JobType.ZOHO_SYNC.value())) { + LOG.info("Executing zoho sync"); + ZohoSyncReport zohoSyncReport = zohoSyncService.synchronizeModifiedZohoOrganizations(); + LOG.info("Synchronization Report: {}", zohoSyncReport.toString()); + } + + if (tasks.contains(JobType.SCHEDULE_DELETION.value())) { + // run also the deletions called through the API directly + LOG.info("Executing scheduled deletions"); + batchUpdateExecutor.runScheduledDeprecationsAndDeletions(); + // TODO: should read the number of scheduled deletions and deprecations from the database + // and write it to the logs + } + + if (tasks.contains(JobType.SCHEDULE_UPDATE.value())) { + LOG.info("Executing scheduled updates"); + batchUpdateExecutor.runScheduledUpdate(); + // TODO: should read the number of scheduled deletions and deprecations from the database + // and write it to the logs } - // if no arguments then web server should be started - return; } /** validates the arguments passed */ diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/config/AppConfig.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/config/AppAutoconfig.java similarity index 63% rename from entity-management-web/src/main/java/eu/europeana/entitymanagement/config/AppConfig.java rename to entity-management-web/src/main/java/eu/europeana/entitymanagement/config/AppAutoconfig.java index a3fb651e1..7e063926b 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/config/AppConfig.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/config/AppAutoconfig.java @@ -5,9 +5,12 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import java.util.stream.Collectors; import javax.annotation.PostConstruct; import javax.annotation.Resource; +import javax.xml.bind.JAXBContext; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.context.MessageSource; @@ -22,32 +25,70 @@ import eu.europeana.entitymanagement.common.config.DataSource; import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration; import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants; +import eu.europeana.entitymanagement.definitions.model.Vocabulary; +import eu.europeana.entitymanagement.exception.ApplicationInitializationException; +import eu.europeana.entitymanagement.mongo.repository.VocabularyRepository; +import eu.europeana.entitymanagement.web.MetisDereferenceUtils; +import eu.europeana.entitymanagement.web.xml.model.RdfXmlUtils; +import eu.europeana.entitymanagement.web.xml.model.XmlBaseEntityImpl; +import eu.europeana.entitymanagement.web.xml.model.XmlConceptImpl; /** @author GordeaS */ @Configuration -public class AppConfig extends AppConfigConstants { +public class AppAutoconfig extends AppConfigConstants { - private static final Logger LOG = LogManager.getLogger(AppConfig.class); + private static final Logger LOG = LogManager.getLogger(AppAutoconfig.class); @Resource private EntityManagementConfiguration emConfiguration; @Resource(name = BEAN_XML_MAPPER) private XmlMapper xmlMapper; - - public AppConfig() { + + @Resource(name = BEAN_VOCABULARY_REPO) + private VocabularyRepository vocabRepository; + + @Resource protected JAXBContext jaxbContext; + + public AppAutoconfig() { LOG.info("Initializing EntityManagementConfiguration bean as: configuration"); } @PostConstruct - public void init() { + public void init() throws ApplicationInitializationException { if (emConfiguration.isAuthReadEnabled() || emConfiguration.isAuthWriteEnabled()) { String jwtTokenSignatureKey = emConfiguration.getApiKeyPublicKey(); if (jwtTokenSignatureKey == null || jwtTokenSignatureKey.isBlank()) { throw new IllegalStateException("The jwt token signature key cannot be null or empty."); } } + //ensure data + ensureDatabaseInitialization(); + } + public void ensureDatabaseInitialization() throws ApplicationInitializationException { + if(vocabRepository.countRecords() < 1) { + List> xmlEntities; + try { + xmlEntities = MetisDereferenceUtils.parseMetisResponseMany( + jaxbContext.createUnmarshaller(), emConfiguration.loadRoleVocabulary()); + } catch (Exception e) { + throw new ApplicationInitializationException("Cannot load vocabulary from resources!", e); + } + + List roles = new ArrayList<>(xmlEntities.size()); + for(XmlBaseEntityImpl xmlEntity : xmlEntities) { + XmlConceptImpl xmlConcept = (XmlConceptImpl) xmlEntity; + Vocabulary vocab = new Vocabulary(); + vocab.setUri(xmlConcept.getAbout()); + vocab.setInScheme(RdfXmlUtils.toStringList(xmlConcept.getInScheme())); + vocab.setPrefLabel(RdfXmlUtils.toLanguageMap(xmlConcept.getPrefLabel())); + roles.add(vocab); + } + vocabRepository.saveBulk(roles); + } + } + @Bean(name = BEAN_EM_DATA_SOURCES) public DataSources getDataSources() throws IOException { String datasourcesXMLConfigFile = emConfiguration.getDatasourcesXMLConfig(); diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/exception/ApplicationInitializationException.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/exception/ApplicationInitializationException.java new file mode 100644 index 000000000..ecf68800b --- /dev/null +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/exception/ApplicationInitializationException.java @@ -0,0 +1,12 @@ +package eu.europeana.entitymanagement.exception; + +public class ApplicationInitializationException extends Exception { + + public ApplicationInitializationException(String message) { + super(message); + } + + public ApplicationInitializationException(String message, Throwable th) { + super(message, th); + } +} diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ConceptSchemeService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ConceptSchemeService.java index 859f71fbe..76d29112f 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ConceptSchemeService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ConceptSchemeService.java @@ -7,7 +7,7 @@ import org.springframework.stereotype.Service; import eu.europeana.api.commons.error.EuropeanaApiException; import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.definitions.model.ConceptScheme; import eu.europeana.entitymanagement.exception.EntityNotFoundException; import eu.europeana.entitymanagement.exception.EntityRemovedException; @@ -18,7 +18,7 @@ import eu.europeana.entitymanagement.utils.EntityUtils; import eu.europeana.entitymanagement.vocabulary.EntityTypes; -@Service(AppConfig.BEAN_CONCEPT_SCHEME_SERVICE) +@Service(AppAutoconfig.BEAN_CONCEPT_SCHEME_SERVICE) public class ConceptSchemeService { private final ConceptSchemeRepository emConceptSchemeRepo; diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java index c0f1d9f66..eb642ba58 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java @@ -33,7 +33,7 @@ import eu.europeana.api.commons.error.EuropeanaApiException; import eu.europeana.entitymanagement.common.config.DataSource; import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.common.vocabulary.AppConfigConstants; import eu.europeana.entitymanagement.config.DataSources; import eu.europeana.entitymanagement.definitions.exceptions.EntityModelCreationException; import eu.europeana.entitymanagement.definitions.exceptions.UnsupportedEntityTypeException; @@ -77,7 +77,7 @@ import eu.europeana.entitymanagement.zoho.utils.ZohoException; import eu.europeana.entitymanagement.zoho.utils.ZohoUtils; -@Service(AppConfig.BEAN_ENTITY_RECORD_SERVICE) +@Service(AppConfigConstants.BEAN_ENTITY_RECORD_SERVICE) public class EntityRecordService { private final EntityRecordRepository entityRecordRepository; @@ -176,7 +176,7 @@ private void dereferenceLinkedEntities(Organization org) { } //dereference role if(org.getEuropeanaRoleIds()!=null && !org.getEuropeanaRoleIds().isEmpty()) { - org.setEuropeanaRole(vocabRepository.findByVocabularyUris(org.getEuropeanaRoleIds())); + org.setEuropeanaRole(vocabRepository.findByUri(org.getEuropeanaRoleIds())); } } @@ -1288,7 +1288,6 @@ && isStringOrPrimitive(mapFirstValue.getClass())) { public void dropRepository() { this.entityRecordRepository.dropCollection(); - this.vocabRepository.dropCollection(); } /** @@ -1485,7 +1484,7 @@ public void processReferenceFields(Entity entity) { } //role reference if(org.getEuropeanaRoleIds()!=null && !org.getEuropeanaRoleIds().isEmpty()) { - List vocabs=vocabRepository.findByVocabularyUris(org.getEuropeanaRoleIds()); + List vocabs=vocabRepository.findByUri(org.getEuropeanaRoleIds()); if (vocabs.isEmpty()) { logger.info( "No vocabularies with the uris: {} were found in the database. Cannot assign role reference to organization with id {}", diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/MetisDereferenceService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/MetisDereferenceService.java index c6333a529..d65632a78 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/MetisDereferenceService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/MetisDereferenceService.java @@ -4,7 +4,7 @@ import static eu.europeana.entitymanagement.web.MetisDereferenceUtils.parseMetisResponse; import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.definitions.model.Entity; import eu.europeana.entitymanagement.dereference.Dereferencer; import eu.europeana.entitymanagement.exception.DatasourceNotReachableException; @@ -37,7 +37,7 @@ import reactor.netty.http.client.HttpClient; /** Handles de-referencing entities from Metis. */ -@Service(AppConfig.BEAN_METIS_DEREF_SERVICE) +@Service(AppAutoconfig.BEAN_METIS_DEREF_SERVICE) public class MetisDereferenceService implements InitializingBean, Dereferencer { private static final Logger logger = LogManager.getLogger(MetisDereferenceService.class); diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java index 682774478..7ac877a0d 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java @@ -19,7 +19,7 @@ import eu.europeana.api.commons.web.model.vocabulary.Operations; import eu.europeana.entitymanagement.batch.service.EntityUpdateService; import eu.europeana.entitymanagement.common.config.EntityManagementConfiguration; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.config.DataSources; import eu.europeana.entitymanagement.definitions.model.EntityRecord; import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException; @@ -36,7 +36,7 @@ import eu.europeana.entitymanagement.zoho.utils.ZohoConstants; import eu.europeana.entitymanagement.zoho.utils.ZohoException; -@Service(AppConfig.BEAN_ZOHO_SYNC_SERVICE) +@Service(AppAutoconfig.BEAN_ZOHO_SYNC_SERVICE) public class ZohoSyncService extends BaseZohoAccess { @Autowired diff --git a/entity-management-web/src/main/resources/entitymanagement.user.properties.template b/entity-management-web/src/main/resources/entitymanagement.user.properties.template index 15c8a4fe3..11a60ed86 100644 --- a/entity-management-web/src/main/resources/entitymanagement.user.properties.template +++ b/entity-management-web/src/main/resources/entitymanagement.user.properties.template @@ -48,5 +48,7 @@ europeana.searchapi.urlPrefix=https:///record/v2/search.json?wskey #enable/disable generation of entity ids for organizations (should be enabled only in productive environment) #zoho.generate.organization.europeanaid=false -zoho.country.mapping= -zoho.role.mapping= \ No newline at end of file +#configuration files for zoho country and role mappings +#zoho.country.mapping=zoho_country_mapping.json +#zoho.role.mapping=zoho_role_mapping.json +#europeana.role.vocabulary=role_vocabulary.xml \ No newline at end of file diff --git a/entity-management-web/src/test/resources/metis-deref-unittest/roles.xml b/entity-management-web/src/main/resources/role_vocabulary.xml similarity index 100% rename from entity-management-web/src/test/resources/metis-deref-unittest/roles.xml rename to entity-management-web/src/main/resources/role_vocabulary.xml diff --git a/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java b/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java index 893be8f31..3de59b956 100644 --- a/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java +++ b/entity-management-web/src/test/java/eu/europeana/entitymanagement/web/UtilityTests.java @@ -7,7 +7,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.test.context.SpringBootTest; -import eu.europeana.entitymanagement.config.AppConfig; +import eu.europeana.entitymanagement.config.AppAutoconfig; import eu.europeana.entitymanagement.definitions.model.Concept; import eu.europeana.entitymanagement.definitions.model.EntityRecord; import eu.europeana.entitymanagement.definitions.model.Vocabulary; @@ -24,7 +24,7 @@ @Disabled("Excluded from automated runs") public class UtilityTests { - @Qualifier(AppConfig.BEAN_EM_SOLR_SERVICE) + @Qualifier(AppAutoconfig.BEAN_EM_SOLR_SERVICE) @Autowired private SolrService emSolrService; @@ -54,6 +54,7 @@ public void reindexSolrFromMongo() throws EntityUpdateException { } //@Test + @Disabled("not needed, the vocabulary is automatically loaded at start time") public void saveVocabulariesToMongo() throws Exception { List> xmlEntities = MetisDereferenceUtils.parseMetisResponseMany( jaxbContext.createUnmarshaller(), UnitTestUtils.loadFile("/metis-deref-unittest/roles.xml")); @@ -61,7 +62,7 @@ public void saveVocabulariesToMongo() throws Exception { XmlConceptImpl xmlConcept = (XmlConceptImpl) xmlEntity; Concept concept = xmlConcept.toEntityModel(); Vocabulary vocab = new Vocabulary(); - vocab.setVocabularyUri(concept.getEntityId()); + vocab.setUri(concept.getEntityId()); vocab.setInScheme(concept.getInScheme()); vocab.setPrefLabel(concept.getPrefLabel()); vocabularyRepo.save(vocab); diff --git a/entity-management-web/src/test/resources/zoho_country_mapping.json b/entity-management-web/src/test/resources/zoho_country_mapping.json deleted file mode 100644 index f8831b757..000000000 --- a/entity-management-web/src/test/resources/zoho_country_mapping.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - { - } -] \ No newline at end of file diff --git a/entity-management-web/src/test/resources/zoho_role_mapping.json b/entity-management-web/src/test/resources/zoho_role_mapping.json deleted file mode 100644 index da1a058a9..000000000 --- a/entity-management-web/src/test/resources/zoho_role_mapping.json +++ /dev/null @@ -1,4 +0,0 @@ -[ - { - } -]