europeanaRole and country fields adjusted to the zoho eu

entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/DataSource.java

@@ -9,8 +9,8 @@ public class DataSource {
   public static final String FREQ_STATIC = "static";
   public static final String EUROPEANA_ID = "europeana";
   public static final String ZOHO_ID = "zoho-crm";
-  public static final String ZOHO_HOST = "crm.zoho.com";
-//  public static final String ZOHO_HOST = "crm.zoho.eu";
+//  public static final String ZOHO_HOST = "crm.zoho.com";
+  public static final String ZOHO_HOST = "crm.zoho.eu";
 
   @JacksonXmlProperty(isAttribute = true)
   private String url;

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/ZohoMapping.java

@@ -3,8 +3,6 @@
 import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ENTITY_URI;
 import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.WIKIDATA_URI;
 import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ZOHO_LABEL;
-import java.util.Arrays;
-import java.util.List;
 import com.fasterxml.jackson.annotation.JsonGetter;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
@@ -52,13 +50,13 @@ public void setWikidataUri(String wikidataUri) {
     this.wikidataUri = wikidataUri;
   }
 
-  public static String getEntityUriFromName(List<ZohoMapping> list ,String name) {
-    for(ZohoMapping cm : list) {
-      List<String> splittedAndTrimmed = Arrays.stream(cm.getZohoLabel().split(",")).map(String::trim).toList();
-      if(splittedAndTrimmed.contains(name)) {
-        return cm.getEntityUri();
-      }
-    }
-    return null;
-  }
+//  public static String getEntityUriFromName(List<ZohoMapping> list ,String name) {
+//    for(ZohoMapping cm : list) {
+//      List<String> splittedAndTrimmed = Arrays.stream(cm.getZohoLabel().split(",")).map(String::trim).toList();
+//      if(splittedAndTrimmed.contains(name)) {
+//        return cm.getEntityUri();
+//      }
+//    }
+//    return null;
+//  }
 }

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/normalization/EntityFieldsCleaner.java

@@ -89,13 +89,15 @@ public void cleanAndNormalize(Entity entity) {
           String[] normalized = normalizedList.toArray(new String[0]);
           entity.setFieldValue(field, normalized);
         } else if (fieldType.isAssignableFrom(List.class)) {
-          List<String> fieldValueList = (List<String>) fieldValue;
-          if (fieldValueList.isEmpty()) {
+          List<?> fieldValueList = (List<?>) fieldValue;
+          if(fieldValueList.isEmpty() || !(fieldValueList.get(0) instanceof String)) {
             continue;
           }
+
+          List<String> fieldValueListString = (List<String>) fieldValue;
 
           // remove spaces from the List<String> fields
-          List<String> normalizedList = normalizeValues(field.getName(), fieldValueList);
+          List<String> normalizedList = normalizeValues(field.getName(), fieldValueListString);
 
           // if Entity field is supposed to contain a single element, remove all elements
           // except the
@@ -125,7 +127,7 @@ public void cleanAndNormalize(Entity entity) {
           e);
     }
   }
-
+  
   private WebResource normalizeWebResource(WebResource existingFieldValue) {
     WebResource webResource = new WebResource(existingFieldValue);
 

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/vocabulary/WebEntityFields.java

@@ -9,8 +9,8 @@ public interface WebEntityFields {
   public static final String LANGUAGE_EN = "en";
   public static final String BASE_DATA_EUROPEANA_URI = "http://data.europeana.eu/";
   public static final String WIKIDATA_HOST = "www.wikidata.org";
-  public static final String ZOHO_CRM_HOST = "crm.zoho.com";
-//  public static final String ZOHO_CRM_HOST = "crm.zoho.eu";
+//  public static final String ZOHO_CRM_HOST = "crm.zoho.com";
+  public static final String ZOHO_CRM_HOST = "crm.zoho.eu";
 
   // common fields
   public static final String ID = "id";

entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/VocabularyRepository.java

@@ -35,4 +35,7 @@ public List<Vocabulary> saveBulk(List<Vocabulary> vocabs) {
     return datastore.save(vocabs);
   }
 
+  public void dropCollection() {
+    datastore.getMapper().getCollection(Vocabulary.class).drop();
+  }
 }

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java

@@ -169,15 +169,15 @@ public class IntegrationTestUtils {
   public static final String INVALID_MIGRATION_ID = "http://www.testing.org/entity/testing";
 
   public static final String ORGANIZATION_BNF_URI_ZOHO =
-      "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000002112001";
+      "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/1482250000002112001";
   public static final String ORGANIZATION_NATURALIS_URI_ZOHO =
-      "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000000370517";
+      "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/1482250000000370517";
   public static final String ORGANIZATION_PCCE_URI_ZOHO =
-      "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000000338555";
+      "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/1482250000000338555";
   public static final String ORGANIZATION_GFM_URI_ZOHO =
-      "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000004503618";
+      "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/1482250000004503618";
   public static final String ORGANIZATION_BERGER_MUSEUM_URI_ZOHO =
-      "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000004477407";
+      "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/1482250000004477407";
   public static final String ORGANIZATION_NATURALIS_URI_WIKIDATA_PATH_SUFFIX = "/entity/Q641676";
   public static final String ORGANIZATION_NATURALIS_URI_WIKIDATA_URI =
       WIKIDATA_BASE_URI + ORGANIZATION_NATURALIS_URI_WIKIDATA_PATH_SUFFIX;

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java

@@ -15,7 +15,7 @@
 public class TestConfig {
 
 
-  public static final String MOCK_ZOHO_BASE_URL = "https://crm.zoho.com/crm/org51823723/tab/Accounts/";
+  public static final String MOCK_ZOHO_BASE_URL = "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/";
   public static final String MOCK_ZOHO_COUNTRY_MAPPING_FILE = "/zoho_country_mapping_test.json";
 
   /**

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/BaseWebControllerTest.java

@@ -30,6 +30,7 @@
 import eu.europeana.entitymanagement.definitions.model.EntityRecord;
 import eu.europeana.entitymanagement.definitions.model.Organization;
 import eu.europeana.entitymanagement.exception.ingestion.EntityUpdateException;
+import eu.europeana.entitymanagement.mongo.repository.VocabularyRepository;
 import eu.europeana.entitymanagement.solr.exception.SolrServiceException;
 import eu.europeana.entitymanagement.solr.service.SolrService;
 import eu.europeana.entitymanagement.testutils.IntegrationTestUtils;
@@ -53,6 +54,8 @@ abstract class BaseWebControllerTest extends AbstractIntegrationTest {
   protected SolrService emSolrService;
 
   @Autowired private WebApplicationContext webApplicationContext;
+
+  @Autowired protected VocabularyRepository vocabRepository;
 
   @BeforeEach
   protected void setup() throws Exception {

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRetrievalIT.java

@@ -14,6 +14,7 @@
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.xpath;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import org.hamcrest.Matchers;
@@ -28,6 +29,7 @@
 import eu.europeana.entitymanagement.batch.service.FailedTaskService;
 import eu.europeana.entitymanagement.definitions.batch.model.ScheduledUpdateType;
 import eu.europeana.entitymanagement.definitions.model.EntityRecord;
+import eu.europeana.entitymanagement.definitions.model.Vocabulary;
 import eu.europeana.entitymanagement.testutils.IntegrationTestUtils;
 import eu.europeana.entitymanagement.vocabulary.EntityTypes;
 import eu.europeana.entitymanagement.vocabulary.FailedTaskJsonFields;
@@ -490,12 +492,19 @@ public void retrievePlaceExternalSchemaOrgShouldBeSuccessful() throws Exception
   }
 
   @Test
-  public void retrieveOrganizationJsonExternalWithCountryDereference() throws Exception {
+  public void retrieveOrganizationJsonExternalWithCountryAndRoleDereference() throws Exception {
     //1. create a place "Sweden" to be used to dereference zoho country for the zoho GFM org
     String europeanaMetadata = loadFile(IntegrationTestUtils.PLACE_REGISTER_SWEDEN_JSON);
     String metisResponse = loadFile(IntegrationTestUtils.PLACE_SWEDEN_XML);
     createEntity(europeanaMetadata, metisResponse, IntegrationTestUtils.PLACE_SWEDEN_URI);
 
+    //2. create a vocabulary for the europeanaRole
+    Vocabulary vocab = new Vocabulary();
+    vocab.setVocabularyUri("http://data.europeana.eu/vocabulary/role/ProvidingInstitution");
+    vocab.setInScheme(List.of("http://data.europeana.eu/vocabulary/role"));
+    vocab.setPrefLabel(Map.of("en", "Providing Institution"));
+    vocabRepository.save(vocab);
+
 //    //forcefully change the country mapping uri to the right one
 //    List<CountryMapping> countryMap= entityRecordService.getCountryMapping();
 //    for(CountryMapping cm : countryMap) {
@@ -524,7 +533,9 @@ public void retrieveOrganizationJsonExternalWithCountryDereference() throws Exce
         .andExpect(jsonPath("$.type", is(EntityTypes.Organization.getEntityType())))
         .andExpect(jsonPath("$.sameAs").isNotEmpty())
         .andExpect(jsonPath("$.countryId", is("http://data.europeana.eu/place/1")))
-        .andExpect(jsonPath("$.countryPlace.prefLabel.en", is("Sweden")));
+        .andExpect(jsonPath("$.europeanaRole[0]", is("http://data.europeana.eu/vocabulary/role/ProvidingInstitution")))
+        .andExpect(jsonPath("$.countryPlace.prefLabel.en", is("Sweden")))
+        .andExpect(jsonPath("$.europeanaRoleVocabularies[0].prefLabel.en", is("Providing Institution")));
   }
 
   @Test

entity-management-tests/src/integration-test/resources/content/organization_register_zoho_berger_museum.json

@@ -1,4 +1,4 @@
 {
   "type": "Organization",
-  "id": "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000004477407"
+  "id": "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/1482250000004477407"
 }

entity-management-tests/src/integration-test/resources/content/organization_register_zoho_bnf.json

@@ -1,6 +1,6 @@
 {
   "type": "Organization",
-  "id": "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000002112001",
+  "id": "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/1482250000002112001",
   "prefLabel": {
     "fr": "Bibliothèque nationale de France"
   },

entity-management-tests/src/integration-test/resources/content/organization_register_zoho_gfm.json

@@ -1,4 +1,4 @@
 {
   "type": "Organization",
-  "id": "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000004503618"
+  "id": "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/1482250000004503618"
 }

entity-management-tests/src/integration-test/resources/content/organization_register_zoho_naturalis.json

@@ -1,6 +1,6 @@
 {
   "type": "Organization",
-  "id": "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000000370517",
+  "id": "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/1482250000000370517",
   "prefLabel": {
     "fr": "Nauralis"
   },

entity-management-tests/src/integration-test/resources/content/organization_register_zoho_pcce.json

@@ -1,4 +1,4 @@
 {
   "type": "Organization",
-  "id": "https://crm.zoho.com/crm/org51823723/tab/Accounts/1482250000000338555"
+  "id": "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/1482250000000338555"
 }

entity-management-tests/src/integration-test/resources/zoho-deref/organization_zoho_berger_museum_response.json

@@ -10,11 +10,11 @@
   "DEA_Signed": true,
   "Account_Name": "Berger museum",
   "Sector": "Cultural Heritage",
-  "Organisation_Role2": [
+  "Organisation_Role": [
     "Data Provider"
   ],
   "SameAs_1": "http://www.wikidata.org/entity/Q23014463",
-  "Country": "Norway",
+  "Organisation_Country": "Norway, NO",
   "Tag": [],
   "Latitude": "-1",
   "Longitude": "-1"

entity-management-tests/src/integration-test/resources/zoho-deref/organization_zoho_bnf_response.json

@@ -13,7 +13,7 @@
 	"Alternative_1": "FR(French)",
 	"City": "Paris",
 	"Longitude": "2.3758333",
-	"Country": "France",
+	"Organisation_Country": "France, FR",
 	"Description": "The Bibliothèque nationale de France (BnF; French: [bi.bli.jɔ.tɛk na.sjɔ.nal də fʁɑ̃s]) is the National Library of France, located in Paris. It is the national repository of all that is published in France and also holds extensive historical collections.",
 	"Geographic_Level": [
 		"National"
@@ -34,7 +34,7 @@
 	"Industry_2": [
 		"Library"
 	],
-	"Organisation_Role2": [
+	"Organisation_Role": [
 		"Data Provider",
 		"Provider"
 	],

entity-management-tests/src/integration-test/resources/zoho-deref/organization_zoho_gfm_response.json

@@ -4,7 +4,7 @@
   "DEA_Sent": false,
   "Lang_Alternative_1":"SV(Swedish)",
   "Image_service_Opt_in": false,
-  "Country": "Sweden",
+  "Organisation_Country": "Sweden, SE",
   "Geographic_Level": [],
   "Alternative_1":"Gotlands försvarsmuseum",
   "Lang_Organisation_Name": "EN(English)",
@@ -13,8 +13,8 @@
   "Sector": "Cultural Heritage",
   "SameAs_1": "http://www.wikidata.org/entity/Q28933300",
   "SameAs_2": "http://isni.org/isni/000000012159802X",
-  "Organisation_Role2": [
-    "Data Provider"
+  "Organisation_Role": [
+    "Providing Institution"
   ],
   "Latitude": "-1",
   "Longitude": "-1"

entity-management-tests/src/integration-test/resources/zoho-deref/organization_zoho_naturalis_response.json

@@ -4,7 +4,7 @@
   "DEA_Sent": true,
   "City": "Leiden",
   "Image_service_Opt_in": false,
-  "Country": "Netherlands",
+  "Organisation_Country": "Netherlands, NL",
   "Geographic_Level": [],
   "Website": "https://www.naturalis.nl/nl/",
   "Lang_Organisation_Name": "EN(English)",
@@ -14,7 +14,7 @@
   "Sector": "Cultural Heritage",
   "SameAs_1": "http://www.wikidata.org/entity/Q641676",
   "SameAs_2": "http://isni.org/isni/000000012159802X",
-  "Organisation_Role2": [
+  "Organisation_Role": [
     "Data Provider"
   ],
   "Tag": [],

entity-management-tests/src/integration-test/resources/zoho-deref/organization_zoho_pcce_response.json

@@ -10,7 +10,7 @@
   "City": "Brussels",
   "Content_Language": "NL(Dutch)",
   "Image_service_Opt_in": false,
-  "Country": "Belgium",
+  "Organisation_Country": "Belgium, BE",
   "Geographic_Level": [
     "Regional"
   ],
@@ -22,7 +22,7 @@
   "ZIP_code": "1000",
   "Account_Name": "PCCE (Provincial Centre for Cultural Heritage)",
   "Sector": "Cultural Heritage",
-  "Organisation_Role2": [
+  "Organisation_Role": [
     "Data Provider"
   ],
   "Tag": [],

entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java

@@ -104,7 +104,7 @@ public class EntityRecordService {
   // Fields to be ignored during consolidation ("type" is final, so it cannot be updated)
   private static final Set<String> ignoredMergeFields = Set.of("type");
 
-  private List<ZohoMapping> countryMapping;
+  private Map<String, String> countryMapping;
   private ObjectMapper emJsonMapper;
 
   @Autowired
@@ -130,7 +130,8 @@ private void readCountryMappingFile() throws IOException {
       assert inputStream != null;
       try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) {
         String contents = reader.lines().collect(Collectors.joining(System.lineSeparator()));
-        countryMapping = emJsonMapper.readValue(contents, new TypeReference<List<ZohoMapping>>(){});
+        List<ZohoMapping> countryMapList = emJsonMapper.readValue(contents, new TypeReference<List<ZohoMapping>>(){});
+        countryMapping=countryMapList.stream().collect(Collectors.toMap(ZohoMapping::getZohoLabel, ZohoMapping::getEntityUri));
       }
     }
   }
@@ -1285,6 +1286,7 @@ private Map<Object, Object> deepCopyOfMap(Map<Object, Object> input) throws Enti
 
   public void dropRepository() {
     this.entityRecordRepository.dropCollection();
+    this.vocabRepository.dropCollection();
   }
 
   /**
@@ -1469,7 +1471,7 @@ public void mapZohoOrgFields(Entity entity) {
 
       //map the country field 
       if(org.getAddress()!=null && org.getAddress().getVcardCountryName()!=null) {
-        String countryUri = ZohoMapping.getEntityUriFromName(countryMapping, org.getAddress().getVcardCountryName());
+        String countryUri = countryMapping.get(org.getAddress().getVcardCountryName());
         if(StringUtils.isBlank(countryUri)) {
           logger.info("The mapping for the country: {}, to the europeana uri does not exist.", org.getAddress().getVcardCountryName());
         }

entity-management-web/src/main/resources/datasources.xml

@@ -10,8 +10,8 @@
     <!-- 'europeana' id required for this data source. See DataSources class  -->
     <source url="https://www.europeana.eu/"  					   rights="https://creativecommons.org/publicdomain/zero/1.0/"    id="europeana"/>
     <!-- 'europeana' id required for this data source. See DataSources class  -->
-    <source url="https://crm.zoho.com/"  					   		rights="https://creativecommons.org/publicdomain/zero/1.0/"    id="zoho-crm"/>
-<!--    <source url="https://crm.zoho.eu/"  					   		rights="https://creativecommons.org/publicdomain/zero/1.0/"    id="zoho-crm"/>  -->
+<!--    <source url="https://crm.zoho.com/"  					   		rights="https://creativecommons.org/publicdomain/zero/1.0/"    id="zoho-crm"/>  -->
+    <source url="https://crm.zoho.eu/"  					   		rights="https://creativecommons.org/publicdomain/zero/1.0/"    id="zoho-crm"/>
     <source url="http://bib.arts.kuleuven.be/photoVocabulary/" 		rights="https://creativecommons.org/publicdomain/zero/1.0/"   id="photoconsortium" frequency="static"/>
 
 </config>

entity-management-web/src/main/resources/zoho_country_mapping_local.json

@@ -1,7 +1,7 @@
 [
   {
-    "Zoho Label": "Sweden, SE",
-    "Entity URI": "http://data.europeana.eu/place/4",
-    "Wikidata URI": "http://www.wikidata.org/entity/Q34"
+    "Zoho Label": "France, FR",
+    "Entity URI": "http://data.europeana.eu/place/5",
+    "Wikidata URI": "http://www.wikidata.org/entity/Q142"
   }
 ]