Merge branch 'develop' of https://github.com/europeana/entity-managem…

…ent.git into develop

entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java

@@ -175,7 +175,14 @@ public class EntityManagementConfiguration implements InitializingBean {
   @Value("${europeana.role.vocabulary:role_vocabulary.xml}")
   private String roleVocabularyFilename;
 
+  /**
+   * Map of <"Zoho Label", ZohoLabelUriMapping>  
+   */
   private final Map<String, ZohoLabelUriMapping> countryMappings = new ConcurrentHashMap<>();
+  /**
+   * Map of <"EntityId", ZohoLabelUriMapping>  
+   */
+  private final Map<String, ZohoLabelUriMapping> countryIdMappings = new ConcurrentHashMap<>();
   private final Map<String, String> roleMappings = new ConcurrentHashMap<>();
 
   @Autowired
@@ -236,6 +243,7 @@ private void initCountryMappings() throws IOException {
         String contents = reader.lines().collect(Collectors.joining(System.lineSeparator()));
         List<ZohoLabelUriMapping> countryMappingList = emJsonMapper.readValue(contents, new TypeReference<List<ZohoLabelUriMapping>>(){});
         addToCountryMappings(countryMappingList);
+        addToCountryIdMappings(countryMappingList);
       }
     }
   }
@@ -247,6 +255,15 @@ void addToCountryMappings(List<ZohoLabelUriMapping> countryMappingList) {
     }
   }
 
+  void addToCountryIdMappings(List<ZohoLabelUriMapping> countryMappingList) {
+    for (ZohoLabelUriMapping countryMapping : countryMappingList) {
+      //init entityID - to ZohoCountry mapping
+      if(countryMapping.getEntityUri() != null) {
+        countryIdMappings.put(countryMapping.getEntityUri(), countryMapping);
+      }
+    }
+  }
+
   private void initRoleMappings() throws IOException {
 
     ClassPathResource resource = new ClassPathResource(getZohoRoleMappingFilename());
@@ -465,5 +482,9 @@ public Map<String, String> getRoleMappings() {
   public String getRoleVocabularyFilename() {
     return roleVocabularyFilename;
   }
+
+  public Map<String, ZohoLabelUriMapping> getCountryIdMappings() {
+    return countryIdMappings;
+  }
 
 }

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/EntityRecord.java

@@ -3,6 +3,7 @@
 import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_EXACT_MATCH;
 import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_SAME_AS;
 import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_AGGREGATED_VIA;
+import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_TYPE;
 import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.BASE_DATA_EUROPEANA_URI;
 import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ID;
 import java.util.ArrayList;
@@ -26,7 +27,10 @@
 
 @JsonInclude(value = JsonInclude.Include.NON_EMPTY)
 @dev.morphia.annotations.Entity("EntityRecord")
-@Indexes({@Index(fields = {@Field(ENTITY_EXACT_MATCH)}), @Index(fields = {@Field(ENTITY_SAME_AS)}), 
+@Indexes({
+  @Index(fields = {@Field(ENTITY_TYPE)}),
+  @Index(fields = {@Field(ENTITY_EXACT_MATCH)}), 
+  @Index(fields = {@Field(ENTITY_SAME_AS)}),
   @Index(fields = {@Field(ENTITY_AGGREGATED_VIA)})})
 @EntityListeners(EntityRecordWatcher.class)
 public class EntityRecord {

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/ZohoLabelUriMapping.java

@@ -3,6 +3,7 @@
 import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ENTITY_URI;
 import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.WIKIDATA_URI;
 import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ZOHO_LABEL;
+import org.apache.commons.lang3.StringUtils;
 import com.fasterxml.jackson.annotation.JsonGetter;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonPropertyOrder;
@@ -49,4 +50,12 @@ public String getWikidataUri() {
   public void setWikidataUri(String wikidataUri) {
     this.wikidataUri = wikidataUri;
   }
+
+  public String getCountryISOCode() {
+    if (getZohoLabel() != null) {
+      return StringUtils.substringAfterLast(getZohoLabel(), ",").trim();
+    }
+    return null;
+  }
+
 }

entity-management-mongo/src/main/java/eu/europeana/entitymanagement/mongo/repository/VocabularyRepository.java

@@ -18,13 +18,39 @@ public class VocabularyRepository  {
   Datastore datastore;
 
   private static final String ID = "id";
+
+  private List<Vocabulary> europeanaRoles;
+
+  public List<Vocabulary> getEuropeanaRoles() {
+    synchronized(this) {
+      if(europeanaRoles==null) {
+        europeanaRoles=new ArrayList<>();
+        europeanaRoles.addAll(
+            datastore.find(Vocabulary.class)
+            .iterator()
+            .toList());
+      }
+      return europeanaRoles;
+    }
+  }
 
   /**
-   * retrieve records by their id
-   * @param vocabularyIds
-   * @return
+   * retrieve records by their id, from the in-memory collection
+   * @param vocabularyIds ids to search for
+   * @return list of Vocabularies
    */
   public List<Vocabulary> findByUri(List<String> vocabularyIds) {
+    return getEuropeanaRoles().stream()
+    .filter(el -> vocabularyIds.contains(el.getId()))
+    .toList();
+  }
+
+  /**
+   * retrieve records by their id
+   * @param vocabularyIds ids to search for
+   * @return list of Vocabularies
+   */
+  public List<Vocabulary> findInDbByUri(List<String> vocabularyIds) {
     List<Filter> filters = new ArrayList<>();
     filters.add(in(ID, vocabularyIds));
     return datastore.find(Vocabulary.class)
@@ -65,4 +91,5 @@ public void dropCollection() {
   public long countRecords() {
     return datastore.find(Vocabulary.class).count();
   }
+
 }

entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java

@@ -92,12 +92,12 @@ public SolrOrganization(Organization organization) {
 
     this.country=new ArrayList<>();
     String orgCountryId=organization.getCountryId();
-    String orgCoutryISO=organization.getCountryISO();
+    String orgCountryISO=organization.getCountryISO();
     if(orgCountryId!=null) {
       this.country.add(orgCountryId);
     }
-    if(orgCoutryISO!=null) {
-      this.country.add(orgCoutryISO);
+    if(orgCountryISO!=null) {
+      this.country.add(orgCountryISO);
     }
     if(organization.getCountry() != null) {
       this.setCountryLabel(organization.getCountry().getPrefLabel()); 

entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/EntityRecordDatabaseReader.java

@@ -13,6 +13,7 @@
 import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord;
 import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTaskType;
 import eu.europeana.entitymanagement.definitions.model.EntityRecord;
+import eu.europeana.entitymanagement.vocabulary.EntityProfile;
 import eu.europeana.entitymanagement.web.service.EntityRecordService;
 
 /** {@link ItemReader} that reads documents from MongoDB via a paging technique. */
@@ -42,7 +43,7 @@ protected Iterator<BatchEntityRecord> doPageRead() {
     int start = page * pageSize;
     //the dereference profile is not needed here as the consolidation is taking care of processing references
     List<EntityRecord> result =
-        entityRecordService.findEntitiesWithFilter(start, pageSize, queryFilters, null);
+        entityRecordService.findEntitiesWithFilter(start, pageSize, queryFilters, EntityProfile.dereference.name());
 
     List<BatchEntityRecord> batchEntityRecords = toBatchEntityRecords(result, scheduledTaskType);
 

entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/ScheduledTaskDatabaseReader.java

@@ -10,6 +10,7 @@
 import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord;
 import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTask;
 import eu.europeana.entitymanagement.definitions.model.EntityRecord;
+import eu.europeana.entitymanagement.vocabulary.EntityProfile;
 import eu.europeana.entitymanagement.web.service.EntityRecordService;
 
 /**
@@ -42,7 +43,7 @@ protected Iterator<BatchEntityRecord> doPageRead() {
 
     List<String> entityIds = scheduledTasks.stream().map(st -> st.getEntityId()).toList();
     //no need to use the dereference profile here as the consolidation takes care of processing references
-    List<EntityRecord> records = entityRecordService.retrieveMultipleByEntityIds(entityIds, false, true, null);
+    List<EntityRecord> records = entityRecordService.retrieveMultipleByEntityIds(entityIds, false, true, EntityProfile.dereference.name());
     List<BatchEntityRecord> batchRecords = toBatchEntityRecords(records, scheduledTasks);
 
     if (logger.isDebugEnabled()) {

entity-management-web/src/main/java/eu/europeana/entitymanagement/config/AppAutoconfig.java

@@ -86,7 +86,7 @@ public void ensureDatabaseInitialization() throws ApplicationInitializationExcep
           roles.add(vocab);
         }
         vocabRepository.saveBulk(roles);
-      }
+    }
   }
 
   @Bean(name = BEAN_EM_DATA_SOURCES)

entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java

@@ -39,6 +39,7 @@
 import eu.europeana.entitymanagement.definitions.model.Organization;
 import eu.europeana.entitymanagement.definitions.model.Place;
 import eu.europeana.entitymanagement.definitions.model.TimeSpan;
+import eu.europeana.entitymanagement.definitions.model.ZohoLabelUriMapping;
 import eu.europeana.entitymanagement.definitions.web.EntityIdDisabledStatus;
 import eu.europeana.entitymanagement.exception.EntityAlreadyExistsException;
 import eu.europeana.entitymanagement.exception.EntityCreationException;
@@ -182,6 +183,12 @@ public void dereferenceLinkedEntities(Organization org) {
       EntityRecord countryRecord = entityRecordRepository.findByEntityId(org.getCountryId(),
           new String[] {EntityRecordFields.ENTITY});
       setDereferencedCountry(org, countryRecord);
+
+      ZohoLabelUriMapping mapping = emConfiguration.getCountryIdMappings().get(org.getCountryId());
+      if(mapping != null) {
+        //extract ISO code from ZohoCountry
+        org.setCountryISO(mapping.getCountryISOCode());  
+      }
     }
     // dereference role
     if (org.getEuropeanaRoleIds() != null && !org.getEuropeanaRoleIds().isEmpty()) {

entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java

@@ -80,12 +80,12 @@ public static Organization convertToOrganizationEntity(Record zohoRecord, String
       //update address country
       address.setVcardCountryName(extractCountryName(zohoCountryLabel));
 
-      org.setCountryISO(StringUtils.substringAfterLast(zohoCountryLabel, ",").trim());
       //update organization country id
       if(countryMappings.containsKey(zohoCountryLabel)) {
         //get country ID from mappings
-        String entityUri = countryMappings.get(zohoCountryLabel).getEntityUri();
-        org.setCountryId(entityUri);
+        ZohoLabelUriMapping zohoLabelUriMapping = countryMappings.get(zohoCountryLabel);
+        org.setCountryId(zohoLabelUriMapping.getEntityUri());
+        org.setCountryISO(zohoLabelUriMapping.getCountryISOCode());
       } else if(logger.isInfoEnabled()){
         logger.info("The mapping for the zoho country label: {}, to the europeana uri does not exist.", zohoCountryLabel);
       }