diff --git a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java index cfc40b59..ae387bcc 100644 --- a/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java +++ b/entity-management-common/src/main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java @@ -175,7 +175,14 @@ public class EntityManagementConfiguration implements InitializingBean { @Value("${europeana.role.vocabulary:role_vocabulary.xml}") private String roleVocabularyFilename; + /** + * Map of <"Zoho Label", ZohoLabelUriMapping> + */ private final Map countryMappings = new ConcurrentHashMap<>(); + /** + * Map of <"EntityId", ZohoLabelUriMapping> + */ + private final Map countryIdMappings = new ConcurrentHashMap<>(); private final Map roleMappings = new ConcurrentHashMap<>(); @Autowired @@ -236,6 +243,7 @@ private void initCountryMappings() throws IOException { String contents = reader.lines().collect(Collectors.joining(System.lineSeparator())); List countryMappingList = emJsonMapper.readValue(contents, new TypeReference>(){}); addToCountryMappings(countryMappingList); + addToCountryIdMappings(countryMappingList); } } } @@ -247,6 +255,15 @@ void addToCountryMappings(List countryMappingList) { } } + void addToCountryIdMappings(List countryMappingList) { + for (ZohoLabelUriMapping countryMapping : countryMappingList) { + //init entityID - to ZohoCountry mapping + if(countryMapping.getEntityUri() != null) { + countryIdMappings.put(countryMapping.getEntityUri(), countryMapping); + } + } + } + private void initRoleMappings() throws IOException { ClassPathResource resource = new ClassPathResource(getZohoRoleMappingFilename()); @@ -465,5 +482,9 @@ public Map getRoleMappings() { public String getRoleVocabularyFilename() { return roleVocabularyFilename; } + + public Map getCountryIdMappings() { + return countryIdMappings; + } } diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/EntityRecord.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/EntityRecord.java index 30b3103c..92ce922c 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/EntityRecord.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/EntityRecord.java @@ -3,6 +3,7 @@ import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_EXACT_MATCH; import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_SAME_AS; import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_AGGREGATED_VIA; +import static eu.europeana.entitymanagement.definitions.EntityRecordFields.ENTITY_TYPE; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.BASE_DATA_EUROPEANA_URI; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ID; import java.util.ArrayList; @@ -26,7 +27,10 @@ @JsonInclude(value = JsonInclude.Include.NON_EMPTY) @dev.morphia.annotations.Entity("EntityRecord") -@Indexes({@Index(fields = {@Field(ENTITY_EXACT_MATCH)}), @Index(fields = {@Field(ENTITY_SAME_AS)}), +@Indexes({ + @Index(fields = {@Field(ENTITY_TYPE)}), + @Index(fields = {@Field(ENTITY_EXACT_MATCH)}), + @Index(fields = {@Field(ENTITY_SAME_AS)}), @Index(fields = {@Field(ENTITY_AGGREGATED_VIA)})}) @EntityListeners(EntityRecordWatcher.class) public class EntityRecord { diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/ZohoLabelUriMapping.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/ZohoLabelUriMapping.java index 8272e115..e5f3bc3c 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/ZohoLabelUriMapping.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/definitions/model/ZohoLabelUriMapping.java @@ -3,6 +3,7 @@ import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ENTITY_URI; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.WIKIDATA_URI; import static eu.europeana.entitymanagement.vocabulary.WebEntityFields.ZOHO_LABEL; +import org.apache.commons.lang3.StringUtils; import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonPropertyOrder; @@ -49,4 +50,12 @@ public String getWikidataUri() { public void setWikidataUri(String wikidataUri) { this.wikidataUri = wikidataUri; } + + public String getCountryISOCode() { + if (getZohoLabel() != null) { + return StringUtils.substringAfterLast(getZohoLabel(), ",").trim(); + } + return null; + } + } diff --git a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java index 13c83812..ad07a050 100644 --- a/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java +++ b/entity-management-solr/src/main/java/eu/europeana/entitymanagement/solr/model/SolrOrganization.java @@ -92,12 +92,12 @@ public SolrOrganization(Organization organization) { this.country=new ArrayList<>(); String orgCountryId=organization.getCountryId(); - String orgCoutryISO=organization.getCountryISO(); + String orgCountryISO=organization.getCountryISO(); if(orgCountryId!=null) { this.country.add(orgCountryId); } - if(orgCoutryISO!=null) { - this.country.add(orgCoutryISO); + if(orgCountryISO!=null) { + this.country.add(orgCountryISO); } if(organization.getCountry() != null) { this.setCountryLabel(organization.getCountry().getPrefLabel()); diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/EntityRecordDatabaseReader.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/EntityRecordDatabaseReader.java index 0e88f615..d193e14b 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/EntityRecordDatabaseReader.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/EntityRecordDatabaseReader.java @@ -13,6 +13,7 @@ import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord; import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTaskType; import eu.europeana.entitymanagement.definitions.model.EntityRecord; +import eu.europeana.entitymanagement.vocabulary.EntityProfile; import eu.europeana.entitymanagement.web.service.EntityRecordService; /** {@link ItemReader} that reads documents from MongoDB via a paging technique. */ @@ -42,7 +43,7 @@ protected Iterator doPageRead() { int start = page * pageSize; //the dereference profile is not needed here as the consolidation is taking care of processing references List result = - entityRecordService.findEntitiesWithFilter(start, pageSize, queryFilters, null); + entityRecordService.findEntitiesWithFilter(start, pageSize, queryFilters, EntityProfile.dereference.name()); List batchEntityRecords = toBatchEntityRecords(result, scheduledTaskType); diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/ScheduledTaskDatabaseReader.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/ScheduledTaskDatabaseReader.java index 154e226f..c62755d8 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/ScheduledTaskDatabaseReader.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/batch/reader/ScheduledTaskDatabaseReader.java @@ -10,6 +10,7 @@ import eu.europeana.entitymanagement.definitions.batch.model.BatchEntityRecord; import eu.europeana.entitymanagement.definitions.batch.model.ScheduledTask; import eu.europeana.entitymanagement.definitions.model.EntityRecord; +import eu.europeana.entitymanagement.vocabulary.EntityProfile; import eu.europeana.entitymanagement.web.service.EntityRecordService; /** @@ -42,7 +43,7 @@ protected Iterator doPageRead() { List entityIds = scheduledTasks.stream().map(st -> st.getEntityId()).toList(); //no need to use the dereference profile here as the consolidation takes care of processing references - List records = entityRecordService.retrieveMultipleByEntityIds(entityIds, false, true, null); + List records = entityRecordService.retrieveMultipleByEntityIds(entityIds, false, true, EntityProfile.dereference.name()); List batchRecords = toBatchEntityRecords(records, scheduledTasks); if (logger.isDebugEnabled()) { diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java index 22752e0b..bf1b27ba 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/EntityRecordService.java @@ -39,6 +39,7 @@ import eu.europeana.entitymanagement.definitions.model.Organization; import eu.europeana.entitymanagement.definitions.model.Place; import eu.europeana.entitymanagement.definitions.model.TimeSpan; +import eu.europeana.entitymanagement.definitions.model.ZohoLabelUriMapping; import eu.europeana.entitymanagement.definitions.web.EntityIdDisabledStatus; import eu.europeana.entitymanagement.exception.EntityAlreadyExistsException; import eu.europeana.entitymanagement.exception.EntityCreationException; @@ -182,6 +183,12 @@ public void dereferenceLinkedEntities(Organization org) { EntityRecord countryRecord = entityRecordRepository.findByEntityId(org.getCountryId(), new String[] {EntityRecordFields.ENTITY}); setDereferencedCountry(org, countryRecord); + + ZohoLabelUriMapping mapping = emConfiguration.getCountryIdMappings().get(org.getCountryId()); + if(mapping != null) { + //extract ISO code from ZohoCountry + org.setCountryISO(mapping.getCountryISOCode()); + } } // dereference role if (org.getEuropeanaRoleIds() != null && !org.getEuropeanaRoleIds().isEmpty()) { diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java index 286d2d02..346d54ad 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoOrganizationConverter.java @@ -80,12 +80,12 @@ public static Organization convertToOrganizationEntity(Record zohoRecord, String //update address country address.setVcardCountryName(extractCountryName(zohoCountryLabel)); - org.setCountryISO(StringUtils.substringAfterLast(zohoCountryLabel, ",").trim()); //update organization country id if(countryMappings.containsKey(zohoCountryLabel)) { //get country ID from mappings - String entityUri = countryMappings.get(zohoCountryLabel).getEntityUri(); - org.setCountryId(entityUri); + ZohoLabelUriMapping zohoLabelUriMapping = countryMappings.get(zohoCountryLabel); + org.setCountryId(zohoLabelUriMapping.getEntityUri()); + org.setCountryISO(zohoLabelUriMapping.getCountryISOCode()); } else if(logger.isInfoEnabled()){ logger.info("The mapping for the zoho country label: {}, to the europeana uri does not exist.", zohoCountryLabel); }