From 4b2a3eba15f5cd6a0346d065dbaa05da73f4ab85 Mon Sep 17 00:00:00 2001 From: gsergiu <4517853+gsergiu@users.noreply.github.com> Date: Thu, 30 Jan 2025 15:58:40 +0100 Subject: [PATCH] handle aggregators in zoho sync #EA-4079 --- .../utils/EntityObjectFactory.java | 1 + .../service/DereferenceServiceIT.java | 2 +- .../testutils/IntegrationTestUtils.java | 16 +++--- .../testutils/TestConfig.java | 2 +- .../web/EntityRegistrationIT.java | 3 +- .../web/service/BaseZohoAccess.java | 52 +++++++++++++++---- .../web/service/ZohoSyncService.java | 5 +- .../zoho/ZohoAccessClient.java | 7 ++- .../organization/ZohoDereferenceService.java | 8 +++ .../zoho/utils/WikidataUtils.java | 2 +- 10 files changed, 72 insertions(+), 26 deletions(-) diff --git a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/utils/EntityObjectFactory.java b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/utils/EntityObjectFactory.java index 914d5083d..afc24d4a0 100644 --- a/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/utils/EntityObjectFactory.java +++ b/entity-management-definitions/src/main/java/eu/europeana/entitymanagement/utils/EntityObjectFactory.java @@ -142,6 +142,7 @@ public static SchemaOrgEntity createSchemaOrgEntity(Entity case Concept: return (SchemaOrgEntity) new SchemaOrgConcept((Concept) entity); case Organization: + case Aggregator: return (SchemaOrgEntity) new SchemaOrgOrganization((Organization) entity); case TimeSpan: return (SchemaOrgEntity) new SchemaOrgTimeSpan((TimeSpan) entity); diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java index 846e9a2fb..ae8945878 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java @@ -176,7 +176,7 @@ public void wikidataOrganizationGFMDereferenceTest() throws ZohoException, Excep // (not available in test data) public void wikidataOrganizationSNHDereferenceTest() throws ZohoException, Exception { // SNH - dereferenceWikidataOrganization("https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000938800"); + dereferenceWikidataOrganization(TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000000938800"); } diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java index c5863d8e1..82151b487 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java @@ -174,20 +174,20 @@ public class IntegrationTestUtils { public static final String VALID_MIGRATION_ID = "http://www.wikidata.org/entity/testing"; public static final String INVALID_MIGRATION_ID = "http://www.testing.org/entity/testing"; - public static final String ORGANIZATION_BNF_URI_ZOHO = - "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000938399"; + public static final String ORGANIZATION_BNF_URI_ZOHO = + TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000000938399"; public static final String ORGANIZATION_NATURALIS_URI_ZOHO = - "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000923816"; + TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000000923816"; public static final String ORGANIZATION_PCCE_URI_ZOHO = - "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000923271"; + TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000000923271"; public static final String ORGANIZATION_GFM_URI_ZOHO = - "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000940433"; + TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000000940433"; public static final String ORGANIZATION_BERGER_MUSEUM_URI_ZOHO = - "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000939337"; + TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000000939337"; public static final String ORGANIZATION_ARMA_URI_ZOHO = - "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000004375001"; + TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000004375001"; public static final String ORGANIZATION_EUSKARIANA_URI_ZOHO = - "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000939318"; + TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000000939318"; public static final String ORGANIZATION_EUSKARIANA_NAME = "Euskariana"; public static final String ORGANIZATION_ARMA_NAME = "Armagintzaren Museoa"; diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java index ef661cf78..7e637653b 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java @@ -17,7 +17,7 @@ public class TestConfig { - public static final String MOCK_ZOHO_BASE_URL = "https://crm.zoho.eu/crm/org20085137532/tab/"; + public static final String MOCK_ZOHO_BASE_URL = "https://crm.zoho.eu/crm/org20085137532/tab_test/"; public static final String MOCK_ZOHO_COUNTRY_MAPPING_FILE = "/zoho_country_mapping_test.json"; @Autowired diff --git a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java index 96dd2e599..b372fea29 100644 --- a/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java +++ b/entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java @@ -319,7 +319,8 @@ public void zohoCheckAggregatedViaOverZohoOrgSearch() throws Exception { // check if indexing is successfull by searching the organization in solr SolrAggregator aggreg = emSolrService.searchById(SolrAggregator.class, - WebEntityFields.BASE_DATA_EUROPEANA_URI + EntityTypes.Aggregator.getEntityType().toLowerCase() + "/1"); + EntityRecordUtils.buildEntityIdUri(EntityTypes.Aggregator, "1")); + //WebEntityFields.BASE_DATA_EUROPEANA_URI + EntityTypes.Aggregator.getEntityType().toLowerCase() + "/1" assertNotNull(aggreg.getHasAddress()); assertNotNull(aggreg.getEuropeanaRole()); assertNotNull(aggreg.getHeritageDomain()); diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java index 73d3434a2..d92f547b1 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java @@ -24,6 +24,7 @@ import eu.europeana.entitymanagement.config.DataSources; import eu.europeana.entitymanagement.definitions.batch.model.ScheduledUpdateType; import eu.europeana.entitymanagement.definitions.exceptions.UnsupportedEntityTypeException; +import eu.europeana.entitymanagement.definitions.model.Entity; import eu.europeana.entitymanagement.definitions.model.EntityRecord; import eu.europeana.entitymanagement.definitions.model.Organization; import eu.europeana.entitymanagement.exception.EntityCreationException; @@ -37,6 +38,7 @@ import eu.europeana.entitymanagement.web.model.ZohoSyncReport; import eu.europeana.entitymanagement.web.model.ZohoSyncReportFields; import eu.europeana.entitymanagement.zoho.organization.ZohoConfiguration; +import eu.europeana.entitymanagement.zoho.organization.ZohoDereferenceService; import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter; import eu.europeana.entitymanagement.zoho.utils.ZohoUtils; @@ -58,6 +60,8 @@ public class BaseZohoAccess { final ZohoSyncRepository zohoSyncRepo; + final ZohoDereferenceService zohoDereferenceService; + /** * Constructor for service initialization * @@ -71,8 +75,8 @@ public class BaseZohoAccess { */ public BaseZohoAccess(EntityRecordService entityRecordService, EntityUpdateService entityUpdateService, EntityManagementConfiguration emConfiguration, - DataSources datasources, ZohoConfiguration zohoConfiguration, - ZohoSyncRepository zohoSyncRepo) { + DataSources datasources, ZohoConfiguration zohoConfiguration, ZohoSyncRepository zohoSyncRepo, + ZohoDereferenceService zohoDereferenceService) { this.entityRecordService = entityRecordService; this.entityUpdateService = entityUpdateService; this.emConfiguration = emConfiguration; @@ -80,6 +84,7 @@ public BaseZohoAccess(EntityRecordService entityRecordService, this.zohoConfiguration = zohoConfiguration; this.zohoDataSource = initZohoDataSource(); this.zohoSyncRepo = zohoSyncRepo; + this.zohoDereferenceService = zohoDereferenceService; } protected DataSource initZohoDataSource() { @@ -223,7 +228,8 @@ void performDeprecationOperations(SortedSet deprecateOperations, } String generateZohoOrganizationUrl(Long zohoRecordId) { - return ZohoUtils.buildZohoRecordUrl(zohoConfiguration.getZohoBaseUrlOrganizations(), zohoRecordId); + return ZohoUtils.buildZohoRecordUrl(zohoConfiguration.getZohoBaseUrlOrganizations(), + zohoRecordId); } private void performDeprecation(ZohoSyncReport zohoSyncReport, Operation operation) { @@ -372,13 +378,24 @@ List performEntityRegistration(SortedSet createOperations, */ private Optional performEntityRegistration(Operation operation, ZohoSyncReport zohoSyncReport, List entitiesToUpdate) { - Organization zohoOrganization=new Organization(); - ZohoOrganizationConverter.fillOrganizationInfoFromZohoRecord(zohoOrganization, - operation.getZohoRecord(), zohoConfiguration.getZohoBaseUrlOrganizations(), - emConfiguration.getCountryMappings(), emConfiguration.getRoleMappings()); - + // Organization zohoOrganization=new Organization(); + // ZohoOrganizationConverter.fillOrganizationInfoFromZohoRecord(zohoOrganization, + // operation.getZohoRecord(), zohoConfiguration.getZohoBaseUrlOrganizations(), + // emConfiguration.getCountryMappings(), emConfiguration.getRoleMappings()); + // use dereference service to retrieve also aggregator info Optional res = Optional.empty(); + + // dereference organization + Long zohoId = operation.getZohoRecord().getId(); + Organization zohoOrganization = dereferenceFullOrganization(zohoId); + if (zohoOrganization == null) { + // should not happen, except for wrong configurations + zohoSyncReport.addFailedOperation(zohoId.toString(), "Cannot dereference organization", + "operation.getZohoRecord().getId() :" + zohoId, null); + } + + // perform registration try { List existingEntities = findDupplicateOrganization(operation, zohoOrganization); if (!existingEntities.isEmpty()) { @@ -421,6 +438,21 @@ private Optional performEntityRegistration(Operation operation, return res; } + Organization dereferenceFullOrganization(Long zohoId) { + try { + Optional orgOptional = Optional.empty(); + orgOptional = zohoDereferenceService.dereferenceOrganizationByZohoRecordId(zohoId); + if (orgOptional.isPresent()) { + return (Organization) orgOptional.get(); + } else { + return null; + } + } catch (Exception e) { + logger.warn("Cannot dereference organization by zoho record id: {}", zohoId, e); + return null; + } + } + List findDupplicateOrganization(Operation operation, Organization zohoOrganization) { List allCorefs = new ArrayList<>(); @@ -489,8 +521,8 @@ protected List getDeletedEntitiesZohoCoref(final List del List deletedEntityIds = new ArrayList(); // get the id list from Zoho deleted Record if (!deletedInZoho.isEmpty()) { - deletedInZoho.forEach(deletedRecord -> deletedEntityIds - .add(generateZohoOrganizationUrl(deletedRecord.getId()) + deletedInZoho.forEach( + deletedRecord -> deletedEntityIds.add(generateZohoOrganizationUrl(deletedRecord.getId()) // EntityRecordUtils. // buildEntityIdUri( // EntityTypes.Organization, deletedRecord.getId().toString()) diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java index d2691072a..845376441 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java @@ -37,6 +37,7 @@ import eu.europeana.entitymanagement.web.model.ZohoSyncReport; import eu.europeana.entitymanagement.web.model.ZohoSyncReportFields; import eu.europeana.entitymanagement.zoho.organization.ZohoConfiguration; +import eu.europeana.entitymanagement.zoho.organization.ZohoDereferenceService; import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter; import eu.europeana.entitymanagement.zoho.utils.ZohoConstants; import eu.europeana.entitymanagement.zoho.utils.ZohoException; @@ -52,10 +53,10 @@ public class ZohoSyncService extends BaseZohoAccess { public ZohoSyncService(EntityRecordService entityRecordService, EntityUpdateService entityUpdateService, EntityManagementConfiguration emConfiguration, DataSources datasources, ZohoConfiguration zohoConfiguration, - ZohoSyncRepository zohoSyncRepo) { + ZohoSyncRepository zohoSyncRepo, ZohoDereferenceService zohoDereferenceService) { super(entityRecordService, entityUpdateService, emConfiguration, datasources, zohoConfiguration, - zohoSyncRepo); + zohoSyncRepo, zohoDereferenceService); } /** diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/ZohoAccessClient.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/ZohoAccessClient.java index 2cfdc220c..606a0b5d1 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/ZohoAccessClient.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/ZohoAccessClient.java @@ -53,7 +53,7 @@ public class ZohoAccessClient { private static final Logger LOGGER = LogManager.getLogger(ZohoAccessClient.class); - + /** * Constructor with all parameters. * @@ -77,6 +77,7 @@ public class ZohoAccessClient { */ public ZohoAccessClient(TokenStore tokenStore, String zohoEmail, String clientId, String clientSecret, String refreshToken, String redirectUrl) throws ZohoException { + try { UserSignature userSignature = new UserSignature(zohoEmail); Token token = @@ -227,8 +228,9 @@ public Optional searchZohoAggregatedViaModule(@NonNull String orgName, @ * @param fieldValue the new value * @throws ZohoException wrapping the original SDK exception */ - public void updateZohoRecordOrganizationStringField(String zohoUrl, String fieldName, + public boolean updateZohoRecordOrganizationStringField(String zohoUrl, String fieldName, String fieldValue) throws ZohoException { + String zohoId = EntityRecordUtils.getIdentifierFromUrl(zohoUrl); try { RecordOperations recordOperations = new RecordOperations(); @@ -243,6 +245,7 @@ public void updateZohoRecordOrganizationStringField(String zohoUrl, String field } catch (SDKException e) { throw new ZohoException("Zoho update the organization field threw an exception.", e); } + return true; } BodyWrapper buildUpdateRequest(String fieldName, String fieldValue) { diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java index b80873bab..2ac59f776 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java @@ -34,6 +34,12 @@ public ZohoDereferenceService(ZohoConfiguration zohoConfiguration, EntityManagem this.zohoConfiguration = zohoConfiguration; this.emConfig = emConfig; } + + public Optional dereferenceOrganizationByZohoRecordId(@NonNull Long zohoRecordId) throws Exception { + String url = ZohoUtils.buildZohoRecordUrl(zohoConfiguration.getZohoBaseUrlOrganizations(), zohoRecordId); + return dereferenceEntityById(url); + } + @Override public Optional dereferenceEntityById(@NonNull String url) throws Exception { @@ -76,12 +82,14 @@ private Organization createOrganizationFromZohoRecords(Optional zohoOrga Organization org=null; if(zohoOrganization.isPresent()) { if(zohoAggregator.isPresent()) { + //fill aggregator properties org = new Aggregator(); ZohoOrganizationConverter.fillAggregatorInfoFromZohoRecord((Aggregator)org, zohoAggregator.get(), zohoConfiguration.getZohoBaseUrlAggregators()); } else { org=new Organization(); } + //fill common organization properties ZohoOrganizationConverter.fillOrganizationInfoFromZohoRecord(org, zohoOrganization.get(), zohoConfiguration.getZohoBaseUrlOrganizations(), emConfig.getCountryMappings(), emConfig.getRoleMappings()); } diff --git a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/utils/WikidataUtils.java b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/utils/WikidataUtils.java index c6864adc2..7816368ed 100644 --- a/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/utils/WikidataUtils.java +++ b/entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/utils/WikidataUtils.java @@ -17,7 +17,7 @@ public class WikidataUtils { * @return true if given entity is a wikidata organization, false otherwise */ public static boolean isWikidataOrganization(String id, String entityType) { - return EntityTypes.Organization.getEntityType().equals(entityType) && isWikidataEntity(id); + return EntityTypes.isOrganization(entityType) && isWikidataEntity(id); } /**