handle aggregators in zoho sync #EA-4079

entity-management-definitions/src/main/java/eu/europeana/entitymanagement/utils/EntityObjectFactory.java

@@ -142,6 +142,7 @@ public static <T extends Entity> SchemaOrgEntity<T> createSchemaOrgEntity(Entity
       case Concept:
         return (SchemaOrgEntity<T>) new SchemaOrgConcept((Concept) entity);
       case Organization:
+      case Aggregator:  
         return (SchemaOrgEntity<T>) new SchemaOrgOrganization((Organization) entity);
       case TimeSpan:
         return (SchemaOrgEntity<T>) new SchemaOrgTimeSpan((TimeSpan) entity);

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/service/DereferenceServiceIT.java

@@ -176,7 +176,7 @@ public void wikidataOrganizationGFMDereferenceTest() throws ZohoException, Excep
   // (not available in test data)
   public void wikidataOrganizationSNHDereferenceTest() throws ZohoException, Exception {
     // SNH 
-    dereferenceWikidataOrganization("https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000938800");
+    dereferenceWikidataOrganization(TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000000938800");
   }
 
 

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/IntegrationTestUtils.java

@@ -174,20 +174,20 @@ public class IntegrationTestUtils {
   public static final String VALID_MIGRATION_ID = "http://www.wikidata.org/entity/testing";
   public static final String INVALID_MIGRATION_ID = "http://www.testing.org/entity/testing";
 
-  public static final String ORGANIZATION_BNF_URI_ZOHO =
-      "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000938399";
+  public static final String ORGANIZATION_BNF_URI_ZOHO = 
+      TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000000938399";
   public static final String ORGANIZATION_NATURALIS_URI_ZOHO =
-      "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000923816";
+      TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000000923816";
   public static final String ORGANIZATION_PCCE_URI_ZOHO =
-      "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000923271";
+      TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000000923271";
   public static final String ORGANIZATION_GFM_URI_ZOHO =
-      "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000940433";
+      TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000000940433";
   public static final String ORGANIZATION_BERGER_MUSEUM_URI_ZOHO =
-      "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000939337";
+      TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000000939337";
   public static final String ORGANIZATION_ARMA_URI_ZOHO =
-      "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000004375001";
+      TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000004375001";
   public static final String ORGANIZATION_EUSKARIANA_URI_ZOHO =
-      "https://crm.zoho.eu/crm/org20085137532/tab/Accounts/486281000000939318"; 
+      TestConfig.MOCK_ZOHO_BASE_URL + "Accounts/486281000000939318"; 
   public static final String ORGANIZATION_EUSKARIANA_NAME = "Euskariana";
   public static final String ORGANIZATION_ARMA_NAME = "Armagintzaren Museoa";
 

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/testutils/TestConfig.java

@@ -17,7 +17,7 @@
 public class TestConfig {
 
 
-  public static final String MOCK_ZOHO_BASE_URL = "https://crm.zoho.eu/crm/org20085137532/tab/";
+  public static final String MOCK_ZOHO_BASE_URL = "https://crm.zoho.eu/crm/org20085137532/tab_test/";
   public static final String MOCK_ZOHO_COUNTRY_MAPPING_FILE = "/zoho_country_mapping_test.json";
 
   @Autowired

entity-management-tests/src/integration-test/java/eu/europeana/entitymanagement/web/EntityRegistrationIT.java

@@ -319,7 +319,8 @@ public void zohoCheckAggregatedViaOverZohoOrgSearch() throws Exception {
 
     // check if indexing is successfull by searching the organization in solr
     SolrAggregator aggreg = emSolrService.searchById(SolrAggregator.class, 
-        WebEntityFields.BASE_DATA_EUROPEANA_URI + EntityTypes.Aggregator.getEntityType().toLowerCase() + "/1");
+        EntityRecordUtils.buildEntityIdUri(EntityTypes.Aggregator, "1"));
+    //WebEntityFields.BASE_DATA_EUROPEANA_URI + EntityTypes.Aggregator.getEntityType().toLowerCase() + "/1"
     assertNotNull(aggreg.getHasAddress());
     assertNotNull(aggreg.getEuropeanaRole());
     assertNotNull(aggreg.getHeritageDomain());

entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java

@@ -24,6 +24,7 @@
 import eu.europeana.entitymanagement.config.DataSources;
 import eu.europeana.entitymanagement.definitions.batch.model.ScheduledUpdateType;
 import eu.europeana.entitymanagement.definitions.exceptions.UnsupportedEntityTypeException;
+import eu.europeana.entitymanagement.definitions.model.Entity;
 import eu.europeana.entitymanagement.definitions.model.EntityRecord;
 import eu.europeana.entitymanagement.definitions.model.Organization;
 import eu.europeana.entitymanagement.exception.EntityCreationException;
@@ -37,6 +38,7 @@
 import eu.europeana.entitymanagement.web.model.ZohoSyncReport;
 import eu.europeana.entitymanagement.web.model.ZohoSyncReportFields;
 import eu.europeana.entitymanagement.zoho.organization.ZohoConfiguration;
+import eu.europeana.entitymanagement.zoho.organization.ZohoDereferenceService;
 import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter;
 import eu.europeana.entitymanagement.zoho.utils.ZohoUtils;
 
@@ -58,6 +60,8 @@ public class BaseZohoAccess {
 
   final ZohoSyncRepository zohoSyncRepo;
 
+  final ZohoDereferenceService zohoDereferenceService;
+
   /**
    * Constructor for service initialization
    * 
@@ -71,15 +75,16 @@ public class BaseZohoAccess {
    */
   public BaseZohoAccess(EntityRecordService entityRecordService,
       EntityUpdateService entityUpdateService, EntityManagementConfiguration emConfiguration,
-      DataSources datasources, ZohoConfiguration zohoConfiguration,
-      ZohoSyncRepository zohoSyncRepo) {
+      DataSources datasources, ZohoConfiguration zohoConfiguration, ZohoSyncRepository zohoSyncRepo,
+      ZohoDereferenceService zohoDereferenceService) {
     this.entityRecordService = entityRecordService;
     this.entityUpdateService = entityUpdateService;
     this.emConfiguration = emConfiguration;
     this.datasources = datasources;
     this.zohoConfiguration = zohoConfiguration;
     this.zohoDataSource = initZohoDataSource();
     this.zohoSyncRepo = zohoSyncRepo;
+    this.zohoDereferenceService = zohoDereferenceService;
   }
 
   protected DataSource initZohoDataSource() {
@@ -223,7 +228,8 @@ void performDeprecationOperations(SortedSet<Operation> deprecateOperations,
   }
 
   String generateZohoOrganizationUrl(Long zohoRecordId) {
-    return ZohoUtils.buildZohoRecordUrl(zohoConfiguration.getZohoBaseUrlOrganizations(), zohoRecordId);
+    return ZohoUtils.buildZohoRecordUrl(zohoConfiguration.getZohoBaseUrlOrganizations(),
+        zohoRecordId);
   }
 
   private void performDeprecation(ZohoSyncReport zohoSyncReport, Operation operation) {
@@ -372,13 +378,24 @@ List<String> performEntityRegistration(SortedSet<Operation> createOperations,
    */
   private Optional<EntityRecord> performEntityRegistration(Operation operation,
       ZohoSyncReport zohoSyncReport, List<String> entitiesToUpdate) {
-    Organization zohoOrganization=new Organization();
-    ZohoOrganizationConverter.fillOrganizationInfoFromZohoRecord(zohoOrganization, 
-        operation.getZohoRecord(), zohoConfiguration.getZohoBaseUrlOrganizations(),
-        emConfiguration.getCountryMappings(), emConfiguration.getRoleMappings());
-
+    // Organization zohoOrganization=new Organization();
+    // ZohoOrganizationConverter.fillOrganizationInfoFromZohoRecord(zohoOrganization,
+    // operation.getZohoRecord(), zohoConfiguration.getZohoBaseUrlOrganizations(),
+    // emConfiguration.getCountryMappings(), emConfiguration.getRoleMappings());
+    // use dereference service to retrieve also aggregator info
     Optional<EntityRecord> res = Optional.empty();
 
+
+    // dereference organization
+    Long zohoId = operation.getZohoRecord().getId();
+    Organization zohoOrganization = dereferenceFullOrganization(zohoId);
+    if (zohoOrganization == null) {
+      // should not happen, except for wrong configurations
+      zohoSyncReport.addFailedOperation(zohoId.toString(), "Cannot dereference organization",
+          "operation.getZohoRecord().getId() :" + zohoId, null);
+    }
+
+    // perform registration
     try {
       List<EntityRecord> existingEntities = findDupplicateOrganization(operation, zohoOrganization);
       if (!existingEntities.isEmpty()) {
@@ -421,6 +438,21 @@ private Optional<EntityRecord> performEntityRegistration(Operation operation,
     return res;
   }
 
+  Organization dereferenceFullOrganization(Long zohoId) {
+    try {
+      Optional<Entity> orgOptional = Optional.empty();
+      orgOptional = zohoDereferenceService.dereferenceOrganizationByZohoRecordId(zohoId);
+      if (orgOptional.isPresent()) {
+        return (Organization) orgOptional.get();
+      } else {
+        return null;
+      }
+    } catch (Exception e) {
+      logger.warn("Cannot dereference organization by zoho record id: {}", zohoId, e);
+      return null;
+    }
+  }
+
   List<EntityRecord> findDupplicateOrganization(Operation operation,
       Organization zohoOrganization) {
     List<String> allCorefs = new ArrayList<>();
@@ -489,8 +521,8 @@ protected List<String> getDeletedEntitiesZohoCoref(final List<DeletedRecord> del
     List<String> deletedEntityIds = new ArrayList<String>();
     // get the id list from Zoho deleted Record
     if (!deletedInZoho.isEmpty()) {
-      deletedInZoho.forEach(deletedRecord -> deletedEntityIds
-          .add(generateZohoOrganizationUrl(deletedRecord.getId())
+      deletedInZoho.forEach(
+          deletedRecord -> deletedEntityIds.add(generateZohoOrganizationUrl(deletedRecord.getId())
           // EntityRecordUtils.
           // buildEntityIdUri(
           // EntityTypes.Organization, deletedRecord.getId().toString())

entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java

@@ -37,6 +37,7 @@
 import eu.europeana.entitymanagement.web.model.ZohoSyncReport;
 import eu.europeana.entitymanagement.web.model.ZohoSyncReportFields;
 import eu.europeana.entitymanagement.zoho.organization.ZohoConfiguration;
+import eu.europeana.entitymanagement.zoho.organization.ZohoDereferenceService;
 import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter;
 import eu.europeana.entitymanagement.zoho.utils.ZohoConstants;
 import eu.europeana.entitymanagement.zoho.utils.ZohoException;
@@ -52,10 +53,10 @@ public class ZohoSyncService extends BaseZohoAccess {
   public ZohoSyncService(EntityRecordService entityRecordService,
       EntityUpdateService entityUpdateService, EntityManagementConfiguration emConfiguration,
       DataSources datasources, ZohoConfiguration zohoConfiguration, 
-      ZohoSyncRepository zohoSyncRepo) {
+      ZohoSyncRepository zohoSyncRepo, ZohoDereferenceService zohoDereferenceService) {
 
     super(entityRecordService, entityUpdateService, emConfiguration, datasources, zohoConfiguration,
-        zohoSyncRepo);
+        zohoSyncRepo, zohoDereferenceService);
   }
 
   /**

entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/ZohoAccessClient.java

@@ -53,7 +53,7 @@
 public class ZohoAccessClient {
 
   private static final Logger LOGGER = LogManager.getLogger(ZohoAccessClient.class);
-
+  
   /**
    * Constructor with all parameters.
    *
@@ -77,6 +77,7 @@ public class ZohoAccessClient {
    */
   public ZohoAccessClient(TokenStore tokenStore, String zohoEmail, String clientId,
       String clientSecret, String refreshToken, String redirectUrl) throws ZohoException {
+
     try {
       UserSignature userSignature = new UserSignature(zohoEmail);
       Token token =
@@ -227,8 +228,9 @@ public Optional<Record> searchZohoAggregatedViaModule(@NonNull String orgName, @
    * @param fieldValue the new value
    * @throws ZohoException wrapping the original SDK exception
    */
-  public void updateZohoRecordOrganizationStringField(String zohoUrl, String fieldName,
+  public boolean updateZohoRecordOrganizationStringField(String zohoUrl, String fieldName,
       String fieldValue) throws ZohoException {
+
     String zohoId = EntityRecordUtils.getIdentifierFromUrl(zohoUrl);
     try {
       RecordOperations recordOperations = new RecordOperations();
@@ -243,6 +245,7 @@ public void updateZohoRecordOrganizationStringField(String zohoUrl, String field
     } catch (SDKException e) {
       throw new ZohoException("Zoho update the organization field threw an exception.", e);
     }
+    return true;
   }
 
   BodyWrapper buildUpdateRequest(String fieldName, String fieldValue) {

entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/organization/ZohoDereferenceService.java

@@ -34,6 +34,12 @@ public ZohoDereferenceService(ZohoConfiguration zohoConfiguration, EntityManagem
     this.zohoConfiguration = zohoConfiguration;
     this.emConfig = emConfig;
   }
+
+  public Optional<Entity> dereferenceOrganizationByZohoRecordId(@NonNull Long zohoRecordId) throws Exception {
+    String url = ZohoUtils.buildZohoRecordUrl(zohoConfiguration.getZohoBaseUrlOrganizations(), zohoRecordId);
+    return dereferenceEntityById(url);
+  }  
+
 
   @Override
   public Optional<Entity> dereferenceEntityById(@NonNull String url) throws Exception {
@@ -76,12 +82,14 @@ private Organization createOrganizationFromZohoRecords(Optional<Record> zohoOrga
     Organization org=null;
     if(zohoOrganization.isPresent()) {    
       if(zohoAggregator.isPresent()) {
+        //fill aggregator properties
         org = new Aggregator();
         ZohoOrganizationConverter.fillAggregatorInfoFromZohoRecord((Aggregator)org, zohoAggregator.get(), zohoConfiguration.getZohoBaseUrlAggregators());
       }
       else {
         org=new Organization();
       }
+      //fill common organization properties
       ZohoOrganizationConverter.fillOrganizationInfoFromZohoRecord(org, zohoOrganization.get(),
           zohoConfiguration.getZohoBaseUrlOrganizations(), emConfig.getCountryMappings(), emConfig.getRoleMappings());
     }

entity-management-zoho/src/main/java/eu/europeana/entitymanagement/zoho/utils/WikidataUtils.java

@@ -17,7 +17,7 @@ public class WikidataUtils {
    * @return true if given entity is a wikidata organization, false otherwise
    */
   public static boolean isWikidataOrganization(String id, String entityType) {
-    return EntityTypes.Organization.getEntityType().equals(entityType) && isWikidataEntity(id);
+    return EntityTypes.isOrganization(entityType) && isWikidataEntity(id);
   }
 
   /**