From 40eb9d660cdd45cc8795a8530b1a852f215d34b9 Mon Sep 17 00:00:00 2001 From: GordeaS Date: Mon, 29 Jul 2024 17:58:03 +0200 Subject: [PATCH] items deleted in zoho must be retrieved by zoho coref to get the new organization id #EA-3896 --- .../web/service/BaseZohoAccess.java | 19 ++++++++++----- .../web/service/ZohoSyncService.java | 23 ++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java index ca242bfb..e1d164db 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/BaseZohoAccess.java @@ -128,7 +128,7 @@ protected void performOperations(BatchOperations operations, ZohoSyncReport zoho // scheduled updates at the end, otherwise the other operations may overwrite the record in the db with the old captured in the operation performUpdateOperations(operations.getUpdateOperations(), zohoSyncReport); - + } void performPermanentDeleteOperations(SortedSet permanentDeleteOperations, ZohoSyncReport zohoSyncReport) { @@ -151,9 +151,12 @@ void performPermanentDeleteOperations(SortedSet permanentDeleteOperat } } - void runPermanentDelete(List entitiesDeletedInZoho, ZohoSyncReport zohoSyncReport) + void runPermanentDelete(List entitiesToDelete, ZohoSyncReport zohoSyncReport) throws SolrServiceException { - long deleted = entityRecordService.deleteBulk(entitiesDeletedInZoho, true); + if(entitiesToDelete == null || entitiesToDelete.isEmpty()) { + return; + } + long deleted = entityRecordService.deleteBulk(entitiesToDelete, true); zohoSyncReport.increaseDeleted(deleted); } @@ -428,7 +431,7 @@ protected Set getZohoOrganizationUrls(final List orgList) { } - protected List getDeletedEntityIds(final List deletedInZoho) { + protected List getDeletedEntitiesZohoCoref(final List deletedInZoho) { List deletedEntityIds = new ArrayList(); // get the id list from Zoho deleted Record @@ -436,8 +439,12 @@ protected List getDeletedEntityIds(final List deletedInZo deletedInZoho.forEach( deletedRecord -> deletedEntityIds.add( - EntityRecordUtils.buildEntityIdUri( - EntityTypes.Organization, deletedRecord.getId().toString()))); + generateZohoOrganizationUrl(deletedRecord.getId().longValue()) +// EntityRecordUtils. +// buildEntityIdUri( +// EntityTypes.Organization, deletedRecord.getId().toString()) + ) + ); } return deletedEntityIds; } diff --git a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java index 9fa0e5d2..5cff51db 100644 --- a/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java +++ b/entity-management-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java @@ -136,7 +136,7 @@ void synchronizeZohoOrganizations(@NonNull OffsetDateTime modifiedSince, if (isLastPage(orgList.size(), pageSize)) { if (logger.isDebugEnabled()) { logger.debug( - "Processing of deleted records is complete on page on page {}, pageSize {}, items on last page {}", + "Processing updated zoho records finished at page {}, pageSize {}, items on last page {}", page, pageSize, orgList.size()); } hasNext = false; @@ -166,8 +166,10 @@ void synchronizeDeletedZohoOrganizations(OffsetDateTime modifiedSince, int pageSize = emConfiguration.getZohoSyncBatchSize(); boolean hasNext = true; int currentPageSize = 0; - List entitiesDeletedInZoho = null; + List entitiesZohoCoref = null; // Zoho doesn't return the total results + List deletedEntityRecords; + List entityIdsToDelete; while (hasNext) { try { // list of (europeana) organizations ids @@ -177,10 +179,15 @@ void synchronizeDeletedZohoOrganizations(OffsetDateTime modifiedSince, currentPageSize = deletedRecordsInZoho.size(); // check exists in EM (Note: zoho doesn't support filtering by lastModified for deleted // entities) - entitiesDeletedInZoho = getDeletedEntityIds(deletedRecordsInZoho); - - // build delete operations set - runPermanentDelete(entitiesDeletedInZoho, zohoSyncReport); + //build the Zoho Coref URL + entitiesZohoCoref = getDeletedEntitiesZohoCoref(deletedRecordsInZoho); + //retrieve records by coref + deletedEntityRecords = entityRecordService.retrieveMultipleByEntityIdsOrCoreference(entitiesZohoCoref, null); + //fetch entity IDs + entityIdsToDelete = deletedEntityRecords.stream().map(er -> er.getEntityId()).toList(); + + //perform permanent deletion + runPermanentDelete(entityIdsToDelete, zohoSyncReport); } catch (ZohoException e) { logger.error( "Zoho synchronization exception occured when handling organizations deleted in Zoho", @@ -192,7 +199,7 @@ void synchronizeDeletedZohoOrganizations(OffsetDateTime modifiedSince, e); String message = buildErrorMessage("Unexpected error occured when deleting organizations with ids: ", - entitiesDeletedInZoho); + entitiesZohoCoref); zohoSyncReport.addFailedOperation(null, ZohoSyncReportFields.ENTITY_DELETION_ERROR, message, e); } @@ -201,7 +208,7 @@ void synchronizeDeletedZohoOrganizations(OffsetDateTime modifiedSince, // last page: if no more organizations exist in Zoho if (logger.isDebugEnabled()) { logger.debug( - "Processing of deleted records is complete on page on page {}, pageSize {}, items on last page {}", + "Processing of deleted records is completes at page {}, pageSize {}, items on last page {}", startPage, pageSize, currentPageSize); } hasNext = false;