Skip to content

Commit

Permalink
publish zoho sync report to slack #EA-3874
Browse files Browse the repository at this point in the history
  • Loading branch information
@gsergiu
gsergiu committed Aug 19, 2024
1 parent 8c617e5 commit 2425bf9
Show file tree
Hide file tree
Showing 3 changed files with 125 additions and 39 deletions.
9 changes: 8 additions & 1 deletion .../main/java/eu/europeana/entitymanagement/common/config/EntityManagementConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,10 @@ public class EntityManagementConfiguration implements InitializingBean {

@Value("${europeana.role.vocabulary:role_vocabulary.xml}")
private String roleVocabularyFilename;


@Value("${slack.webhook:}")
private String slackWebHook;

/**
* Map of <"Zoho Label", ZohoLabelUriMapping>
*/
Expand Down Expand Up @@ -493,5 +496,9 @@ public Map<String, ZohoLabelUriMapping> getCountryIdMappings() {
public int getZohoSyncDeleteOffsetDays() {
return zohoSyncDeleteOffsetDays;
}

public String getSlackWebHook() {
return slackWebHook;
}

}
150 changes: 113 additions & 37 deletions ...nagement-web/src/main/java/eu/europeana/entitymanagement/web/service/ZohoSyncService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,13 @@
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.reactive.function.client.WebClient.ResponseSpec;
import org.springframework.web.reactive.function.client.WebClientResponseException;
import com.zoho.crm.api.record.DeletedRecord;
import com.zoho.crm.api.record.Record;
import dev.morphia.query.filters.Filter;
Expand All @@ -28,6 +33,7 @@
import eu.europeana.entitymanagement.solr.exception.SolrServiceException;
import eu.europeana.entitymanagement.solr.service.SolrService;
import eu.europeana.entitymanagement.web.model.BatchOperations;
import eu.europeana.entitymanagement.web.model.FailedOperation;
import eu.europeana.entitymanagement.web.model.Operation;
import eu.europeana.entitymanagement.web.model.ZohoSyncReport;
import eu.europeana.entitymanagement.web.model.ZohoSyncReportFields;
Expand All @@ -39,16 +45,18 @@
@Service(AppAutoconfig.BEAN_ZOHO_SYNC_SERVICE)
public class ZohoSyncService extends BaseZohoAccess {

public static final String ZOHO_SYNC_SLACK_TEMPLATE =
"%d organisations in Zoho were synchronised with the following actions:\\n"
+ "created: %d, updated: %d, deprecated: %d, undeprecated: %d, permanently deleted: %d, failed: %d";

@Autowired
public ZohoSyncService(EntityRecordService entityRecordService,
EntityUpdateService entityUpdateService,
EntityManagementConfiguration emConfiguration, DataSources datasources,
ZohoConfiguration zohoConfiguration,
SolrService solrService,
EntityUpdateService entityUpdateService, EntityManagementConfiguration emConfiguration,
DataSources datasources, ZohoConfiguration zohoConfiguration, SolrService solrService,
ZohoSyncRepository zohoSyncRepo) {

super(entityRecordService, entityUpdateService, emConfiguration,
datasources, zohoConfiguration, solrService, zohoSyncRepo);
super(entityRecordService, entityUpdateService, emConfiguration, datasources, zohoConfiguration,
solrService, zohoSyncRepo);
}

/**
Expand All @@ -70,7 +78,6 @@ public ZohoSyncReport synchronizeModifiedZohoOrganizations() throws EntityUpdate

// for development debugging purposes use modifiedSince = generateFixDate();
// modifiedSince = generateFixDate();

return synchronizeZohoOrganizations(modifiedSince);
}

Expand All @@ -84,9 +91,11 @@ public ZohoSyncReport synchronizeModifiedZohoOrganizations() throws EntityUpdate
public ZohoSyncReport synchronizeZohoOrganizations(@NonNull OffsetDateTime modifiedSince)
throws EntityUpdateException {

OffsetDateTime deletedSince = modifiedSince.minusDays(emConfiguration.getZohoSyncDeleteOffsetDays());
OffsetDateTime deletedSince =
modifiedSince.minusDays(emConfiguration.getZohoSyncDeleteOffsetDays());
if (logger.isInfoEnabled()) {
logger.info("Synchronizing organizations updated after date: {}, and delete after date :{}", modifiedSince, deletedSince);
logger.info("Synchronizing organizations updated after date: {}, and delete after date :{}",
modifiedSince, deletedSince);
}

ZohoSyncReport zohoSyncReport = new ZohoSyncReport(new Date());
Expand All @@ -97,9 +106,75 @@ public ZohoSyncReport synchronizeZohoOrganizations(@NonNull OffsetDateTime modif

logger.info("Zoho update operations completed successfully:\n {}", zohoSyncReport);

publishReport(zohoSyncReport);

return zohoSyncRepo.save(zohoSyncReport);
}


private void publishReport(ZohoSyncReport zohoSyncReport) {
if (logger.isDebugEnabled()) {
logger.debug("Sending report to slack : {}", zohoSyncReport);
}

try {
if (StringUtils.isBlank(emConfiguration.getSlackWebHook())) {
logger
.warn("Slack webhook not configured, status report will not be published over Slack!");
return;
}

String jsonMessage = buildSyncReportMessageForSlackWebHook(zohoSyncReport);

WebClient webClient = WebClient.builder().baseUrl(emConfiguration.getSlackWebHook()).build();
// send message to webhook
ResponseSpec resp = webClient.post().contentType(MediaType.APPLICATION_JSON).bodyValue(jsonMessage).retrieve();
ResponseEntity<String> response = resp.toEntity(String.class).block();
logger.debug("Received webhook response: {}", response.getBody());
} catch (WebClientResponseException e) {
logger.warn("Exception occurred while sending slack message!", e);
}
}

String buildSyncReportMessageForSlackWebHook(ZohoSyncReport zohoSyncReport) {
// String template = "X organisations in Zoho were synchronised with the following actions:\n"+
// "created: C, updated: U, deprecated: D, undeprecated: UD, permanently deleted: PD, failed:
// F\n\n" +
// "The following organisations failed synchronisation:\n" +
// "<ZOHO_URL> because of Y";
long synced = zohoSyncReport.getCreatedItems() + zohoSyncReport.getUpdatedItems()
+ zohoSyncReport.getDeprecatedItems();

long failures = zohoSyncReport.getFailed() != null? zohoSyncReport.getFailed().size() : 0;

String slackMessage = String.format(ZOHO_SYNC_SLACK_TEMPLATE, synced, zohoSyncReport.getCreatedItems(),
zohoSyncReport.getUpdatedItems(), zohoSyncReport.getDeprecatedItems(),
zohoSyncReport.getEnabledItems(), zohoSyncReport.getDeletedItems(),
failures, generateFailedMessage(zohoSyncReport));

//{"channel": "#my-channel-here\", "username": "webhookbot", "text": "my text"}
//could use a proper object and json serializer later
String webHoockMessage = "{\"text\":\"" + slackMessage + "\"}";
return webHoockMessage;
}



private String generateFailedMessage(ZohoSyncReport zohoSyncReport) {
if(zohoSyncReport.getFailed() == null || zohoSyncReport.getFailed().isEmpty()) {
return "";
}

StringBuilder builder = new StringBuilder("\\n\\nThe following organisations failed synchronisation:\\n");
for (FailedOperation failed : zohoSyncReport.getFailed()) {
builder.append(failed.getZohoId())
.append(" because of error: ")
.append(failed.getMessage()).append("\n");
}

return builder.toString();
}

void synchronizeZohoOrganizations(@NonNull OffsetDateTime modifiedSince,
ZohoSyncReport zohoSyncReport) {
List<Record> orgList;
Expand All @@ -115,8 +190,8 @@ void synchronizeZohoOrganizations(@NonNull OffsetDateTime modifiedSince,
// OffsetDateTime offsetDateTime = modifiedSince.toInstant()
// .atOffset(ZoneOffset.UTC);
try {
orgList = zohoConfiguration.getZohoAccessClient().getZcrmRecordOrganizations(page,
pageSize, modifiedSince);
orgList = zohoConfiguration.getZohoAccessClient().getZcrmRecordOrganizations(page, pageSize,
modifiedSince);

logExecutionProgress(orgList, page, pageSize);
} catch (ZohoException e) {
Expand Down Expand Up @@ -179,11 +254,11 @@ void synchronizeDeletedZohoOrganizations(OffsetDateTime modifiedSince,
currentPageSize = deletedRecordsInZoho.size();
// check exists in EM (Note: zoho doesn't support filtering by lastModified for deleted
// entities)
//build the Zoho Coref URL
// build the Zoho Coref URL
entitiesZohoCoref = getDeletedEntitiesZohoCoref(deletedRecordsInZoho);
entityIdsToDelete = getEntityIdsByZohoCorefs(entitiesZohoCoref);

//perform permanent deletion
// perform permanent deletion
runPermanentDelete(entityIdsToDelete, zohoSyncReport);
} catch (ZohoException e) {
logger.error(
Expand All @@ -194,9 +269,8 @@ void synchronizeDeletedZohoOrganizations(OffsetDateTime modifiedSince,
logger.error(
"Zoho synchronization exception occured when handling organizations deleted in Zoho",
e);
String message =
buildErrorMessage("Unexpected error occured when deleting organizations with ids: ",
entitiesZohoCoref);
String message = buildErrorMessage(
"Unexpected error occured when deleting organizations with ids: ", entitiesZohoCoref);
zohoSyncReport.addFailedOperation(null, ZohoSyncReportFields.ENTITY_DELETION_ERROR, message,
e);
}
Expand All @@ -219,28 +293,31 @@ void synchronizeDeletedZohoOrganizations(OffsetDateTime modifiedSince,
List<String> getEntityIdsByZohoCorefs(List<String> entitiesZohoCoref) {
List<EntityRecord> recordsToDelete;
List<String> entityIdsToDelete = new ArrayList<String>(entitiesZohoCoref.size());
//retrieve records by coref
recordsToDelete = entityRecordService.retrieveMultipleByEntityIdsOrCoreference(entitiesZohoCoref, null);
// retrieve records by coref
recordsToDelete =
entityRecordService.retrieveMultipleByEntityIdsOrCoreference(entitiesZohoCoref, null);
String zohoProxyId;
for (EntityRecord entityRecord : recordsToDelete) {
zohoProxyId = getZohoProxyId(entityRecord);
if(zohoProxyId == null && logger.isWarnEnabled()) {
if (zohoProxyId == null && logger.isWarnEnabled()) {
logger.warn(
"Cannot get zohoProxyId! Organization does not have a zoho proxy, but a zoho coref: {} ", entityRecord);
"Cannot get zohoProxyId! Organization does not have a zoho proxy, but a zoho coref: {} ",
entityRecord);
}
//for deprecated organizations, make sure to not delete the valid organizations which may contain the deprecated id in corefs
//threrefore, delete only records which have the deleted zoho record id in zoho proxy id
if(entitiesZohoCoref.contains(zohoProxyId)) {
// for deprecated organizations, make sure to not delete the valid organizations which may
// contain the deprecated id in corefs
// threrefore, delete only records which have the deleted zoho record id in zoho proxy id
if (entitiesZohoCoref.contains(zohoProxyId)) {
entityIdsToDelete.add(entityRecord.getEntityId());
}
}

return entityIdsToDelete;
}

String getZohoProxyId(EntityRecord entityRecord) {
for (String proxyId : entityRecord.getExternalProxyIds()) {
if(proxyId.startsWith(zohoConfiguration.getZohoBaseUrl())) {
if (proxyId.startsWith(zohoConfiguration.getZohoBaseUrl())) {
return proxyId;
}
}
Expand Down Expand Up @@ -298,9 +375,10 @@ BatchOperations fillOperations(final List<Record> orgList) {
*/
List<EntityRecord> findEntityRecordsByProxyId(Set<String> modifiedInZoho) {
Filter proxyIdsFilter = Filters.in("proxies.proxyId", modifiedInZoho);
//save to index operations are run asynchronuously, no need to dereference organizations here
return entityRecordService.findEntitiesWithFilter(0, modifiedInZoho.size(), new Filter[] {proxyIdsFilter}, null);

// save to index operations are run asynchronuously, no need to dereference organizations here
return entityRecordService.findEntitiesWithFilter(0, modifiedInZoho.size(),
new Filter[] {proxyIdsFilter}, null);

}

private void addOperation(BatchOperations operations, Long zohoId, Record zohoOrg,
Expand Down Expand Up @@ -343,11 +421,10 @@ String identifyOperationType(Long zohoId, String zohoRecordEuropeanaID, EntityRe
return shouldCreate(zohoId, zohoRecordEuropeanaID, hasDpsOwner, markedForDeletion)
? Operations.CREATE
: null;
} else if(emConfiguration.isGenerateOrganizationEuropeanaId() && isModifiedByApi(zohoOrg)) {
if(logger.isInfoEnabled()) {
logger.info(
"Skip Organization organization, it was last modified by the API by the {}",
zohoId);
} else if (emConfiguration.isGenerateOrganizationEuropeanaId() && isModifiedByApi(zohoOrg)) {
if (logger.isInfoEnabled()) {
logger.info("Skip Organization organization, it was last modified by the API by the {}",
zohoId);
}
return null;
} else if (shouldDisable(hasDpsOwner, markedForDeletion)) {
Expand Down Expand Up @@ -416,10 +493,9 @@ private boolean skipNoZohoEuropeanaId(String zohoRecordEuropeanaID,
}

boolean skipNonExisting(boolean hasDpsOwner, boolean markedForDeletion) {
//skip deprecated if not explicitly enabled in configs
//skip if owner is not DPS Team
return (markedForDeletion && !emConfiguration.isRegisterDeprecated())
|| !hasDpsOwner;
// skip deprecated if not explicitly enabled in configs
// skip if owner is not DPS Team
return (markedForDeletion && !emConfiguration.isRegisterDeprecated()) || !hasDpsOwner;
}

boolean needsToBeEnabled(EntityRecord entityRecord, boolean hasDpsOwner,
Expand Down
5 changes: 4 additions & 1 deletion entity-management-web/src/main/resources/entitymanagement.user.properties.template
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,4 +54,7 @@ europeana.searchapi.urlPrefix=https://<api_endpoint>/record/v2/search.json?wskey
#configuration files for zoho country and role mappings
#zoho.country.mapping=zoho_country_mapping.json
#zoho.role.mapping=zoho_role_mapping.json
#europeana.role.vocabulary=role_vocabulary.xml
#europeana.role.vocabulary=role_vocabulary.xml

#optional, webhook to send messages (i.e. ZohoSyncReport) to slack channels
#slack.webhook

0 comments on commit 2425bf9

Please sign in to comment.