Skip to content

Commit

Permalink
fixes for integration tests #EA-4079
Browse files Browse the repository at this point in the history
  • Loading branch information
gsergiu committed Feb 6, 2025
1 parent e7a4801 commit e43992a
Show file tree
Hide file tree
Showing 14 changed files with 258 additions and 167 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -242,13 +242,11 @@ private static void addLabels(
private static boolean isEnrichmentDisabled(EntityRecord record) {
// entity is not consolidated, should not be index at this stage
boolean notConsolidated = record.getEntity() == null || record.getEntity().getIsAggregatedBy() == null;
// check if flag is set to false
boolean noEnrichment = Boolean.FALSE.equals(record.getEntity().getIsAggregatedBy().getEnrich());
if (notConsolidated || noEnrichment) {
if(notConsolidated) {
return true;
}

return false;
// check if flag is set to false, noEnrichments
return Boolean.FALSE.equals(record.getEntity().getIsAggregatedBy().getEnrich());
}

private static void setMetricsAndFilters(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package eu.europeana.entitymanagement.testutils;

import static eu.europeana.entitymanagement.zoho.utils.ZohoConstants.*;
import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
import com.zoho.crm.api.record.Record;
import com.zoho.crm.api.util.Choice;

/** Helper class to deserialize JSON into Zoho {@link Record} to make testing easier */
public abstract class BaseZohoRecordDeserializer<T> extends StdDeserializer<T> {

/** */
private static final long serialVersionUID = 7519475270154623735L;
/**
* Contains list of fields to read from JSON file.
*
* <p>Fields with numeric suffixes (sameAs, lang code) and ID field are handled separately
*/
private static final List<String> ZOHO_JSON_FIELDS =
List.of(
ID_FIELD,
ACCOUNT_NAME_FIELD,
ORGANIZATION_ROLE_FIELD,
LANG_ORGANIZATION_NAME_FIELD,
LANG_ACRONYM_FIELD,
ACRONYM_FIELD,
LANG_ACRONYM_1_FIELD,
ACRONYM_1_FIELD,
LOGO_LINK_TO_WIKIMEDIACOMMONS_FIELD,
WEBSITE_FIELD,
STREET_FIELD,
CITY_FIELD,
COUNTRY_FIELD,
ZIP_CODE_FIELD,
PO_BOX_FIELD,
OFFICIAL_LANGUAGE_FIELD,
LATITUDE_FIELD,
LONGITUDE_FIELD,
HIDDEN_LABEL1_FIELD,
HIDDEN_LABEL2_FIELD,
HIDDEN_LABEL3_FIELD,
HIDDEN_LABEL4_FIELD,
HIDDEN_LABEL_FIELD,
INDUSTRY_FIELD,
HERITAGE_DOMAIN,
PUBLIC_EMAIL,
GEOGRAPHIC_SCOPE,
MEDIA_TYPE,
DATA_ACTIVITY,
AUDIENCE_ENGAGEMENT_ACTIVITY,
CAPACITY_BUILDING,
AGGREGATORS,
EUROPEANA_ID_FIELD,
AGGREGATING_FROM);

public BaseZohoRecordDeserializer() {
this(null);
}

public BaseZohoRecordDeserializer(Class<?> vc) {
super(vc);
}

Record deserializeSingleRecord(JsonNode node) {
Record record = new Record();

for (String key : ZOHO_JSON_FIELDS) {
JsonNode currentNode = node.get(key);

if (currentNode == null) {
continue;
}

// JSON contains strings, arrays
if (currentNode.isTextual()) {
record.addKeyValue(key, currentNode.asText());
} else if (currentNode.isArray()) {
List<Choice<?>> values = new ArrayList<Choice<?>>();
currentNode.elements().forEachRemaining(v -> values.add(new Choice<String>(v.asText())));
record.addKeyValue(key, values);
}else if(currentNode.isObject()) {
System.out.println("object node: " + key);
if(AGGREGATORS.equals(key) || AGGREGATING_FROM.equals(key)) {
Record subRecord = new Record();
subRecord.setId(currentNode.get(ID_FIELD).asLong());
subRecord.addKeyValue(NAME_FIELD, currentNode.get(NAME_FIELD).asText());
record.addKeyValue(key, subRecord);
}
}else if (currentNode.isContainerNode()){
System.out.println("container node: " + key);
}else if(currentNode.isPojo()) {
System.out.println("pojo node: " + key);
}
}

// add fields with numeric suffixes
addMultiField(node, record, ALTERNATIVE_FIELD, LANGUAGE_CODE_LENGTH);
addMultiField(node, record, LANG_ALTERNATIVE_FIELD, LANGUAGE_CODE_LENGTH);
addMultiField(node, record, SAME_AS_FIELD, SAME_AS_CODE_LENGTH);

// add ID
record.setId(node.get(ID_FIELD).asLong());

return record;
}

private void addMultiField(JsonNode node, Record record, String fieldName, int length) {
for (int i = 1; i <= length; i++) {
JsonNode currentNode = node.get(fieldName + "_" + i);
if (currentNode != null && currentNode.isTextual()) {
record.addKeyValue(fieldName + "_" + i, currentNode.asText());
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
Expand All @@ -13,6 +16,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.zoho.crm.api.record.Record;
import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter;

public class IntegrationTestUtils {

Expand Down Expand Up @@ -256,7 +260,10 @@ public class IntegrationTestUtils {
new SimpleModule(
"SimpleModule",
Version.unknownVersion(),
Map.of(Record.class, new ZohoRecordTestDeserializer())));
Map.of(
Record.class, new ZohoRecordTestDeserializer(),
(new ArrayList<Record>()).getClass(), new ZohoRecordListTestDeserializer()
)));

/** Maps ZOHO organization URIs to mocked JSON responses */
public static final Map<String, String> ZOHO_ORG_URL_RESPONSE_MAP =
Expand Down Expand Up @@ -346,13 +353,20 @@ public static Optional<Record> getZohoAggregatorByOrgUrl(String orgUrl) throws E
}
}

public static Optional<Record> searchZohoAggregatedViaModule(@NonNull String orgName) throws Exception {
@SuppressWarnings("unchecked")
public static List<Record> searchZohoAggregatedViaModule(@NonNull String orgName) throws Exception {
if(ZOHO_ORG_AGGREG_LINKING_RESPONSE_MAP.containsKey(orgName)) {
String zohoResponseData = loadFile(ZOHO_ORG_AGGREG_LINKING_RESPONSE_MAP.get(orgName));
return Optional.ofNullable(zohoResponseObjectMapper.readValue(zohoResponseData, Record.class));
//TODO: update code to support multiple aggregators
// Record aggregatedVia = zohoResponseObjectMapper.readValue(zohoResponseData, Record.class);
// JsonParser jsonParser = zohoResponseObjectMapper.createParser(zohoResponseData);
// jsonParser.
// jsonParser.readValuesAs(Record.class)
List<Record> aggregatedViaList = zohoResponseObjectMapper.readValue(zohoResponseData, (new ArrayList<Record>()).getClass());
return ZohoOrganizationConverter.getAggregatorRecordsFromAggregatedVia(orgName, aggregatedViaList);
}
else {
return Optional.empty();
return Collections.emptyList();
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package eu.europeana.entitymanagement.testutils;

import java.util.List;
import java.util.Optional;
import org.mockito.ArgumentMatchers;
import org.mockito.Mockito;
Expand All @@ -11,6 +12,7 @@
import com.zoho.crm.api.record.Record;
import eu.europeana.entitymanagement.zoho.ZohoAccessClient;
import eu.europeana.entitymanagement.zoho.organization.ZohoConfiguration;
import eu.europeana.entitymanagement.zoho.organization.ZohoOrganizationConverter;
import eu.europeana.entitymanagement.zoho.utils.ZohoConstants;

@TestConfiguration
Expand Down Expand Up @@ -51,8 +53,8 @@ public ZohoConfiguration configureZoho() throws Exception {
Mockito.doAnswer(
(Answer<Optional<Record>>)
invocation -> {
String zohoUrl = invocation.getArgument(0);
return IntegrationTestUtils.searchZohoOrganizationByName(zohoUrl);
String orgName = invocation.getArgument(0);
return IntegrationTestUtils.searchZohoOrganizationByName(orgName);
})
.when(zohoClient)
.searchZohoOrganizationByName(ArgumentMatchers.any(String.class));
Expand All @@ -67,7 +69,7 @@ public ZohoConfiguration configureZoho() throws Exception {
.getZohoAggregatorByOrgUrl(ArgumentMatchers.any(String.class));

Mockito.doAnswer(
(Answer<Optional<Record>>)
(Answer<List<Record>>)
invocation -> {
String orgName = invocation.getArgument(0);
//String aggregName = invocation.getArgument(1);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package eu.europeana.entitymanagement.testutils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.DeserializationContext;
import com.fasterxml.jackson.databind.JsonNode;
import com.zoho.crm.api.record.Record;

/** Helper class to deserialize JSON into Zoho {@link Record} to make testing easier */
public class ZohoRecordListTestDeserializer extends BaseZohoRecordDeserializer<List<Record>> {


public ZohoRecordListTestDeserializer() {
this(null);
}

public ZohoRecordListTestDeserializer(Class<?> vc) {
super(vc);
}

@Override
public List<Record> deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
JsonNode jsonRootNode = p.getCodec().readTree(p);
if(!jsonRootNode.isArray()) {
throw new IOException("Verify json input, it must be an array!");
}

List<Record> res = new ArrayList<>();
for (JsonNode jsonNode : jsonRootNode) {
res.add(deserializeSingleRecord(jsonNode));
}
return res;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
import com.zoho.crm.api.util.Choice;

/** Helper class to deserialize JSON into Zoho {@link Record} to make testing easier */
public class ZohoRecordTestDeserializer extends StdDeserializer<Record> {
public class ZohoRecordTestDeserializer extends BaseZohoRecordDeserializer<Record> {

/** */
private static final long serialVersionUID = 7519475270154623735L;
Expand Down Expand Up @@ -107,55 +107,6 @@ public ZohoRecordTestDeserializer(Class<?> vc) {

@Override
public Record deserialize(JsonParser p, DeserializationContext ctxt) throws IOException {
JsonNode node = p.getCodec().readTree(p);
Record record = new Record();

for (String key : ZOHO_JSON_FIELDS) {
JsonNode currentNode = node.get(key);

if (currentNode == null) {
continue;
}

// JSON contains strings, arrays
if (currentNode.isTextual()) {
record.addKeyValue(key, currentNode.asText());
} else if (currentNode.isArray()) {
List<Choice<?>> values = new ArrayList<Choice<?>>();
currentNode.elements().forEachRemaining(v -> values.add(new Choice<String>(v.asText())));
record.addKeyValue(key, values);
}else if(currentNode.isObject()) {
System.out.println("object node: " + key);
if(AGGREGATORS.equals(key) || AGGREGATING_FROM.equals(key)) {
Record subRecord = new Record();
subRecord.setId(currentNode.get(ID_FIELD).asLong());
subRecord.addKeyValue(NAME_FIELD, currentNode.get(NAME_FIELD));
record.addKeyValue(key, subRecord);
}
}else if (currentNode.isContainerNode()){
System.out.println("container node: " + key);
}else if(currentNode.isPojo()) {
System.out.println("pojo node: " + key);
}
}

// add fields with numeric suffixes
addMultiField(node, record, ALTERNATIVE_FIELD, LANGUAGE_CODE_LENGTH);
addMultiField(node, record, LANG_ALTERNATIVE_FIELD, LANGUAGE_CODE_LENGTH);
addMultiField(node, record, SAME_AS_FIELD, SAME_AS_CODE_LENGTH);

// add ID
record.setId(node.get(ID_FIELD).asLong());

return record;
}

private void addMultiField(JsonNode node, Record record, String fieldName, int length) {
for (int i = 1; i <= length; i++) {
JsonNode currentNode = node.get(fieldName + "_" + i);
if (currentNode != null && currentNode.isTextual()) {
record.addKeyValue(fieldName + "_" + i, currentNode.asText());
}
}
return deserializeSingleRecord(p.getCodec().readTree(p));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -355,12 +355,6 @@ public void zohoCheckAggregatedViaOverZohoLinkingAndLocalDbSearch() throws Excep
.contentType(MediaType.APPLICATION_JSON_VALUE));
assertNotNull(response);

/*
* 2. remove the entry from the map that mocks the zoho response for the organization
* with the name of the aggregator
*/
IntegrationTestUtils.ZOHO_ORG_NAME_RESPONSE_MAP.remove(IntegrationTestUtils.ORGANIZATION_EUSKARIANA_NAME);

/*
* 3. register zoho Arma organization, which is aggregated via the previously registered
* organization (Euskariana), and check the aggregatedVia field
Expand All @@ -376,13 +370,6 @@ public void zohoCheckAggregatedViaOverZohoLinkingAndLocalDbSearch() throws Excep
.andExpect(jsonPath("$.id", any(String.class)))
.andExpect(jsonPath("$.type", is(EntityTypes.Organization.getEntityType())))
.andExpect(jsonPath("$.aggregatedVia", hasSize(1)));

/*
* 4. put back the removed entry from the map in step 2.
*/
IntegrationTestUtils.ZOHO_ORG_NAME_RESPONSE_MAP.put(
IntegrationTestUtils.ORGANIZATION_EUSKARIANA_NAME,
IntegrationTestUtils.ORGANIZATION_EUSKARIANA_ZOHO_RESPONSE);

}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
[
{
"Aggregating_from": {
"name": "Armagintzaren Museoa",
Expand All @@ -14,4 +15,5 @@
},
"id": "486281000006450920"
}
]

Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
"Subsector": "Museum",
"Alternative_3": null,
"Lang_Alternative_3": "ES(Spanish)",
"Aggregators": "Euskariana",
"Aggregators": "Euskariana,FAKE_AGG_FORCE_SEARCH",
"Alternative_2": "Museo de la Industria Armera",
"Lang_Alternative_4": null,
"Alternative_5": null,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
"Lang_Acronym": null,
"Street": "Donostia San Sebastian Kalea, 1",
"id": "486281000000939318",
"Europeana_org_ID": "http://data.europeana.eu/aggregator/8",
"Europeana_org_ID": "http://data.europeana.eu/organization/4140",
"DEA_signed": true,
"Created_Time": "2023-10-13T10:18:22+02:00",
"Lang_Alternative_1": "EU(Basque)",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -529,7 +529,6 @@ public ResponseEntity<String> registerEntity(@RequestBody Entity europeanaProxyE

Entity datasourceResponse = dereferenceEntity(creationRequestId, creationRequestType);

// TODO: implement support for zoho EuropeanaID
EntityRecord savedEntityRecord = entityRecordService
.createEntityFromRequest(europeanaProxyEntity, datasourceResponse, dataSource, null);
logger.debug("Created Entity record for externalId={}; entityId={}", creationRequestId,
Expand Down
Loading

0 comments on commit e43992a

Please sign in to comment.