Skip to content

Commit

Permalink
https://github.com/metadatacenter/cedar-project/issues/1198
Browse files Browse the repository at this point in the history
  • Loading branch information
Attila L. Egyedi committed Nov 8, 2023
1 parent 0ed16f9 commit a359cc8
Show file tree
Hide file tree
Showing 20 changed files with 56 additions and 115 deletions.
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package org.metadatacenter.cadsr.ingestor.category;

import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,9 @@ public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (!(obj instanceof Value)) {
if (!(obj instanceof Value other)) {
return false;
}
Value other = (Value) obj;
return Objects.equals(id, other.id)
&& Objects.equals(version, other.version)
&& Objects.equals(dbLabel, other.dbLabel)
Expand All @@ -105,4 +104,4 @@ public boolean equals(Object obj) {
&& Objects.equals(beginDate, other.beginDate)
&& Objects.equals(endDate, other.endDate);
}
}
}
Original file line number Diff line number Diff line change
@@ -1,14 +1,11 @@
package org.metadatacenter.cadsr.ingestor.cde.action;

import org.metadatacenter.cadsr.ingestor.util.CdeUtil;
import org.metadatacenter.cadsr.ingestor.util.CedarServices;
import org.metadatacenter.cadsr.ingestor.util.Constants.CedarServer;
import org.metadatacenter.model.ModelNodeNames;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;
import java.util.Map;

public class AttachCdeToCategoriesAction implements CdeAction {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,13 @@ public String execute(CedarServer cedarEnvironment, String apiKey) {
status = CedarServices.loadValueSetsOntologyStatus(cedarEnvironment, apiKey);
}

if (status.equals("COMPLETE"))
return "Done";
else if (status.equals("ERROR"))
return "Error";
else if (status.equals("NOT_YET_INITIATED"))
return "Failed to start";
else if (status.equals("IN_PROGRESS"))
return "Failed to finish";
else
return "Error - unknown status";
return switch (status) {
case "COMPLETE" -> "Done";
case "ERROR" -> "Error";
case "NOT_YET_INITIATED" -> "Failed to start";
case "IN_PROGRESS" -> "Failed to finish";
default -> "Error - unknown status";
};

} catch (IOException | InterruptedException e) {
String message = "Error calling load value sets ontology endpoint: " + e.getMessage();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import org.metadatacenter.cadsr.cde.schema.DataElement;
import org.metadatacenter.cadsr.cde.schema.VALUEDOMAIN;
import org.metadatacenter.cadsr.ingestor.cde.CadsrConstants;
import org.metadatacenter.cadsr.ingestor.exception.UnsupportedDataElementException;
import org.metadatacenter.model.ModelNodeNames;
import org.metadatacenter.model.ModelNodeValues;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public VersionHandler handle(DataElement dataElement) throws UnsupportedDataElem
private void setDataElementStatus(DataElement dataElement) throws UnsupportedDataElementException {
String status = dataElement.getWORKFLOWSTATUS().getContent();
if (Strings.isNullOrEmpty(status)) {
String reason = String.format("Data element status is empty (Unknown)");
String reason = "Data element status is empty (Unknown)";
throw new UnsupportedDataElementException(dataElement, reason);
}
if (!CadsrStatus.STATUS_LIST.contains(status)) {
Expand All @@ -40,7 +40,7 @@ private void setDataElementStatus(DataElement dataElement) throws UnsupportedDat
private void setDataElementVersion(DataElement dataElement) throws UnsupportedDataElementException {
String version = dataElement.getVERSION().getContent();
if (Strings.isNullOrEmpty(version)) {
String reason = String.format("Data element version is empty (Unknown)");
String reason = "Data element version is empty (Unknown)";
throw new UnsupportedDataElementException(dataElement, reason);
}
this.version = CdeUtil.reformatVersioningNumber(version);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,7 @@ public List<String> getMessages(String reportId) {
}

public void remove(String reportId) {
if (messagesMap.containsKey(reportId)) {
messagesMap.remove(reportId);
}
messagesMap.remove(reportId);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import org.metadatacenter.cadsr.ingestor.util.CdeUtil;
import org.metadatacenter.cadsr.ingestor.util.CedarServerUtil;
import org.metadatacenter.cadsr.ingestor.util.Constants;
import org.metadatacenter.cadsr.ingestor.util.Constants.*;
import org.metadatacenter.cadsr.ingestor.util.GeneralUtil;
import org.metadatacenter.config.CedarConfig;
import org.metadatacenter.config.environment.CedarEnvironmentVariableProvider;
Expand All @@ -21,10 +20,7 @@
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.*;

import static org.metadatacenter.cadsr.ingestor.util.Constants.*;

Expand Down Expand Up @@ -188,17 +184,15 @@ private static Object setDefaultUi() {

// Target: required
private static Object setRequired() {
return Arrays.asList(new String[]{
ModelNodeNames.JSON_LD_CONTEXT,
return Arrays.asList(ModelNodeNames.JSON_LD_CONTEXT,
ModelNodeNames.JSON_LD_ID,
ModelNodeNames.SCHEMA_IS_BASED_ON,
ModelNodeNames.SCHEMA_ORG_NAME,
ModelNodeNames.SCHEMA_ORG_DESCRIPTION,
ModelNodeNames.PAV_CREATED_ON,
ModelNodeNames.PAV_CREATED_BY,
ModelNodeNames.PAV_LAST_UPDATED_ON,
ModelNodeNames.OSLC_MODIFIED_BY,
});
ModelNodeNames.OSLC_MODIFIED_BY);
}

// Target: properties
Expand Down Expand Up @@ -278,7 +272,7 @@ private static Object setDefaultPropertiesContextPropertiesUri(String uri) {
Map<String, Object> content = Maps.newHashMap();
content.put(ModelNodeNames.JSON_SCHEMA_TYPE, ModelNodeValues.STRING);
content.put(ModelNodeNames.JSON_SCHEMA_FORMAT, ModelNodeValues.URI);
content.put(ModelNodeNames.JSON_SCHEMA_ENUM, Arrays.asList(new String[]{uri}));
content.put(ModelNodeNames.JSON_SCHEMA_ENUM, Collections.singletonList(uri));
return content;
}

Expand All @@ -297,7 +291,7 @@ private static Object setDefaultPropertiesContextPropertiesAtTypeObject(String t
// Target: X.properties.@type
Map<String, Object> contentPropertiesAtType = Maps.newHashMap();
contentPropertiesAtType.put(ModelNodeNames.JSON_SCHEMA_TYPE, ModelNodeValues.STRING);
contentPropertiesAtType.put(ModelNodeNames.JSON_SCHEMA_ENUM, Arrays.asList(new String[]{type}));
contentPropertiesAtType.put(ModelNodeNames.JSON_SCHEMA_ENUM, Collections.singletonList(type));

// Target: X.properties
Map<String, Object> contentProperties = Maps.newHashMap();
Expand Down Expand Up @@ -333,7 +327,7 @@ private static Map<String, Object> setStringMinLength() {

private static Map<String, Object> setUriStringOrNull() {
Map<String, Object> uriString = Maps.newHashMap();
uriString.put(ModelNodeNames.JSON_SCHEMA_TYPE, Arrays.asList(new String[]{ModelNodeValues.STRING, ModelNodeValues.NULL}));
uriString.put(ModelNodeNames.JSON_SCHEMA_TYPE, List.of(ModelNodeValues.STRING, ModelNodeValues.NULL));
uriString.put(ModelNodeNames.JSON_SCHEMA_FORMAT, ModelNodeValues.URI);
return uriString;
}
Expand All @@ -358,7 +352,7 @@ private static Map<String, Object> setUriStringArray() {

private static Map<String, Object> setDateTimeStringOrNull() { // We allow null because the value is set by the server
Map<String, Object> uriString = Maps.newHashMap();
uriString.put(ModelNodeNames.JSON_SCHEMA_TYPE, Arrays.asList(new String[]{ModelNodeValues.STRING, ModelNodeValues.NULL}));
uriString.put(ModelNodeNames.JSON_SCHEMA_TYPE, List.of(ModelNodeValues.STRING, ModelNodeValues.NULL));
uriString.put(ModelNodeNames.JSON_SCHEMA_FORMAT, ModelNodeValues.DATE_TIME);
return uriString;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
import java.io.IOException;
import java.util.*;

import static org.metadatacenter.cadsr.ingestor.util.Constants.*;
import static org.metadatacenter.cadsr.ingestor.util.Constants.LARGE_PAGE_SIZE;
import static org.metadatacenter.cadsr.ingestor.util.Constants.MOVE_ACTIONS_THRESHOLD;
import static org.metadatacenter.model.ModelNodeNames.*;

public class TemplateFieldsHandler implements ModelHandler {
Expand Down Expand Up @@ -137,8 +138,7 @@ private Map<String, Object> customizeCdeValues(Map<String, Object> cde, List<Val
// also sort them alphabetically in the UI, so we need to make that comparison. We cannot retrieve the values
// alphabetically sorted by BioPortal, so we sort them here. We will compare their position to the valid values
// display order later (see generateMoveActions method) to determine if it's necessary to generate move actions.
Collections.sort(cdeValues,
(v1, v2) -> v1.get("notation").compareTo(v2.get("notation")));
cdeValues.sort((v1, v2) -> v1.get("notation").compareTo(v2.get("notation")));

// Create Map with the values indexed by prefLabel to access them quickly. We save the position as a new
// attribute. We'll use it to generate the move actions
Expand All @@ -156,8 +156,7 @@ private Map<String, Object> customizeCdeValues(Map<String, Object> cde, List<Val
// 2. Compare (ignoring case) the retrieved values to the values in the form's XML to identify exclusions
// Correspondences between CEDAR's CDE model and the form's xml validValue model:
// - notation <-> value
Map<String, Map<String, String>> excludedCdeValuesMap = new HashMap<>();
excludedCdeValuesMap.putAll(cdeValuesMap);
Map<String, Map<String, String>> excludedCdeValuesMap = new HashMap<>(cdeValuesMap);
for (ValidValue validValue : validValues) {
String valueKey = validValue.getValue().toLowerCase();
if (excludedCdeValuesMap.containsKey(valueKey)) {
Expand Down Expand Up @@ -252,11 +251,9 @@ private Map<String, Object> getUpdatedUi(String fieldName, String fieldDescripti
private Map<String, Object> getUpdatedPropertiesContextProperties(String fieldName, Map<String, Object> templateMap) {
Map<String, Object> properties = (Map<String, Object>) templateMap.get(JSON_SCHEMA_PROPERTIES);
Map<String, Object> propertiesContext = (Map<String, Object>) properties.get(JSON_LD_CONTEXT);
Map<String, Object> propertiesContextProperties =
(Map<String, Object>) propertiesContext.get(JSON_SCHEMA_PROPERTIES);
Map<String, Object> propertiesContextProperties = (Map<String, Object>) propertiesContext.get(JSON_SCHEMA_PROPERTIES);
propertiesContextProperties.put(fieldName, new HashMap<String, List<String>>() {{
put(JSON_SCHEMA_ENUM,
Arrays.asList(new String[]{"https://schema.metadatacenter.org/properties/" + UUID.randomUUID()}));
put(JSON_SCHEMA_ENUM, List.of("https://schema.metadatacenter.org/properties/" + UUID.randomUUID()));
}});
return propertiesContextProperties;
}
Expand Down Expand Up @@ -285,17 +282,15 @@ private List<String> getUpdatedRequired(String fieldName, Map<String, Object> te
* @param modules
*/
private void sortModules(List<Module> modules) {
Collections.sort(modules, new Comparator<>() {
public int compare(Module m1, Module m2) {
if (m1.getDisplayOrder() == null && m2.getDisplayOrder() == null) {
return 0;
} else if (m1.getDisplayOrder() == null) {
return 1;
} else if (m2.getDisplayOrder() == null) {
return -1;
}
return Integer.compare(Integer.parseInt(m1.getDisplayOrder()), Integer.parseInt(m2.getDisplayOrder()));
modules.sort((m1, m2) -> {
if (m1.getDisplayOrder() == null && m2.getDisplayOrder() == null) {
return 0;
} else if (m1.getDisplayOrder() == null) {
return 1;
} else if (m2.getDisplayOrder() == null) {
return -1;
}
return Integer.compare(Integer.parseInt(m1.getDisplayOrder()), Integer.parseInt(m2.getDisplayOrder()));
});
}

Expand All @@ -305,17 +300,15 @@ public int compare(Module m1, Module m2) {
* @param questions
*/
private void sortQuestions(List<Question> questions) {
Collections.sort(questions, new Comparator<>() {
public int compare(Question q1, Question q2) {
if (q1.getDisplayOrder() == null && q2.getDisplayOrder() == null) {
return 0;
} else if (q1.getDisplayOrder() == null) {
return 1;
} else if (q2.getDisplayOrder() == null) {
return -1;
}
return Integer.compare(Integer.parseInt(q1.getDisplayOrder()), Integer.parseInt(q2.getDisplayOrder()));
questions.sort((q1, q2) -> {
if (q1.getDisplayOrder() == null && q2.getDisplayOrder() == null) {
return 0;
} else if (q1.getDisplayOrder() == null) {
return 1;
} else if (q2.getDisplayOrder() == null) {
return -1;
}
return Integer.compare(Integer.parseInt(q1.getDisplayOrder()), Integer.parseInt(q2.getDisplayOrder()));
});
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,10 @@
package org.metadatacenter.cadsr.ingestor.form.handler;

import org.metadatacenter.cadsr.form.schema.Form;
import org.metadatacenter.cadsr.form.schema.Module;
import org.metadatacenter.cadsr.form.schema.Question;
import org.metadatacenter.cadsr.ingestor.util.CedarFieldUtil;
import org.metadatacenter.cadsr.ingestor.util.CedarServices;
import org.metadatacenter.cadsr.ingestor.util.Constants.CedarServer;
import org.metadatacenter.model.ModelNodeNames;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import java.io.IOException;
import java.util.*;

import static org.metadatacenter.model.ModelNodeNames.*;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,5 @@
package org.metadatacenter.cadsr.ingestor.form.handler;

import com.google.common.base.Strings;
import org.metadatacenter.cadsr.cde.schema.DataElement;
import org.metadatacenter.cadsr.form.schema.Form;
import org.metadatacenter.cadsr.ingestor.cde.CadsrStatus;
import org.metadatacenter.cadsr.ingestor.cde.handler.ModelHandler;
import org.metadatacenter.cadsr.ingestor.exception.UnsupportedDataElementException;
import org.metadatacenter.model.BiboStatus;
import org.metadatacenter.model.ModelNodeNames;

import javax.annotation.Nullable;
import java.util.Map;

//public class VersionHandler implements ModelHandler {
//
// private String status = null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,10 @@

import com.google.common.base.Stopwatch;
import org.metadatacenter.cadsr.form.schema.Form;
import org.metadatacenter.cadsr.ingestor.form.FormParser;
import org.metadatacenter.cadsr.ingestor.form.FormUtil;
import org.metadatacenter.cadsr.ingestor.tools.form.config.ConfigSettings;
import org.metadatacenter.cadsr.ingestor.tools.form.config.ConfigSettingsParser;
import org.metadatacenter.cadsr.ingestor.util.CedarServices;
import org.metadatacenter.cadsr.ingestor.util.GeneralUtil;
import org.metadatacenter.config.CedarConfig;
import org.metadatacenter.config.environment.CedarEnvironmentVariableProvider;
import org.metadatacenter.model.SystemComponent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.metadatacenter.cadsr.ingestor.tools.form.config;

import org.metadatacenter.cadsr.ingestor.util.Constants.CedarServer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package org.metadatacenter.cadsr.ingestor.tools.form.config;

import org.apache.commons.cli.*;
import org.metadatacenter.cadsr.ingestor.util.CedarServerUtil;
import org.metadatacenter.cadsr.ingestor.util.Constants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,7 @@ public static Map<String, CdeSummary> getExistingCedarCdeSummaries(String cedarF
String apiKey) throws IOException {
// Retrieve existing CDEs from CEDAR
logger.info("Retrieving current CDEs from CEDAR (folder id: " + cedarFolderId + ").");
List fieldNamesToInclude = new ArrayList(Arrays.asList(new String[]{"schema:identifier", "pav:version",
"sourceHash"}));
List fieldNamesToInclude = new ArrayList(List.of("schema:identifier", "pav:version", "sourceHash"));
List<CdeSummary> cdeSummaries = CedarServices.findCdeSummariesInFolder(cedarFolderId,
fieldNamesToInclude, true, cedarEnvironment, apiKey);
logger.info("Number of CDEs retrieved from CEDAR: " + cdeSummaries.size() + ".");
Expand Down Expand Up @@ -188,9 +187,7 @@ public static Map<String, Object> removeNonRelevantKeysFromCdeFieldMap(Map<Strin
public static String reformatVersioningNumber(String version) {
String[] originalVersion = version.split("\\.");
String[] baseVersion = {"0", "0", "0"};
for (int i = 0; i < originalVersion.length; i++) {
baseVersion[i] = originalVersion[i];
}
System.arraycopy(originalVersion, 0, baseVersion, 0, originalVersion.length);
return String.join(".", baseVersion);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ public static boolean isFieldInPath(String fieldId, String folderPath, CedarServ
public static Optional<Map<String, Object>> searchCdeByPublicIdAndVersion(String publicId, String version,
CedarServer cedarServer, String apiKey) throws IOException {
String reformattedVersion = CdeUtil.reformatVersioningNumber(version);
List<CedarResourceType> resourceTypes = Arrays.asList(new CedarResourceType[]{CedarResourceType.FIELD});
List<CedarResourceType> resourceTypes = List.of(CedarResourceType.FIELD);
String searchEndpoint = CedarServerUtil.getSearchEndPoint(publicId, resourceTypes, cedarServer);
HttpURLConnection connection = ConnectionUtil.createAndOpenConnection("GET", searchEndpoint, apiKey);
int responseCode = connection.getResponseCode();
Expand Down Expand Up @@ -235,9 +235,7 @@ public static String createCde(Map<String, Object> cdeFieldMap, String cdeHashCo

// Extract the categories from the map if they are still there. They are not part of the CEDAR model, so we
// don't want to post them
if (cdeFieldMap.containsKey(CDE_CATEGORY_IDS_FIELD)) {
cdeFieldMap.remove(CDE_CATEGORY_IDS_FIELD);
}
cdeFieldMap.remove(CDE_CATEGORY_IDS_FIELD);

String payload = objectMapper.writeValueAsString(cdeFieldMap);
conn = ConnectionUtil.createAndOpenConnection("POST", templateFieldsEndpoint, apiKey);
Expand Down Expand Up @@ -448,8 +446,8 @@ public static void attachCdeToCategories(String cedarCdeId, List<String> cedarCa

if (cedarCategoryIds.size() > 0) {
logger.info("Attaching CDE to the following categories: ");
for (int i=0; i<cedarCategoryIds.size(); i++) {
logger.info(" - " + cedarCategoryIds.get(i));
for (String cedarCategoryId : cedarCategoryIds) {
logger.info(" - " + cedarCategoryId);
}
}

Expand Down
Loading

0 comments on commit a359cc8

Please sign in to comment.