diff --git a/karma-app/build b/karma-app/build index c890da29e..2fd3cddc0 100755 --- a/karma-app/build +++ b/karma-app/build @@ -27,9 +27,11 @@ mv $CURRENT/target/cluster*.war $CURRENT/target/clusterService.war cd $CURRENT/target # download tomcat binary -wget http://apache.cs.utah.edu/tomcat/tomcat-7/v7.0.72/bin/apache-tomcat-7.0.72.zip -O tomcat.zip +wget https://github.com/alseambusher/karma-app-deps/archive/master.zip +unzip master.zip +mv karma-app-deps-master/*.tar.gz karma-app-deps-master/*.zip . +rm -rf karma-app-deps-master master.zip -# TODO if this fails, exit unzip tomcat.zip rm tomcat.zip @@ -47,9 +49,16 @@ cd $CURRENT/desktop npm install npm run prerelease cd app + +tar -xzvf $CURRENT/target/jre*windows*.tar.gz electron-packager . --platform=win32 --arch=x64 --icon=../resources/icons/karma.ico --overwrite -electron-packager . --platform=linux --arch=x64 --icon=../resources/icons/karma.png --overwrite +rm -rf jre* +tar -xzvf $CURRENT/target/jre*macosx*.tar.gz electron-packager . --platform=darwin --arch=x64 --icon=../resources/icons/karma.icns --overwrite +rm -rf jre* +tar -xzvf $CURRENT/target/jre*linux*.tar.gz +electron-packager . --platform=linux --arch=x64 --icon=../resources/icons/karma.png --overwrite +rm -rf jre* cp -rf Karma-darwin* $CURRENT/target/Karma-Mac cp -rf Karma-linux* $CURRENT/target/Karma-Linux @@ -57,6 +66,7 @@ cp -rf Karma-win* $CURRENT/target/Karma-Windows rm -rf Karma-* cd $CURRENT/target +rm *.tar.gz #put our license for build_folder in `ls |grep Karma` diff --git a/karma-app/desktop/src/app.js b/karma-app/desktop/src/app.js index f2bdd27d3..3be3528b2 100644 --- a/karma-app/desktop/src/app.js +++ b/karma-app/desktop/src/app.js @@ -9,6 +9,7 @@ var path = require("path"); var fs = require("fs"); var dialogPolyfill = require("dialog-polyfill"); const {ipcRenderer} = require('electron'); +var glob = require("glob"); console.log('Loaded environment variables:', env); @@ -35,14 +36,27 @@ document.addEventListener('DOMContentLoaded', function () { log(data); }); - log("Starting Karma..."); - karma.start(); - function m_launch(){ karma.launch(); log("Launching Karma. Go to http://localhost:8080 if it doesn't launch."); } - // Launches Karma in browser after 10 seconds. + + // set java home to java shipped with app if JAVA_HOME is not set + karma.getJavaHome((_java_home) => { + if (!_java_home){ + glob(path.join(__dirname, "jre*"), (err, files) => { + if (files[0]){ + if (fs.existsSync(path.join(files[0], "bin", "java" + (/^win/.test(process.platform) ? ".exe" : "")))) { + karma.setJavaHome(files[0]); + } + } + }); + } + }); + setTimeout(() => { + log("Starting Karma..."); + karma.start(); + }, 1000); setTimeout(m_launch, 10000); document.getElementById("launch").onclick = m_launch; document.getElementById("restart").onclick = function(){ @@ -62,11 +76,11 @@ document.addEventListener('DOMContentLoaded', function () { document.getElementById("submit_setJavaHome").onclick = function(){ let value = document.getElementById("input_setJavaHome").value; if (!fs.existsSync(path.join(value, "bin", "java" + (/^win/.test(process.platform) ? ".exe" : "")))) { - // if (!fs.existsSync(value + path.sep + "bin" + path.sep + "java" + (/^win/.test(process.platform) ? ".exe" : ""))) { document.querySelector("#dialog_setJavaHome .warning").innerHTML = "Invalid path for Java"; setTimeout(() => {dialog_setJavaHome.showModal();}, 1000); } else { karma.setJavaHome(value); + log("Your JAVA_HOME is set. Restart karma."); } }; @@ -117,6 +131,7 @@ document.addEventListener('DOMContentLoaded', function () { dialog_setJavaHome.showModal(); }); }); + }); diff --git a/karma-app/desktop/src/karma.js b/karma-app/desktop/src/karma.js index b424365b2..6fe66780d 100644 --- a/karma-app/desktop/src/karma.js +++ b/karma-app/desktop/src/karma.js @@ -43,7 +43,9 @@ exports.start = function(){ process.env.JAVA_HOME = _java_home; process.env.JRE_HOME = _java_home; } - exec(exports.tomcat.startcmd, options, function(error, stdout, stderr) { + // cd to catalina_home in windows and it doesnt work in some machines + let cmd = (/^win/.test(process.platform) ? 'cd /D ' + options.cwd + ' & ' : '') + exports.tomcat.startcmd; + exec(cmd, options, function(error, stdout, stderr) { console.log(error); console.log(stdout); console.log(stderr); @@ -108,7 +110,12 @@ exports.setJavaHome = function(value){ exports.getJavaHome = function(callback){ var env = jetpack.cwd(__dirname).read('env.json', 'json'); - callback(env.java_home); + let java_home = env.java_home; + // if it is not set in the app, get it from global environment vars + if (!java_home) { + java_home = process.env.JAVA_HOME ? process.env.JAVA_HOME : process.env.JRE_HOME; + } + callback(java_home); }; exports.setJavaHomeHelp = function() { diff --git a/karma-commands/commands-alignment-openrdf/src/main/java/edu/isi/karma/controller/command/worksheet/ApplyHistoryFromR2RMLModelCommand.java b/karma-commands/commands-alignment-openrdf/src/main/java/edu/isi/karma/controller/command/worksheet/ApplyHistoryFromR2RMLModelCommand.java index 68aef2614..72ec56bc5 100644 --- a/karma-commands/commands-alignment-openrdf/src/main/java/edu/isi/karma/controller/command/worksheet/ApplyHistoryFromR2RMLModelCommand.java +++ b/karma-commands/commands-alignment-openrdf/src/main/java/edu/isi/karma/controller/command/worksheet/ApplyHistoryFromR2RMLModelCommand.java @@ -82,7 +82,7 @@ protected ApplyHistoryFromR2RMLModelCommand(String id, String model, String mode } private enum JsonKeys { - updateType, worksheetId, baseURI, prefix, graphLabel + updateType, worksheetId, baseURI, prefix, graphLabel, GithubURL } @Override @@ -193,6 +193,9 @@ public void generateJson(String prefix, PrintWriter pw, outputObject.put(JsonKeys.prefix.name(), props.getPropertyValue(Property.prefix)); if (props.getPropertyValue(Property.graphLabel) != null && !props.getPropertyValue(Property.graphLabel).trim().isEmpty()) outputObject.put(JsonKeys.graphLabel.name(), props.getPropertyValue(Property.graphLabel)); + if (props.getPropertyValue(Property.GithubURL) != null && !props.getPropertyValue(Property.GithubURL).trim().isEmpty()) + outputObject.put(JsonKeys.GithubURL.name(), props.getPropertyValue(Property.GithubURL)); + pw.println(outputObject.toString()); } catch (JSONException e) { e.printStackTrace(); diff --git a/karma-commands/commands-import/import-csv/src/main/java/edu/isi/karma/controller/update/CSVImportPreviewUpdate.java b/karma-commands/commands-import/import-csv/src/main/java/edu/isi/karma/controller/update/CSVImportPreviewUpdate.java index aefea0688..a1e3fa1fc 100644 --- a/karma-commands/commands-import/import-csv/src/main/java/edu/isi/karma/controller/update/CSVImportPreviewUpdate.java +++ b/karma-commands/commands-import/import-csv/src/main/java/edu/isi/karma/controller/update/CSVImportPreviewUpdate.java @@ -21,10 +21,13 @@ package edu.isi.karma.controller.update; import java.io.File; +import java.io.FileReader; import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintWriter; import java.io.StringReader; +import java.io.InputStreamReader; +import java.io.FileInputStream; import java.util.ArrayList; import java.util.List; import java.util.Scanner; @@ -36,7 +39,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import au.com.bytecode.opencsv.CSVReader; +import com.opencsv.CSVReader; import edu.isi.karma.util.EncodingDetector; import edu.isi.karma.view.VWorkspace; @@ -92,7 +95,7 @@ public CSVImportPreviewUpdate(char delimiterChar, char quoteChar, @Override public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) { - Scanner scanner = null; + // Scanner scanner = null; int rowCount = 0; int previewRowCounter = 0; @@ -105,7 +108,7 @@ public void generateJson(String prefix, PrintWriter pw, if (encoding == null) { encoding = EncodingDetector.detect(csvFile); } - scanner = new Scanner(csvFile, encoding); + // scanner = new Scanner(csvFile, encoding); JSONWriter writer = jsonStr.object().key(JsonKeys.commandId.name()) .value(commandId).key(GenericJsonKeys.updateType.name()) .value("ImportCSVPreview").key(JsonKeys.fileName.name()) @@ -113,18 +116,17 @@ public void generateJson(String prefix, PrintWriter pw, .key("maxNumLines").value(maxNumLines); JSONArray dataRows = new JSONArray(); - while (scanner.hasNextLine()) { + CSVReader reader = new CSVReader( + new InputStreamReader(new FileInputStream(csvFile), encoding), + delimiter, quoteCharacter, escapeCharacter); + String[] rowValues = null; + while ((rowValues = reader.readNext()) != null) { // Check for the header row if (rowCount + 1 == headerRowIndex) { - String line = scanner.nextLine(); - CSVReader reader = new CSVReader(new StringReader(line), - delimiter, quoteCharacter, escapeCharacter); - String[] rowValues = reader.readNext(); List headers = new ArrayList<>(); if (rowValues == null || rowValues.length == 0) { logger.error("No data found in the Header row!"); rowCount++; - scanner.nextLine(); continue; } for (int i = 0; i < rowValues.length; i++) { @@ -137,7 +139,6 @@ public void generateJson(String prefix, PrintWriter pw, JSONArray arr = new JSONArray(headers); writer.key(JsonKeys.headers.name()).value(arr); rowCount++; - reader.close(); continue; } @@ -147,10 +148,6 @@ public void generateJson(String prefix, PrintWriter pw, if (previewRowCounter++ > 5) { break; } - String line = scanner.nextLine(); - CSVReader reader = new CSVReader(new StringReader(line), - delimiter, quoteCharacter, escapeCharacter); - String[] rowValues = reader.readNext(); List vals = new ArrayList<>(); if (rowValues != null) { for (int i = 0; i < rowValues.length; i++) { @@ -160,7 +157,6 @@ public void generateJson(String prefix, PrintWriter pw, vals.add(""); // Add the row index vals.add(0, Integer.toString(rowCount + 1)); - reader.close(); // Add to the data rows JSON dataRows.put(vals); @@ -169,9 +165,9 @@ public void generateJson(String prefix, PrintWriter pw, } rowCount++; - scanner.nextLine(); } + reader.close(); writer.key(JsonKeys.rows.name()).value(dataRows); writer.endObject(); pw.println(jsonStr.toString()); @@ -182,8 +178,8 @@ public void generateJson(String prefix, PrintWriter pw, } catch (JSONException e) { logger.error("Error occured while writing to JSON", e); } finally { - if(scanner != null) - scanner.close(); + // if(scanner != null) + // scanner.close(); } } diff --git a/karma-commands/commands-publish/src/main/java/edu/isi/karma/controller/command/publish/PublishGithubCommand.java b/karma-commands/commands-publish/src/main/java/edu/isi/karma/controller/command/publish/PublishGithubCommand.java new file mode 100644 index 000000000..903da3c1e --- /dev/null +++ b/karma-commands/commands-publish/src/main/java/edu/isi/karma/controller/command/publish/PublishGithubCommand.java @@ -0,0 +1,314 @@ +package edu.isi.karma.controller.command.publish; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.DefaultHttpClient; +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import edu.isi.karma.config.ModelingConfiguration; +import edu.isi.karma.config.ModelingConfigurationRegistry; +import edu.isi.karma.controller.command.Command; +import edu.isi.karma.controller.command.CommandException; +import edu.isi.karma.controller.command.CommandType; +import edu.isi.karma.controller.update.AbstractUpdate; +import edu.isi.karma.controller.update.ErrorUpdate; +import edu.isi.karma.controller.update.InfoUpdate; +import edu.isi.karma.controller.update.UpdateContainer; +import edu.isi.karma.rep.Worksheet; +import edu.isi.karma.rep.Workspace; +import edu.isi.karma.rep.metadata.WorksheetProperties; +import edu.isi.karma.rep.metadata.WorksheetProperties.Property; +import edu.isi.karma.view.VWorkspace; +import edu.isi.karma.webserver.ContextParametersRegistry; +import edu.isi.karma.webserver.ServletContextParameterMap; +import edu.isi.karma.webserver.WorkspaceKarmaHomeRegistry; + +/** + * Created by alse on 11/21/16. + * This command publishes dot, model, model json and report to github + */ +public class PublishGithubCommand extends Command { + private static Logger logger = LoggerFactory.getLogger(PublishGithubCommand.class); + private String worksheetId; + private String repo; + private String auth; + private String branch; + private HashMap fileSHAMap = new HashMap<>(); + private String graphvizServer; + + private enum JsonKeys { + updateType, url, worksheetId + } + + /* + worksheetId - is the id of the worksheet that has to be published + repo - is the github url of the repo where the files have to be published + branch - is the branch of the repo where the files have to be published + auth - is the base64 encoded string of username:password required for authentication + */ + public PublishGithubCommand(String id, String model, String worksheetId, String githubUrl, String auth) { + super(id, model); + this.worksheetId = worksheetId; + + String repoDetails = githubUrl.split("github\\.com")[1]; + int treeIdx = repoDetails.indexOf("/tree/"); + if(treeIdx != -1) { + int endIdx = repoDetails.indexOf("/", treeIdx+6); + String rest = ""; + if(endIdx != -1) { + this.branch = repoDetails.substring(treeIdx+6, endIdx); + rest = repoDetails.substring(endIdx); + } else { + this.branch = repoDetails.substring(treeIdx+6); + } + repoDetails = repoDetails.substring(0, treeIdx) + + "/contents" + rest + "/"; + } else { + this.branch = "master"; + repoDetails = repoDetails + "/contents/"; + } + + this.repo = "https://api.github.com/repos" + repoDetails; + this.auth = auth; + } + @Override + public String getCommandName() { + return this.getClass().getSimpleName(); + } + + @Override + public String getTitle() { + return "Publish Github Command"; + } + + @Override + public String getDescription() { + return null; + } + + @Override + public CommandType getCommandType() { + return CommandType.notInHistory; + } + + @Override + public UpdateContainer doIt(Workspace workspace) throws CommandException { + UpdateContainer uc = new UpdateContainer(); + try{ + try { + this.buildFileSHAMap(); + } catch (FileNotFoundException fe) { + logger.warn("Github URL does not exist. Will try to see if it can be created."); + } + ModelingConfiguration modelingConfiguration = ModelingConfigurationRegistry.getInstance() + .getModelingConfiguration(WorkspaceKarmaHomeRegistry.getInstance().getKarmaHome(workspace.getId())); + this.graphvizServer = modelingConfiguration.getGraphvizServer(); + + Worksheet worksheet = workspace.getWorksheet(this.worksheetId); + + WorksheetProperties props = worksheet.getMetadataContainer().getWorksheetProperties(); + String modelName = props.getPropertyValue(Property.graphLabel); + String worksheetTitle = worksheet.getTitle(); + + String dotFile = ContextParametersRegistry.getInstance() + .getContextParameters(ContextParametersRegistry.getInstance().getDefault().getId()) + .getParameterValue(ServletContextParameterMap.ContextParameter.GRAPHVIZ_MODELS_DIR) + + worksheetTitle + ".model.dot"; + + String modelFile = ContextParametersRegistry.getInstance() + .getContextParameters(ContextParametersRegistry.getInstance().getDefault().getId()) + .getParameterValue(ServletContextParameterMap.ContextParameter.R2RML_PUBLISH_DIR) + + modelName + "-model.ttl"; + + String reportFile = ContextParametersRegistry.getInstance() + .getContextParameters(ContextParametersRegistry.getInstance().getDefault().getId()) + .getParameterValue(ServletContextParameterMap.ContextParameter.REPORT_PUBLISH_DIR) + + worksheetTitle + ".md"; + + String modelJsonFile = ContextParametersRegistry.getInstance() + .getContextParameters(ContextParametersRegistry.getInstance().getDefault().getId()) + .getParameterValue(ServletContextParameterMap.ContextParameter.JSON_MODELS_DIR) + + worksheetTitle + ".model.json"; + + if (fileExists(dotFile)) { + String contents = getFileContents(dotFile); + push(modelName + "-model.dot", contents); + try { + //Use hosted graphviz server to convert dot to pdf + //https://github.com/omerio/graphviz-server + contents = contents.replace("digraph n0", "digraph G"); + InputStream pdfStream = this.getGraphizPdf(contents); + push(modelName + "-model.pdf", pdfStream); + } catch(Exception e) { + logger.error("Error generating png for the dot file", e); + } + } + + if (fileExists(modelFile)) { + String contents = getFileContents(modelFile); + push(modelName + "-model.ttl", contents); + } + + if (fileExists(reportFile)) { + String contents = getFileContents(reportFile); + push(modelName + "-model.md", contents); + } + + if (fileExists(modelJsonFile)) { + String contents = getFileContents(modelJsonFile); + push(modelName + "-model.json", contents); + } + + uc.add(new AbstractUpdate() { + @Override + public void generateJson(String prefix, PrintWriter pw, VWorkspace vWorkspace) { + try { + JSONObject outputObject = new JSONObject(); + outputObject.put(JsonKeys.updateType.name(), "PublishGithubUpdate"); + outputObject.put(JsonKeys.url.name(), repo); + outputObject.put(JsonKeys.worksheetId.name(), worksheetId); + pw.println(outputObject.toString(4)); + pw.println(","); + new InfoUpdate("Succesfully pushed model to Github").generateJson(prefix, pw, vWorkspace); + } catch (Exception e) { + logger.error("Error unable to set Github", e); + } + } + }); + } catch ( FileNotFoundException fe) { + logger.error("Error pushing to Github:" , fe); + uc.add(new ErrorUpdate("Error pushing to Github.
Github URL is invalid")); + } catch (Exception e) { + logger.error("Error pushing to Github:" , e); + uc.add(new ErrorUpdate("Error pushing to Github:
" + e.getMessage())); + } + return uc; + } + + @Override + public UpdateContainer undoIt(Workspace workspace) { + return null; + } + + /* + This function does the push operation to github + */ + private Integer push(String fileName, String fileContents) throws IOException{ + return push(fileName, fileContents.getBytes()); + } + + private Integer push(String fileName, InputStream fileStream) throws IOException { + byte[] bytes = IOUtils.toByteArray(fileStream); + return push(fileName, bytes); + } + + private Integer push(String fileName, byte[] bytes) throws IOException{ + URL url = new URL(this.repo + fileName); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("PUT"); + connection.setDoOutput(true); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestProperty("Accept", "application/json"); + connection.setRequestProperty ("Authorization", "Basic " + this.auth); + OutputStreamWriter osw = new OutputStreamWriter(connection.getOutputStream()); + + String b64FileContent = new String(Base64.encodeBase64(bytes)); + String fileSHA = getFileSHA(fileName); + if (fileSHA == null) { + osw.write("{\"message\": \"Create file " + fileName + + "\", \"branch\":\"" + this.branch + + "\", \"content\": \"" + + b64FileContent + "\"}"); + } else { + osw.write("{\"message\": \"Update file " + fileName + + "\", \"branch\":\"" + this.branch + + "\", \"content\": \"" + + b64FileContent + "\", \"sha\": \"" + fileSHA + "\"}"); + } + osw.flush(); + osw.close(); + return connection.getResponseCode(); + } + + private Boolean fileExists(String path) { + File f = new File(path); + return f.exists() && !f.isDirectory(); + } + + private String getFileContents(String path) throws IOException { + byte[] encoded = Files.readAllBytes(Paths.get(path)); + return new String(encoded, "UTF-8"); + } + + private InputStream getGraphizPdf(String dotContents) throws ClientProtocolException, IOException { + HttpClient httpClient = new DefaultHttpClient(); + HttpPost httpPost = new HttpPost(this.graphvizServer + "pdf"); + httpPost.setEntity(new StringEntity(dotContents)); + HttpResponse response = httpClient.execute(httpPost); + + // Parse the response and store it in a String + HttpEntity entity = response.getEntity(); + return entity.getContent(); + } + + // Whenever we are doing an update instead of create, we need the blob sha of the file which exists on github. + private void buildFileSHAMap() throws IOException { + this.fileSHAMap.clear(); + String repo; //Need to remove the extra / if present, else API does not use the ref parameter + if(this.repo.endsWith("/")) + repo = this.repo.substring(0, this.repo.length()-1); + else + repo = this.repo; + String urlStr = repo + "?ref=" + this.branch; + URL url = new URL(urlStr); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + connection.setDoOutput(true); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestProperty("Accept", "application/json"); + connection.setRequestProperty("Authorization", "Basic " + this.auth); + + BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream())); + String inputLine; + StringBuffer response = new StringBuffer(); + + while ((inputLine = in.readLine()) != null) { + response.append(inputLine); + } + in.close(); + + JSONArray fileTree = new JSONArray(response.toString()); + for(int i=0; i getCorrespondingCommand() { + return PublishGithubCommand.class; + } +} diff --git a/karma-commands/commands-publish/src/main/java/edu/isi/karma/controller/command/publish/PublishReportCommand.java b/karma-commands/commands-publish/src/main/java/edu/isi/karma/controller/command/publish/PublishReportCommand.java index 821594b1f..bebf1f598 100644 --- a/karma-commands/commands-publish/src/main/java/edu/isi/karma/controller/command/publish/PublishReportCommand.java +++ b/karma-commands/commands-publish/src/main/java/edu/isi/karma/controller/command/publish/PublishReportCommand.java @@ -3,6 +3,7 @@ import java.io.File; import java.io.IOException; import java.io.PrintWriter; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; @@ -33,7 +34,7 @@ import edu.isi.karma.rep.alignment.Label; import edu.isi.karma.rep.alignment.LabeledLink; import edu.isi.karma.rep.alignment.Node; -import edu.isi.karma.rep.alignment.NodeType; +import edu.isi.karma.view.VWorksheet; import edu.isi.karma.view.VWorkspace; import edu.isi.karma.webserver.ContextParametersRegistry; import edu.isi.karma.webserver.ServletContextParameterMap; @@ -71,7 +72,7 @@ public String getDescription() { @Override public CommandType getCommandType() { - return CommandType.undoable; + return CommandType.notInHistory; } @Override @@ -98,18 +99,31 @@ public void generateJson(String prefix, PrintWriter pw, parentDir.mkdirs(); PrintWriter fileWriter = new PrintWriter(f); - fileWriter.println("## " + worksheet.getTitle()); + fileWriter.println("# " + worksheet.getTitle()); fileWriter.println(); - fileWriter.println("### PyTransforms"); + fileWriter.println("## Add Column"); + writeAddColumns(finalWorkspace, worksheet, fileWriter); + + fileWriter.println(); + fileWriter.println("## Add Node/Literal"); + writeAddNodes(finalWorkspace, worksheet, fileWriter); + + fileWriter.println(); + fileWriter.println("## PyTransforms"); writePyTransforms(finalWorkspace, worksheet, fileWriter); fileWriter.println(); - fileWriter.println("### Semantic Types"); - writeSemanticTypes(finalWorkspace, worksheet, fileWriter); + fileWriter.println("## Selections"); + writeSelections(finalWorkspace, worksheet, fileWriter); fileWriter.println(); - fileWriter.println("### Links"); + fileWriter.println("## Semantic Types"); + VWorksheet viewWorksheet = vWorkspace.getViewFactory().getVWorksheetByWorksheetId(worksheetId); + writeSemanticTypes(vWorkspace, viewWorksheet, fileWriter); + + fileWriter.println(); + fileWriter.println("## Links"); writeLinks(finalWorkspace, worksheet, fileWriter); fileWriter.close(); @@ -122,7 +136,7 @@ public void generateJson(String prefix, PrintWriter pw, pw.println(","); new InfoUpdate("Succesfully generated report").generateJson(prefix, pw, vWorkspace); } catch(Exception e) { - pw.println("Error generting report"); + pw.println("Error generating report"); logger.error("Error generating report", e); } @@ -145,42 +159,145 @@ private void writePyTransforms(Workspace workspace, Worksheet worksheet, PrintWr JSONArray history = CommandHistory.getHistorySaver(workspace.getId()).loadHistory(historyFile); for(int i=0; i stringStringEntry : pyTransformMap.entrySet()) { + pw.print(stringStringEntry.getValue()); + } + } + + private void writeSelections(Workspace workspace, Worksheet worksheet, PrintWriter pw) throws IOException { + + //Map for py transforms, so only the latest pyTransform for a column gets written + String linebreak = System.getProperty("line.separator"); + + String historyFile = CommandHistory.getHistorySaver(workspace.getId()).getHistoryFilepath(worksheetId); + try { + JSONArray history = CommandHistory.getHistorySaver(workspace.getId()).loadHistory(historyFile); + + for(int i=0; iOperation: `" + operation + "`").append(linebreak); + pyTransform.append("``` python").append(linebreak); + pyTransform.append(code).append(linebreak); + pyTransform.append("```").append(linebreak); + pyTransform.append(linebreak); + + pw.print(pyTransform.toString()); } - - StringBuffer pyTransform = new StringBuffer("#### _" + columnName + "_"); - pyTransform.append(linebreak); - - //Pedro: throw-away code, ought to have a better way to construct column paths. - JSONArray hNodeIdArray = HistoryJsonUtil.getJSONArrayValue("hNodeId", inputParamArr); - String invocationColumnName = ""; - String sep = ""; - for(int j=0; j pyTransformMap = new LinkedHashMap<>(); + String linebreak = System.getProperty("line.separator"); + + String historyFile = CommandHistory.getHistorySaver(workspace.getId()).getHistoryFilepath(worksheetId); + try { + + + JSONArray history = CommandHistory.getHistorySaver(workspace.getId()).loadHistory(historyFile); + + for(int i=0; iValue: `" + value + "`").append(linebreak); + + pyTransform.append(linebreak); + + pyTransformMap.put(columnName, pyTransform.toString()); } - pyTransform.append("From column: _" + invocationColumnName + "_").append(linebreak); - - pyTransform.append("``` python").append(linebreak); - pyTransform.append(code).append(linebreak); - pyTransform.append("```").append(linebreak); - pyTransform.append(linebreak); - - pyTransformMap.put(columnName, pyTransform.toString()); + } catch(Exception e) { + logger.error("Error writing AddColumns for Publish Report"); } } } catch(Exception e) { @@ -190,27 +307,93 @@ private void writePyTransforms(Workspace workspace, Worksheet worksheet, PrintWr //Now get all transforms from the map and write them to the writer for(Map.Entry stringStringEntry : pyTransformMap.entrySet()) { pw.print(stringStringEntry.getValue()); - } + } } + private void writeAddNodes(Workspace workspace, Worksheet worksheet, PrintWriter pw) throws IOException { + //Map for py transforms, so only the latest pyTransform for a column gets written + LinkedHashMap pyTransformMap = new LinkedHashMap<>(); + String linebreak = System.getProperty("line.separator"); + + String historyFile = CommandHistory.getHistorySaver(workspace.getId()).getHistoryFilepath(worksheetId); + try { + + + JSONArray history = CommandHistory.getHistorySaver(workspace.getId()).loadHistory(historyFile); + + for(int i=0; iId: `" + node_id + "`").append(linebreak); + pyTransform.append(linebreak); + pyTransformMap.put(nodeLabel, pyTransform.toString()); + } else if(commandName.equals("AddLiteralNodeCommand")) { + String value = HistoryJsonUtil.getStringValue("literalValue", inputParamArr); + String literalType = HistoryJsonUtil.getStringValue("literalType", inputParamArr); + String language = HistoryJsonUtil.getStringValue("language", inputParamArr); + boolean isUri = HistoryJsonUtil.getBooleanValue("isUri", inputParamArr); + + StringBuffer pyTransform = new StringBuffer("#### Literal Node: `" + value + "`"); + pyTransform.append(linebreak); + pyTransform.append("Literal Type: `" + literalType + "`").append(linebreak); + pyTransform.append("
Language: `" + language + "`").append(linebreak); + pyTransform.append("
isUri: `" + isUri + "`").append(linebreak); + + pyTransform.append(linebreak); + pyTransformMap.put(value, pyTransform.toString()); + } + } catch(Exception e) { + logger.error("Error writing AddNodes for Publish Report"); + } + } + } catch(Exception e) { + logger.error("Error reading history file:" + historyFile); + } + + //Now get all transforms from the map and write them to the writer + for(Map.Entry stringStringEntry : pyTransformMap.entrySet()) { + pw.print(stringStringEntry.getValue()); + } + + } - private void writeSemanticTypes(Workspace workspace, Worksheet worksheet, PrintWriter pw) { + private void writeSemanticTypes(VWorkspace vWorkspace, VWorksheet vWorksheet, PrintWriter pw) { pw.println("| Column | Property | Class |"); pw.println("| ----- | -------- | ----- |"); - Alignment alignment = AlignmentManager.Instance().getAlignment(workspace.getId(), worksheet.getId()); + List hNodeIdList = vWorksheet.getHeaderVisibleLeafNodes(); + + String alignmentId = AlignmentManager.Instance().constructAlignmentId( + vWorkspace.getWorkspace().getId(), vWorksheet.getWorksheetId()); + Alignment alignment = AlignmentManager.Instance().getAlignment(alignmentId); + if(alignment != null) { - List columnNodes = Arrays.asList(alignment.getNodesByType(NodeType.ColumnNode).toArray(new Node[0])); - Collections.sort(columnNodes, new Comparator() { - @Override - public int compare(Node n1, Node n2) { - String l1name = getClassName(n1); - String l2name = getClassName(n2); - return l1name.compareTo(l2name); - } - }); + List columnNodes = new ArrayList<>(); + for(String hNodeId: hNodeIdList) { + Node columnNode = alignment.getNodeById(hNodeId); + columnNodes.add(columnNode); + } + Collections.sort(columnNodes, new Comparator() { + @Override + public int compare(Node n1, Node n2) { + String l1name = getClassName(n1); + String l2name = getClassName(n2); + return l1name.compareTo(l2name); + } + }); if(columnNodes != null) { for(Node node : columnNodes) { diff --git a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/SetWorksheetPropertiesCommand.java b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/SetWorksheetPropertiesCommand.java index 333726457..ee46abbeb 100644 --- a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/SetWorksheetPropertiesCommand.java +++ b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/SetWorksheetPropertiesCommand.java @@ -61,18 +61,27 @@ public String getDescription() { JSONObject propertiesJson = new JSONObject(properties); String desc = ""; String sep = ""; - if (propertiesJson.getBoolean(Property.hasServiceProperties.name())) { + if (propertiesJson.has(Property.hasServiceProperties.name()) && propertiesJson.getBoolean(Property.hasServiceProperties.name())) { desc = "Service"; sep = ", "; } - if (propertiesJson.getBoolean("hasPrefix")) { + if (propertiesJson.has(Property.graphLabel.name()) && + !propertiesJson.getString(Property.graphLabel.name()).isEmpty()) { + desc = desc + sep + "Model Name: " + propertiesJson.getString(Property.graphLabel.name()); + sep = ", "; + } + if (propertiesJson.has("hasPrefix") && propertiesJson.getBoolean("hasPrefix")) { desc = desc + sep + "Prefix: " + propertiesJson.getString(Property.prefix.name()); sep = ", "; } - if (propertiesJson.getBoolean("hasBaseURI")) { + if (propertiesJson.has("hasBaseURI") && propertiesJson.getBoolean("hasBaseURI")) { desc = desc + sep + "Base URI: " + propertiesJson.getString(Property.baseURI.name()); sep = ", "; } + if (propertiesJson.has("hasGithubURL") && propertiesJson.getBoolean("hasGithubURL")) { + desc = desc + sep + "Github URL: " + propertiesJson.getString(Property.GithubURL.name()); + sep = ", "; + } return desc; } @@ -116,6 +125,10 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException { propertiesJson.getString(Property.serviceDataPostMethod.name())); } } + + if (propertiesJson.has(Property.GithubURL.name()) && !propertiesJson.getString(Property.GithubURL.name()).isEmpty()) + props.setPropertyValue(Property.GithubURL, propertiesJson.getString(Property.GithubURL.name())); + if (propertiesJson.getBoolean("hasPrefix")) { props.setPropertyValue(Property.prefix, propertiesJson.getString(Property.prefix.name())); diff --git a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/SplitColumnByDelimiter.java b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/SplitColumnByDelimiter.java index c17212d60..f8f3a5f73 100644 --- a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/SplitColumnByDelimiter.java +++ b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/SplitColumnByDelimiter.java @@ -13,7 +13,7 @@ import com.kenai.jffi.Array; -import au.com.bytecode.opencsv.CSVReader; +import com.opencsv.CSVReader; import edu.isi.karma.controller.command.selection.SuperSelection; import edu.isi.karma.er.helper.CloneTableUtils; import edu.isi.karma.rep.CellValue; diff --git a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/ClearSelectionCommand.java b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/ClearSelectionCommand.java index f79cb629c..ab099f1a7 100644 --- a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/ClearSelectionCommand.java +++ b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/ClearSelectionCommand.java @@ -78,9 +78,14 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException { } } } + WorksheetUpdateFactory.detectSelectionStatusChange(worksheetId, workspace, this); - UpdateContainer uc = WorksheetUpdateFactory.createWorksheetHierarchicalAndCleaningResultsUpdates(worksheetId, superSel, workspace.getContextId()); - return uc; + if(!this.isExecutedInBatch()) { + UpdateContainer uc = WorksheetUpdateFactory.createWorksheetHierarchicalAndCleaningResultsUpdates(worksheetId, superSel, workspace.getContextId()); + return uc; + } else { + return new UpdateContainer(); + } } @Override diff --git a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/OperateSelectionCommand.java b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/OperateSelectionCommand.java index 5658b0b5c..55e3f136b 100644 --- a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/OperateSelectionCommand.java +++ b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/OperateSelectionCommand.java @@ -92,8 +92,12 @@ public UpdateContainer doIt(Workspace workspace) throws CommandException { return getErrorUpdate("The operation is undefined"); } WorksheetUpdateFactory.detectSelectionStatusChange(worksheetId, workspace, this); - UpdateContainer uc = WorksheetUpdateFactory.createWorksheetHierarchicalAndCleaningResultsUpdates(worksheetId, superSel, workspace.getContextId()); - return uc; + if(!this.isExecutedInBatch()) { + UpdateContainer uc = WorksheetUpdateFactory.createWorksheetHierarchicalAndCleaningResultsUpdates(worksheetId, superSel, workspace.getContextId()); + return uc; + } else { + return new UpdateContainer(); + } } @Override diff --git a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/RefreshSelectionCommand.java b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/RefreshSelectionCommand.java index 1d139b534..f8035deb3 100644 --- a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/RefreshSelectionCommand.java +++ b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/RefreshSelectionCommand.java @@ -10,6 +10,7 @@ import edu.isi.karma.controller.command.CommandException; import edu.isi.karma.controller.command.CommandType; import edu.isi.karma.controller.command.WorksheetSelectionCommand; +import edu.isi.karma.controller.command.ICommand.CommandTag; import edu.isi.karma.controller.command.selection.LargeSelection.Operation; import edu.isi.karma.controller.command.selection.Selection; import edu.isi.karma.controller.command.selection.SelectionManager; @@ -37,6 +38,7 @@ public RefreshSelectionCommand(String id, String model, String worksheetId, super(id, model, worksheetId, selectionId); this.hNodeId = hNodeId; addTag(CommandTag.Selection); + addTag(CommandTag.IgnoreInBatch); } @Override diff --git a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/RefreshSuperSelectionCommand.java b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/RefreshSuperSelectionCommand.java index 449ed60a6..c38b70704 100644 --- a/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/RefreshSuperSelectionCommand.java +++ b/karma-commands/commands-worksheet/src/main/java/edu/isi/karma/controller/command/worksheet/selection/RefreshSuperSelectionCommand.java @@ -11,6 +11,7 @@ import edu.isi.karma.controller.command.CommandException; import edu.isi.karma.controller.command.CommandType; import edu.isi.karma.controller.command.WorksheetSelectionCommand; +import edu.isi.karma.controller.command.ICommand.CommandTag; import edu.isi.karma.controller.command.selection.Selection; import edu.isi.karma.controller.command.selection.SuperSelection; import edu.isi.karma.controller.history.HistoryJsonUtil.ParameterType; @@ -22,6 +23,7 @@ import edu.isi.karma.rep.Worksheet; import edu.isi.karma.rep.Workspace; import edu.isi.karma.util.CommandInputJSONUtil; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +35,7 @@ public RefreshSuperSelectionCommand(String id, String model, String worksheetId, String selectionId) { super(id, model, worksheetId, selectionId); addTag(CommandTag.Selection); + addTag(CommandTag.IgnoreInBatch); } private class SelectionWithPriority implements Comparable{ diff --git a/karma-common/pom.xml b/karma-common/pom.xml index ac3631279..5f7ed02e8 100644 --- a/karma-common/pom.xml +++ b/karma-common/pom.xml @@ -122,10 +122,11 @@ reflections - - net.sf.opencsv - opencsv - + + com.opencsv + opencsv + 3.8 + org.apache.httpcomponents diff --git a/karma-common/src/main/java/edu/isi/karma/config/ModelingConfiguration.java b/karma-common/src/main/java/edu/isi/karma/config/ModelingConfiguration.java index 3de248bcd..0afeac724 100644 --- a/karma-common/src/main/java/edu/isi/karma/config/ModelingConfiguration.java +++ b/karma-common/src/main/java/edu/isi/karma/config/ModelingConfiguration.java @@ -81,6 +81,7 @@ public class ModelingConfiguration { private Boolean showModelsWithoutMatching; private String defaultProperty = null; + private String graphvizServer = null; private final String newLine = System.getProperty("line.separator"); @@ -163,7 +164,8 @@ public class ModelingConfiguration { "##########################################################################################" + newLine + "" + newLine + "models.display.nomatching=false" + newLine + - "history.store.old=false" + "history.store.old=false" + newLine + + "graphiz.server=http://52.38.65.60/graphviz/" ; @@ -287,9 +289,19 @@ public void load() { if(defaultProperty == null) { //need to add this property to the end PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(file, true))); + out.println(); out.println("default.property=http://schema.org/name"); out.close(); } + + graphvizServer = modelingProperties.getProperty("graphviz.server"); + if(graphvizServer == null) { + PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(file, true))); + out.println(); + graphvizServer = "http://52.38.65.60/graphviz/"; + out.println("graphviz.server=" + graphvizServer); + out.close(); + } } catch (IOException e) { logger.error("Error occured while reading config file ...", e); System.exit(1); @@ -550,5 +562,8 @@ public void setManualAlignment() knownModelsAlignment = false; } + public String getGraphvizServer() { + return graphvizServer; + } } diff --git a/karma-common/src/main/java/edu/isi/karma/imp/csv/CSVImport.java b/karma-common/src/main/java/edu/isi/karma/imp/csv/CSVImport.java index 63ecebd26..b29b499de 100644 --- a/karma-common/src/main/java/edu/isi/karma/imp/csv/CSVImport.java +++ b/karma-common/src/main/java/edu/isi/karma/imp/csv/CSVImport.java @@ -1,6 +1,6 @@ package edu.isi.karma.imp.csv; -import au.com.bytecode.opencsv.CSVReader; +import com.opencsv.CSVReader; import edu.isi.karma.imp.Import; import edu.isi.karma.rep.*; import edu.isi.karma.rep.HNode.HNodeType; @@ -19,7 +19,7 @@ public class CSVImport extends Import { private static Logger logger = LoggerFactory.getLogger(CSVImport.class); - protected final int headerRowIndex; + protected final int headerRowIndex; protected final int dataStartRowIndex; protected final char delimiter; protected final char quoteCharacter; @@ -63,113 +63,96 @@ public CSVImport(int headerRowIndex, int dataStartRowIndex, public Worksheet generateWorksheet() throws IOException, KarmaException { Table dataTable = getWorksheet().getDataTable(); - BufferedReader br = getLineReader(); - - // Index for row currently being read int rowCount = 0; Map hNodeIdList = new HashMap<>(); - // If no row is present for the column headers - if (headerRowIndex == 0) { - hNodeIdList = addEmptyHeaders(getWorksheet(), getFactory(), br); - if (hNodeIdList == null || hNodeIdList.isEmpty()) { - br.close(); - throw new KarmaException("Error occured while counting header " - + "nodes for the worksheet!"); - } - } + CSVReader reader = getCSVReader(); // Populate the worksheet model - String line = null; - while ((line = br.readLine()) != null) { - logger.debug("Read line: '" + line + "'"); + String[] rowValues = null; + while ((rowValues = reader.readNext()) != null) { + // logger.debug("Read line: '" + line + "'"); // Check for the header row if (rowCount + 1 == headerRowIndex) { - hNodeIdList = addHeaders(getWorksheet(), getFactory(), line, br); + hNodeIdList = addHeaders(getWorksheet(), getFactory(), rowValues, reader); rowCount++; continue; } // Populate the model with data rows if (rowCount + 1 >= dataStartRowIndex) { - boolean added = addRow(getWorksheet(), getFactory(), line, hNodeIdList, dataTable); + boolean added = addRow(getWorksheet(), getFactory(), rowValues, hNodeIdList, dataTable); if(added) { - rowCount++; - - if(maxNumLines > 0 && (rowCount - dataStartRowIndex) >= maxNumLines-1) { - break; - } + rowCount++; + if(maxNumLines > 0 && (rowCount - dataStartRowIndex) >= maxNumLines-1) { + break; + } } continue; } rowCount++; } - br.close(); + reader.close(); getWorksheet().getMetadataContainer().getWorksheetProperties().setPropertyValue(Property.sourceType, SourceTypes.CSV.toString()); return getWorksheet(); } - protected BufferedReader getLineReader() throws IOException { - // Prepare the reader for reading file line by line - + protected BufferedReader getLineReader() throws IOException { + // Prepare the reader for reading file line by line InputStreamReader isr = EncodingDetector.getInputStreamReader(is, encoding); - return new BufferedReader(isr); - } + } + + protected CSVReader getCSVReader() throws IOException { + BufferedReader br = getLineReader(); + return new CSVReader(br, delimiter, quoteCharacter, escapeCharacter); + } private Map addHeaders(Worksheet worksheet, RepFactory fac, - String line, BufferedReader br) throws IOException { + String[] rowValues, CSVReader reader) throws IOException { HTable headers = worksheet.getHeaders(); Map headersMap = new HashMap<>(); - CSVReader reader = new CSVReader(new StringReader(line), delimiter, - quoteCharacter, escapeCharacter); - String[] rowValues = null; - rowValues = reader.readNext(); - - if (rowValues == null || rowValues.length == 0) { - reader.close(); - return addEmptyHeaders(worksheet, fac, br); - } - for (int i = 0; i < rowValues.length; i++) { HNode hNode = null; if (headerRowIndex == 0) { - if (isVisible("Column_" + (i + 1))) - hNode = headers.addHNode("Column_" + (i + 1), HNodeType.Regular, worksheet, fac); + if (isVisible("Column_" + (i + 1))) + hNode = headers.addHNode("Column_" + (i + 1), HNodeType.Regular, worksheet, fac); } else { - if (isVisible(rowValues[i])) - hNode = headers.addHNode(rowValues[i], HNodeType.Regular, worksheet, fac); + if (isVisible(rowValues[i])) + hNode = headers.addHNode(rowValues[i], HNodeType.Regular, worksheet, fac); } if (hNode != null) - headersMap.put(i, hNode.getId()); + headersMap.put(i, hNode.getId()); } - reader.close(); + return headersMap; } - private boolean addRow(Worksheet worksheet, RepFactory fac, String line, - Map hNodeIdMap, Table dataTable) throws IOException { - CSVReader reader = new CSVReader(new StringReader(line), delimiter, - quoteCharacter, escapeCharacter); - String[] rowValues = null; - rowValues = reader.readNext(); + private boolean addRow(Worksheet worksheet, RepFactory fac, String[] rowValues, + Map hNodeIdMap, Table dataTable) throws IOException { + if (rowValues == null || rowValues.length == 0) { - reader.close(); return false; } Row row = dataTable.addRow(fac); int size = hNodeIdMap.size(); if (columnsJson != null) - size = columnsJson.length(); + size = columnsJson.length(); for (int i = 0; i < rowValues.length; i++) { + if(i >= size) { + HTable headers = worksheet.getHeaders(); + HNode hNode = headers.addHNode("Column_" + (i + 1), HNodeType.Regular, worksheet, fac); + hNodeIdMap.put(i, hNode.getId()); + size = hNodeIdMap.size(); + } if (i < size) { - String hNodeId = hNodeIdMap.get(i); - if (hNodeId != null) - row.setValue(hNodeId, rowValues[i], fac); + String hNodeId = hNodeIdMap.get(i); + if (hNodeId != null) + row.setValue(hNodeId, rowValues[i], fac); } else { // TODO Our model does not allow a value to be added to a row // without its associated HNode. In CSVs, there could be case @@ -177,58 +160,17 @@ private boolean addRow(Worksheet worksheet, RepFactory fac, String line, logger.error("More data elements detected in the row than number of headers!"); } } - reader.close(); return true; } - private Map addEmptyHeaders(Worksheet worksheet, - RepFactory fac, BufferedReader br) throws IOException { - HTable headers = worksheet.getHeaders(); - Map headersMap = new HashMap<>(); - - - br.mark(1000000); - br.readLine(); - - // Use the first data row to count the number of columns we need to add - int rowCount = 0; - String line = null; - while (null != (line = br.readLine())) { - if (rowCount + 1 == dataStartRowIndex) { - line = br.readLine(); - CSVReader reader = new CSVReader(new StringReader(line), - delimiter, quoteCharacter, escapeCharacter); - String[] rowValues = null; - try { - rowValues = reader.readNext(); - } catch (IOException e) { - logger.error("Error reading Line:" + line, e); - } - for (int i = 0; i < rowValues.length; i++) { - - HNode hNode = null; - if (isVisible("Column_" + (i + 1))) - hNode = headers.addHNode("Column_" + (i + 1), HNodeType.Regular, - worksheet, fac); - if (hNode != null) - headersMap.put(i, hNode.getId()); - } - reader.close(); - break; - } - rowCount++; - } - br.reset(); - return headersMap; - } private boolean isVisible(String key) { - if (columnsJson == null) - return true; - for (int i = 0; i < columnsJson.length(); i++) { - JSONObject obj = columnsJson.getJSONObject(i); - if (obj.has(key)) - return obj.getBoolean(key); - } - return false; - } + if (columnsJson == null) + return true; + for (int i = 0; i < columnsJson.length(); i++) { + JSONObject obj = columnsJson.getJSONObject(i); + if (obj.has(key)) + return obj.getBoolean(key); + } + return false; + } } \ No newline at end of file diff --git a/karma-common/src/main/java/edu/isi/karma/imp/json/JsonImportValues.java b/karma-common/src/main/java/edu/isi/karma/imp/json/JsonImportValues.java index a34fffc2f..9e76af147 100644 --- a/karma-common/src/main/java/edu/isi/karma/imp/json/JsonImportValues.java +++ b/karma-common/src/main/java/edu/isi/karma/imp/json/JsonImportValues.java @@ -214,6 +214,11 @@ public Iterator getSortedKeysIterator(JSONObject object) { public void addListElement(Object listValue, HTable headers, Table dataTable) throws JSONException { + + if(JSONObject.NULL.equals(listValue)) { + listValue = ""; + } + if (listValue instanceof JSONObject) { if (maxNumLines <= 0 || numObjects < maxNumLines) { Row row = dataTable.addRow(factory); @@ -265,7 +270,8 @@ public void addListElement(Object listValue, HTable headers, } } } else { - logger.error("Cannot handle whatever case is not covered by the if statements. Sorry."); + logger.error("Cannot handle whatever case is not covered by the if statements. Sorry"); + logger.error(listValue.toString()); } } diff --git a/karma-common/src/main/java/edu/isi/karma/modeling/alignment/GraphVizUtil.java b/karma-common/src/main/java/edu/isi/karma/modeling/alignment/GraphVizUtil.java index a7b6296b7..7ebb442bb 100644 --- a/karma-common/src/main/java/edu/isi/karma/modeling/alignment/GraphVizUtil.java +++ b/karma-common/src/main/java/edu/isi/karma/modeling/alignment/GraphVizUtil.java @@ -161,6 +161,8 @@ private static org.kohsuke.graphviz.Graph convertToGraphviz( if (source instanceof ColumnNode) { ColumnNode mappedColumn = (mappingToSourceColumns == null) ? (ColumnNode)source : mappingToSourceColumns.get(source); sourceLabel = mappedColumn.getColumnName(); + } else if(source instanceof LiteralNode) { + sourceLabel = ((LiteralNode)source).getValue(); } else if (nodeLabelType == GraphVizLabelType.Id) sourceLabel = sourceId; else if (nodeLabelType == GraphVizLabelType.LocalId) @@ -196,6 +198,8 @@ else if (source instanceof LiteralNode) // literal if (target instanceof ColumnNode) { ColumnNode mappedColumn = (mappingToSourceColumns == null) ? (ColumnNode)target : mappingToSourceColumns.get(target); targetLabel = mappedColumn.getColumnName(); + } else if(target instanceof LiteralNode) { + targetLabel = ((LiteralNode)target).getValue(); } else if (nodeLabelType == GraphVizLabelType.Id) targetLabel = targetId; else if (nodeLabelType == GraphVizLabelType.LocalId) diff --git a/karma-common/src/main/java/edu/isi/karma/rdf/RdfGenerator.java b/karma-common/src/main/java/edu/isi/karma/rdf/RdfGenerator.java index b1af576db..855936867 100644 --- a/karma-common/src/main/java/edu/isi/karma/rdf/RdfGenerator.java +++ b/karma-common/src/main/java/edu/isi/karma/rdf/RdfGenerator.java @@ -76,6 +76,7 @@ protected void applyHistoryToWorksheet(Workspace workspace, Worksheet worksheet, try { List tags = new ArrayList<>(); + tags.add(CommandTag.Selection); tags.add(CommandTag.Transformation); List ignoreTags = new ArrayList<>(); diff --git a/karma-common/src/main/java/edu/isi/karma/rep/metadata/WorksheetProperties.java b/karma-common/src/main/java/edu/isi/karma/rep/metadata/WorksheetProperties.java index d07e9135f..fe18fe779 100644 --- a/karma-common/src/main/java/edu/isi/karma/rep/metadata/WorksheetProperties.java +++ b/karma-common/src/main/java/edu/isi/karma/rep/metadata/WorksheetProperties.java @@ -44,7 +44,8 @@ public enum Property { graphName, serviceUrl, hasServiceProperties, sourceType, modelUrl, modelRepository, modelContext, baseURI, prefix, graphLabel, inputColumns, - outputColumns, oldCommandHistory + outputColumns, oldCommandHistory, + GithubURL } public enum SourceTypes { diff --git a/karma-web/src/main/webapp/index.jsp b/karma-web/src/main/webapp/index.jsp index 692590aed..65b52a01b 100755 --- a/karma-web/src/main/webapp/index.jsp +++ b/karma-web/src/main/webapp/index.jsp @@ -183,6 +183,7 @@ and related projects, please see: http://www.isi.edu/integration
  • Settings
      +
    • Github
    • Display rdfs:label
      •  rdfs:label First
        • @@ -238,6 +239,7 @@ and related projects, please see: http://www.isi.edu/integration +