From afa454f4c7400acc129c24806becb8d9da9c6236 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Wed, 12 Jul 2023 08:50:47 -0400 Subject: [PATCH 01/11] fix: linter config --- .../wings/portal/classes/config/Config.java | 274 +++++++++--------- 1 file changed, 137 insertions(+), 137 deletions(-) diff --git a/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java b/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java index 2ed15c51..882b9662 100644 --- a/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java +++ b/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java @@ -83,10 +83,10 @@ public class Config { private boolean isSandboxed = false; private PlannerConfig plannerConfig = new PlannerConfig(); - + // Comma separated list of spellbook client hosts private String clients; - + // The following are set from the "request" variable private String viewerId; private String userId; @@ -123,22 +123,24 @@ public void setUserDomainUrl(String userDomainUrl) { this.userDomainUrl = userDomainUrl; } - public Config() {} - + public Config() { + } + public Config(HttpServletRequest request, String userid, String domain) { - // Initialize UserDatabase - this.initializeUserDatabase(); + // Initialize UserDatabase + this.initializeUserDatabase(); - // Initialize portal config - this.initializePortalConfig(request); + // Initialize portal config + this.initializePortalConfig(request); - // Initialize user config - this.initializeUserConfig(request, userid, domain); - } + // Initialize user config + this.initializeUserConfig(request, userid, domain); + } public void getPermissions() { // Check domain, user & viewerid - // Return Permissions (canRead=true/false, canWrite=true/false, canExecute=true/false) + // Return Permissions (canRead=true/false, canWrite=true/false, + // canExecute=true/false) } public boolean checkUser(HttpServletResponse response) { @@ -165,7 +167,7 @@ public boolean checkUser(HttpServletResponse response) { } public void showError(HttpServletRequest request, - HttpServletResponse response, String message) { + HttpServletResponse response, String message) { try { response.setContentType("text/html"); request.setAttribute("message", message); @@ -177,12 +179,12 @@ public void showError(HttpServletRequest request, } public boolean checkDomain(HttpServletRequest request, - HttpServletResponse response) { + HttpServletResponse response) { return this.checkDomain(request, response, true); } public boolean checkDomain(HttpServletRequest request, - HttpServletResponse response, boolean show_error) { + HttpServletResponse response, boolean show_error) { if (!this.checkUser(response)) return false; @@ -209,7 +211,7 @@ public boolean checkDomain(HttpServletRequest request, && !this.viewerId.equals(this.userId)) { // Check domain permissions // TODO: Check read, write & execute permission based on input - // For now: all or none permissions + // For now: all or none permissions Permission perm = this.domain.getPermissionForUser(this.viewerId); if (!perm.canRead()) { if (show_error) @@ -232,8 +234,8 @@ private void initializeUserConfig(HttpServletRequest request, String userid, Str this.scriptPath = request.getRequestURI(); if (this.domainId != null) - this.userDomainUrl = this.contextRootPath + "/" + this.getUsersRelativeDir() - + "/" + this.getUserId() + "/" + this.getDomainId(); + this.userDomainUrl = this.contextRootPath + "/" + this.getUsersRelativeDir() + + "/" + this.getUserId() + "/" + this.getDomainId(); this.sessionId = request.getSession().getId(); @@ -247,8 +249,8 @@ private void initializeUserConfig(HttpServletRequest request, String userid, Str if (!this.checkUser(null)) return; - this.exportUserUrl = serverUrl + contextRootPath + exportServletPath + "/" + usersRelativeDir - + "/" + userId; + this.exportUserUrl = serverUrl + contextRootPath + exportServletPath + "/" + usersRelativeDir + + "/" + userId; this.userDir = storageDirectory + File.separator + usersRelativeDir + File.separator + userId; this.userPath = contextRootPath + "/" + usersRelativeDir + "/" + userId; @@ -277,9 +279,9 @@ private void initializeUserConfig(HttpServletRequest request, String userid, Str } if (this.domain != null) { - this.domainId = this.domain.getDomainName(); - this.userDomainUrl = this.contextRootPath + "/" + this.getUsersRelativeDir() - + "/" + this.getUserId() + "/" + this.domain.getDomainName(); + this.domainId = this.domain.getDomainName(); + this.userDomainUrl = this.contextRootPath + "/" + this.getUsersRelativeDir() + + "/" + this.getUserId() + "/" + this.domain.getDomainName(); } } @@ -333,50 +335,48 @@ private void initializePortalConfig(HttpServletRequest request) { this.workflowOntologyUrl = serverConfig.getString("ontology.workflow"); this.executionOntologyUrl = serverConfig.getString("ontology.execution"); this.resourceOntologyUrl = serverConfig.getString("ontology.resource"); - - if(serverConfig.containsKey("metaworkflows")) - this.hasMetaWorkflows = serverConfig.getBoolean("metaworkflows"); - - if(serverConfig.containsKey("light-reasoner")) - plannerConfig.dataValidation = !serverConfig.getBoolean("light-reasoner"); - - if(serverConfig.containsKey("planner.data-validation")) - plannerConfig.dataValidation = serverConfig.getBoolean("planner.data-validation"); - if(serverConfig.containsKey("planner.specialization")) - plannerConfig.specialization = serverConfig.getBoolean("planner.specialization"); - if(serverConfig.containsKey("planner.use-rules")) - plannerConfig.useRules = serverConfig.getBoolean("planner.use-rules"); - - if(serverConfig.containsKey("planner.max-queue-size")) - plannerConfig.maxQueueSize = serverConfig.getInt("planner.max-queue-size", 1000); + + if (serverConfig.containsKey("metaworkflows")) + this.hasMetaWorkflows = serverConfig.getBoolean("metaworkflows"); + + if (serverConfig.containsKey("light-reasoner")) + plannerConfig.dataValidation = !serverConfig.getBoolean("light-reasoner"); + + if (serverConfig.containsKey("planner.data-validation")) + plannerConfig.dataValidation = serverConfig.getBoolean("planner.data-validation"); + if (serverConfig.containsKey("planner.specialization")) + plannerConfig.specialization = serverConfig.getBoolean("planner.specialization"); + if (serverConfig.containsKey("planner.use-rules")) + plannerConfig.useRules = serverConfig.getBoolean("planner.use-rules"); + + if (serverConfig.containsKey("planner.max-queue-size")) + plannerConfig.maxQueueSize = serverConfig.getInt("planner.max-queue-size", 1000); else - plannerConfig.maxQueueSize = 1000; - if(serverConfig.containsKey("planner.parallelism")) - plannerConfig.parallelism = serverConfig.getInt("planner.parallelism", 10); + plannerConfig.maxQueueSize = 1000; + if (serverConfig.containsKey("planner.parallelism")) + plannerConfig.parallelism = serverConfig.getInt("planner.parallelism", 10); else - plannerConfig.parallelism = 10; - - if(serverConfig.containsKey("storage.logs")) { - this.logsDirectory = serverConfig.getString("storage.logs"); - } - else { - this.logsDirectory = this.storageDirectory + File.separator + "logs"; - } - if(serverConfig.containsKey("storage.delete-run-outputs")) { - this.deleteRunOutputs = serverConfig.getBoolean("storage.delete-run-outputs"); + plannerConfig.parallelism = 10; + + if (serverConfig.containsKey("storage.logs")) { + this.logsDirectory = serverConfig.getString("storage.logs"); + } else { + this.logsDirectory = this.storageDirectory + File.separator + "logs"; } - else { - this.deleteRunOutputs = false; + if (serverConfig.containsKey("storage.delete-run-outputs")) { + this.deleteRunOutputs = serverConfig.getBoolean("storage.delete-run-outputs"); + } else { + this.deleteRunOutputs = false; } // Create logsDir (if it doesn't exist) File logdir = new File(this.logsDirectory); if (!logdir.exists() && !logdir.mkdirs()) System.err.println("Cannot create logs directory : " + logdir.getAbsolutePath()); - + this.exportCommunityUrl = serverUrl + contextRootPath + exportServletPath + "/" - + communityRelativeDir; + + communityRelativeDir; this.communityPath = contextRootPath + "/" + usersRelativeDir + "/" + communityRelativeDir; - + this.communityDir = storageDirectory + File.separator + communityRelativeDir; // Create communityDir (if it doesn't exist) @@ -405,67 +405,66 @@ private void initializePortalConfig(HttpServletRequest request) { } // Load publishing configuration - String publishUrl = serverConfig.getString("publisher.url"); - String publishExportName = serverConfig.getString("publisher.name"); - String tstorePublishUrl = serverConfig.getString("publisher.triple-store.publish"); - String tstoreQueryUrl = serverConfig.getString("publisher.triple-store.query"); - String domainsDir = serverConfig.getString("publisher.triple-store.domains-directory"); - String uploadUrl = serverConfig.getString("publisher.upload-server.url"); - String uploadUsername = serverConfig.getString("publisher.upload-server.username"); - String uploadPassword = serverConfig.getString("publisher.upload-server.password"); - String uploadDir = serverConfig.getString("publisher.upload-server.directory"); - String uploadHost = serverConfig.getString("publisher.upload-server.host"); - String uploadUserId = serverConfig.getString("publisher.upload-server.userid"); - String uploadKey = serverConfig.getString("publisher.upload-server.private-key"); - String sizeString = serverConfig.getString("publisher.upload-server.max-upload-size"); - - this.publisher = new Publisher(); - this.publisher.setUrl(publishUrl); - this.publisher.setExportName(publishExportName); - this.publisher.setDomainsDir(domainsDir); - this.publisher.setTstorePublishUrl(tstorePublishUrl); - this.publisher.setTstoreQueryUrl(tstoreQueryUrl); - - - ServerDetails upserver = new ServerDetails(); - upserver.setUrl(uploadUrl); - upserver.setUsername(uploadUsername); - upserver.setPassword(uploadPassword); - upserver.setHostUserId(uploadUserId); - upserver.setDirectory(uploadDir); - upserver.setHost(uploadHost); - upserver.setPrivateKey(uploadKey); - if(sizeString != null) { + String publishUrl = serverConfig.getString("publisher.url"); + String publishExportName = serverConfig.getString("publisher.name"); + String tstorePublishUrl = serverConfig.getString("publisher.triple-store.publish"); + String tstoreQueryUrl = serverConfig.getString("publisher.triple-store.query"); + String domainsDir = serverConfig.getString("publisher.triple-store.domains-directory"); + String uploadUrl = serverConfig.getString("publisher.upload-server.url"); + String uploadUsername = serverConfig.getString("publisher.upload-server.username"); + String uploadPassword = serverConfig.getString("publisher.upload-server.password"); + String uploadDir = serverConfig.getString("publisher.upload-server.directory"); + String uploadHost = serverConfig.getString("publisher.upload-server.host"); + String uploadUserId = serverConfig.getString("publisher.upload-server.userid"); + String uploadKey = serverConfig.getString("publisher.upload-server.private-key"); + String sizeString = serverConfig.getString("publisher.upload-server.max-upload-size"); + + this.publisher = new Publisher(); + this.publisher.setUrl(publishUrl); + this.publisher.setExportName(publishExportName); + this.publisher.setDomainsDir(domainsDir); + this.publisher.setTstorePublishUrl(tstorePublishUrl); + this.publisher.setTstoreQueryUrl(tstoreQueryUrl); + + ServerDetails upserver = new ServerDetails(); + upserver.setUrl(uploadUrl); + upserver.setUsername(uploadUsername); + upserver.setPassword(uploadPassword); + upserver.setHostUserId(uploadUserId); + upserver.setDirectory(uploadDir); + upserver.setHost(uploadHost); + upserver.setPrivateKey(uploadKey); + if (sizeString != null) { long size = this.getSizeFromString(sizeString); upserver.setMaxUploadSize(size); - } - this.publisher.setUploadServer(upserver); + } + this.publisher.setUploadServer(upserver); } - + private long getSizeFromString(String sizeString) { - long kb = 1024; - long mb = kb*kb; - long gb = kb*mb; - long tb = kb*gb; - - Pattern pat = Pattern.compile("(\\d+)\\s*([KkMmGgTt])[Bb]?"); - Matcher mat = pat.matcher(sizeString); - if(mat.find()) { - long size=Long.parseLong(mat.group(1)); - if(mat.groupCount() > 1) { - String units = mat.group(2).toLowerCase(); - if(units.equals("k")) - return size*kb; - if(units.equals("m")) - return size*mb; - if(units.equals("g")) - return size*gb; - if(units.equals("t")) - return size*tb; + long kb = 1024; + long mb = kb * kb; + long gb = kb * mb; + long tb = kb * gb; + + Pattern pat = Pattern.compile("(\\d+)\\s*([KkMmGgTt])[Bb]?"); + Matcher mat = pat.matcher(sizeString); + if (mat.find()) { + long size = Long.parseLong(mat.group(1)); + if (mat.groupCount() > 1) { + String units = mat.group(2).toLowerCase(); + if (units.equals("k")) + return size * kb; + if (units.equals("m")) + return size * mb; + if (units.equals("g")) + return size * gb; + if (units.equals("t")) + return size * tb; + } + return size; } - return size; - } - return 0; + return 0; } private void initializeUserDatabase() { @@ -478,7 +477,7 @@ private ExeEngine getExeEngine(HierarchicalConfiguration node) { String impl = node.getString("implementation"); ExeEngine.Type type = ExeEngine.Type.valueOf(node.getString("type")); ExeEngine engine = new ExeEngine(name, impl, type); - for (Iterator it = node.getKeys("properties"); it.hasNext(); ) { + for (Iterator it = node.getKeys("properties"); it.hasNext();) { String key = (String) it.next(); String value = node.getString(key); engine.addProperty(key.replace("properties.", ""), value); @@ -530,7 +529,7 @@ private void createDefaultPortalConfig(HttpServletRequest request) { else storageDir = System.getProperty("java.io.tmpdir") + File.separator + "wings" + File.separator + "storage"; - + File storageDirFile = new File(storageDir); if (!storageDirFile.exists() && !storageDirFile.mkdirs()) System.err.println("Cannot create storage directory: " + storageDir); @@ -554,11 +553,13 @@ private void createDefaultPortalConfig(HttpServletRequest request) { this.addEngineConfig(config, new ExeEngine("Distributed", DistributedExecutionEngine.class.getCanonicalName(), ExeEngine.Type.BOTH)); - /*this.addEngineConfig(config, new ExeEngine("OODT", - OODTExecutionEngine.class.getCanonicalName(), ExeEngine.Type.PLAN)); - - this.addEngineConfig(config, new ExeEngine("Pegasus", - PegasusExecutionEngine.class.getCanonicalName(), ExeEngine.Type.PLAN));*/ + /* + * this.addEngineConfig(config, new ExeEngine("OODT", + * OODTExecutionEngine.class.getCanonicalName(), ExeEngine.Type.PLAN)); + * + * this.addEngineConfig(config, new ExeEngine("Pegasus", + * PegasusExecutionEngine.class.getCanonicalName(), ExeEngine.Type.PLAN)); + */ try { config.save(this.configFile); @@ -612,15 +613,15 @@ public Properties getProperties(Domain domain) { if (this.getResourceOntologyUrl() == null) this.setResourceOntologyUrl(ontdirurl + "/resource.owl"); props.setProperty("ont.resource.url", this.getResourceOntologyUrl()); - + props.setProperty("lib.resource.url", this.getExportCommunityUrl() + "/resource/library.owl"); if (domain != null && !domain.getUseSharedTripleStore()) - props.setProperty("lib.resource.map", - "file:" + domain.getDomainDirectory() + File.separator + "ontology" - + File.separator + "resource" + File.separator + "library.owl"); - + props.setProperty("lib.resource.map", + "file:" + domain.getDomainDirectory() + File.separator + "ontology" + + File.separator + "resource" + File.separator + "library.owl"); + props.setProperty("lib.provenance.url", this.getExportCommunityUrl() + "/provenance/library.owl"); @@ -766,15 +767,15 @@ public boolean isAdminViewer() { public void setAdminViewer(boolean isAdminViewer) { this.isAdminViewer = isAdminViewer; } - + public boolean hasMetaWorkflows() { - return hasMetaWorkflows; + return hasMetaWorkflows; } - + public void setMetaWorkflows(boolean hasMetaWorkflows) { this.hasMetaWorkflows = hasMetaWorkflows; } - + public void setExportUserUrl(String exportUserUrl) { this.exportUserUrl = exportUserUrl; } @@ -856,11 +857,11 @@ public void setTripleStoreDir(String tripleStoreDir) { } public String getLogsDirectory() { - return logsDirectory; + return logsDirectory; } public void setLogsDirectory(String logsDirectory) { - this.logsDirectory = logsDirectory; + this.logsDirectory = logsDirectory; } public String getSessionId() { @@ -888,11 +889,11 @@ public void setDotFile(String dotFile) { } public boolean isDeleteRunOutputs() { - return deleteRunOutputs; + return deleteRunOutputs; } public void setDeleteRunOutputs(boolean deleteRunOutputs) { - this.deleteRunOutputs = deleteRunOutputs; + this.deleteRunOutputs = deleteRunOutputs; } public String getClients() { @@ -902,7 +903,7 @@ public String getClients() { public void setClients(String clients) { this.clients = clients; } - + public String getStorageDirectory() { return storageDirectory; } @@ -912,15 +913,14 @@ public void setStorageDirectory(String storageDirectory) { } public PlannerConfig getPlannerConfig() { - return plannerConfig; + return plannerConfig; } public void setPlannerConfig(PlannerConfig plannerConfig) { - this.plannerConfig = plannerConfig; + this.plannerConfig = plannerConfig; } public Set getEnginesList() { return this.engines.keySet(); } } - From 7101a3aacddb68326d447349b36f930338ed4e00 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Wed, 12 Jul 2023 08:54:19 -0400 Subject: [PATCH 02/11] Improve ontology configuration handling (#160) * fix: linter * fix: reorder ontology config --- .../wings/portal/classes/config/Config.java | 101 +++++------------- .../portal/classes/config/OntologyConfig.java | 73 +++++++++++++ .../isi/wings/portal/servlets/ViewConfig.java | 4 +- 3 files changed, 100 insertions(+), 78 deletions(-) create mode 100644 portal/src/main/java/edu/isi/wings/portal/classes/config/OntologyConfig.java diff --git a/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java b/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java index 882b9662..67576005 100644 --- a/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java +++ b/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java @@ -64,14 +64,9 @@ public class Config { private String logsDirectory; private String dotFile; private String serverUrl; - private String workflowOntologyUrl; - private String dataOntologyUrl; - private String componentOntologyUrl; - private String executionOntologyUrl; - private String resourceOntologyUrl; - private boolean deleteRunOutputs; + private OntologyConfig ontologyConfig; - private String ontdirurl = "http://www.wings-workflows.org/ontology"; + private boolean deleteRunOutputs; private HashMap engines; @@ -330,11 +325,8 @@ private void initializePortalConfig(HttpServletRequest request) { this.serverUrl = serverConfig.getString("server"); this.dotFile = serverConfig.getString("graphviz"); this.clients = serverConfig.getString("clients"); - this.dataOntologyUrl = serverConfig.getString("ontology.data"); - this.componentOntologyUrl = serverConfig.getString("ontology.component"); - this.workflowOntologyUrl = serverConfig.getString("ontology.workflow"); - this.executionOntologyUrl = serverConfig.getString("ontology.execution"); - this.resourceOntologyUrl = serverConfig.getString("ontology.resource"); + this.ontologyConfig = new OntologyConfig(serverConfig); + if (serverConfig.containsKey("metaworkflows")) this.hasMetaWorkflows = serverConfig.getBoolean("metaworkflows"); @@ -542,25 +534,19 @@ private void createDefaultPortalConfig(HttpServletRequest request) { File loc1 = new File("/usr/bin/dot"); File loc2 = new File("/usr/local/bin/dot"); config.addProperty("graphviz", loc2.exists() ? loc2.getAbsolutePath() : loc1.getAbsolutePath()); - config.addProperty("ontology.data", ontdirurl + "/data.owl"); - config.addProperty("ontology.component", ontdirurl + "/component.owl"); - config.addProperty("ontology.workflow", ontdirurl + "/workflow.owl"); - config.addProperty("ontology.execution", ontdirurl + "/execution.owl"); - config.addProperty("ontology.resource", ontdirurl + "/resource.owl"); + + OntologyConfig ontConfig = new OntologyConfig(); + config.addProperty("ontology.data", ontConfig.getDataOntologyUrl()); + config.addProperty("ontology.component", ontConfig.getComponentOntologyUrl()); + config.addProperty("ontology.workflow", ontConfig.getWorkflowOntologyUrl()); + config.addProperty("ontology.execution", ontConfig.getExecutionOntologyUrl()); + config.addProperty("ontology.resource", ontConfig.getResourceOntologyUrl()); this.addEngineConfig(config, new ExeEngine("Local", LocalExecutionEngine.class.getCanonicalName(), ExeEngine.Type.BOTH)); this.addEngineConfig(config, new ExeEngine("Distributed", DistributedExecutionEngine.class.getCanonicalName(), ExeEngine.Type.BOTH)); - /* - * this.addEngineConfig(config, new ExeEngine("OODT", - * OODTExecutionEngine.class.getCanonicalName(), ExeEngine.Type.PLAN)); - * - * this.addEngineConfig(config, new ExeEngine("Pegasus", - * PegasusExecutionEngine.class.getCanonicalName(), ExeEngine.Type.PLAN)); - */ - try { config.save(this.configFile); } catch (Exception e) { @@ -588,15 +574,15 @@ public Properties getProperties(Domain domain) { if (domain.isLegacy()) return props; - props.setProperty("ont.dir.url", this.ontdirurl); if (!domain.getUseSharedTripleStore()) props.setProperty("ont.dir.map", "file:" + domain.getDomainDirectory() + File.separator + "ontology"); - props.setProperty("ont.data.url", this.getDataOntologyUrl()); - props.setProperty("ont.component.url", this.getComponentOntologyUrl()); - props.setProperty("ont.workflow.url", this.getWorkflowOntologyUrl()); - props.setProperty("ont.execution.url", this.getExecutionOntologyUrl()); + props.setProperty("ont.dir.url", this.ontologyConfig.getOntologyDefaultUrl()); + props.setProperty("ont.data.url", this.ontologyConfig.getDataOntologyUrl()); + props.setProperty("ont.component.url", this.ontologyConfig.getComponentOntologyUrl()); + props.setProperty("ont.workflow.url", this.ontologyConfig.getWorkflowOntologyUrl()); + props.setProperty("ont.execution.url", this.ontologyConfig.getExecutionOntologyUrl()); if (domain.getUseSharedTripleStore()) props.setProperty("tdb.repository.dir", this.getTripleStoreDir()); @@ -609,11 +595,6 @@ public Properties getProperties(Domain domain) { } props.setProperty("logs.dir", this.getLogsDirectory()); props.setProperty("dot.path", this.getDotFile()); - - if (this.getResourceOntologyUrl() == null) - this.setResourceOntologyUrl(ontdirurl + "/resource.owl"); - props.setProperty("ont.resource.url", this.getResourceOntologyUrl()); - props.setProperty("lib.resource.url", this.getExportCommunityUrl() + "/resource/library.owl"); @@ -669,47 +650,6 @@ public ExecutionMonitorAPI getDomainExecutionMonitor() { return ExecutionToolsFactory.createMonitor(this.getProperties()); } - // Getters and Setters - public String getWorkflowOntologyUrl() { - return this.workflowOntologyUrl; - } - - public String getComponentOntologyUrl() { - return this.componentOntologyUrl; - } - - public String getDataOntologyUrl() { - return this.dataOntologyUrl; - } - - public void setWorkflowOntologyUrl(String workflowOntologyUrl) { - this.workflowOntologyUrl = workflowOntologyUrl; - } - - public void setDataOntologyUrl(String dataOntologyUrl) { - this.dataOntologyUrl = dataOntologyUrl; - } - - public void setComponentOntologyUrl(String componentOntologyUrl) { - this.componentOntologyUrl = componentOntologyUrl; - } - - public String getExecutionOntologyUrl() { - return executionOntologyUrl; - } - - public void setExecutionOntologyUrl(String executionOntologyUrl) { - this.executionOntologyUrl = executionOntologyUrl; - } - - public String getResourceOntologyUrl() { - return resourceOntologyUrl; - } - - public void setResourceOntologyUrl(String resourceOntologyUrl) { - this.resourceOntologyUrl = resourceOntologyUrl; - } - public String getConfigFile() { return configFile; } @@ -923,4 +863,13 @@ public void setPlannerConfig(PlannerConfig plannerConfig) { public Set getEnginesList() { return this.engines.keySet(); } + + public OntologyConfig getOntologyConfig() { + return ontologyConfig; + } + + public void setOntologyConfig(OntologyConfig ontologyConfig) { + this.ontologyConfig = ontologyConfig; + } + } diff --git a/portal/src/main/java/edu/isi/wings/portal/classes/config/OntologyConfig.java b/portal/src/main/java/edu/isi/wings/portal/classes/config/OntologyConfig.java new file mode 100644 index 00000000..ef3fb12c --- /dev/null +++ b/portal/src/main/java/edu/isi/wings/portal/classes/config/OntologyConfig.java @@ -0,0 +1,73 @@ +package edu.isi.wings.portal.classes.config; + +import org.apache.commons.configuration.plist.PropertyListConfiguration; + +public class OntologyConfig { + + private String ontologyDefaultUrl = "http://www.wings-workflows.org/ontology"; + private String resourceOntologyUrl = ontologyDefaultUrl + "/resource.owl"; + private String workflowOntologyUrl = ontologyDefaultUrl + "/workflow.owl"; + private String dataOntologyUrl = ontologyDefaultUrl + "/data.owl"; + private String componentOntologyUrl = ontologyDefaultUrl + "/component.owl"; + private String executionOntologyUrl = ontologyDefaultUrl + "/execution.owl"; + + public OntologyConfig(PropertyListConfiguration serverConfig) { + dataOntologyUrl = serverConfig.getString("ontology.data"); + componentOntologyUrl = serverConfig.getString("ontology.component"); + workflowOntologyUrl = serverConfig.getString("ontology.workflow"); + executionOntologyUrl = serverConfig.getString("ontology.execution"); + resourceOntologyUrl = serverConfig.getString("ontology.resource"); + } + + public OntologyConfig() { + } + + public String getWorkflowOntologyUrl() { + return workflowOntologyUrl; + } + + public String getDataOntologyUrl() { + return dataOntologyUrl; + } + + public String getComponentOntologyUrl() { + return componentOntologyUrl; + } + + public String getExecutionOntologyUrl() { + return executionOntologyUrl; + } + + public String getResourceOntologyUrl() { + return resourceOntologyUrl; + } + + public void setWorkflowOntologyUrl(String workflowOntologyUrl) { + this.workflowOntologyUrl = workflowOntologyUrl; + } + + public void setDataOntologyUrl(String dataOntologyUrl) { + this.dataOntologyUrl = dataOntologyUrl; + } + + public void setComponentOntologyUrl(String componentOntologyUrl) { + this.componentOntologyUrl = componentOntologyUrl; + } + + public void setExecutionOntologyUrl(String executionOntologyUrl) { + this.executionOntologyUrl = executionOntologyUrl; + } + + public void setResourceOntologyUrl(String resourceOntologyUrl) { + this.resourceOntologyUrl = resourceOntologyUrl; + } + + public String getOntologyDefaultUrl() { + return ontologyDefaultUrl; + } + + public void setOntologyDefaultUrl(String ontologyDefaultUrl) { + this.ontologyDefaultUrl = ontologyDefaultUrl; + } + +} diff --git a/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java b/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java index bf74f893..90e63725 100644 --- a/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java +++ b/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java @@ -36,7 +36,7 @@ */ public class ViewConfig extends HttpServlet { private static final long serialVersionUID = 1L; - + /** * @see HttpServlet#HttpServlet() */ @@ -57,7 +57,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) props.put("internal_server", config.getServerUrl()); props.put("storage", config.getStorageDirectory()); props.put("dotpath", config.getDotFile()); - props.put("ontology", config.getWorkflowOntologyUrl()); + props.put("ontology", config.getOntologyConfig().getWorkflowOntologyUrl()); props.put("planner", config.getPlannerConfig()); out.println(json.toJson(props)); out.close(); From 382d5f2fc8c37fcd755ef6cb96c0639bee1b35dd Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Wed, 12 Jul 2023 10:56:15 -0400 Subject: [PATCH 03/11] improve: configuration main config (#161) * fix: main config * fix: improve --- .../wings/portal/classes/config/Config.java | 147 +-- .../portal/classes/config/MainConfig.java | 54 + .../portal/controllers/DataController.java | 1091 ++++++++--------- .../portal/resources/TemplateResource.java | 74 +- .../wings/portal/servlets/ExportGraph.java | 64 +- .../wings/portal/servlets/SparqlEndpoint.java | 119 +- .../isi/wings/portal/servlets/ViewConfig.java | 4 +- 7 files changed, 794 insertions(+), 759 deletions(-) create mode 100644 portal/src/main/java/edu/isi/wings/portal/classes/config/MainConfig.java diff --git a/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java b/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java index 67576005..3ff4c045 100644 --- a/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java +++ b/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java @@ -52,19 +52,36 @@ import edu.isi.wings.portal.controllers.DomainController; public class Config { + private static final String USER_HOME_KEY = "user.home"; + public static final String MAIN_SERVER_URL_KEY = "server"; + public static final String MAIN_GRAPHVIZ_KEY = "graphviz"; + public static final String MAIN_CLIENTS_KEY = "clients"; + public static final String EXECUTION_ENGINE_PROPERTIES = "execution.engine.properties."; + public static final String EXECUTION_ENGINE_TYPE = "execution.engine.type"; + public static final String EXECUTION_ENGINE_IMPLEMENTATION = "execution.engine.implementation"; + public static final String EXECUTION_ENGINE_1_NAME = "execution.engine(-1).name"; + public static final String ONTOLOGY_RESOURCE_KEY = "ontology.resource"; + public static final String ONTOLOGY_EXECUTION_KEY = "ontology.execution"; + public static final String ONTOLOGY_WORKFLOW_KEY = "ontology.workflow"; + public static final String ONTOLOGY_COMPONENT_KEY = "ontology.component"; + public static final String ONTOLOGY_DATA_KEY = "ontology.data"; + public static final String STORAGE_TDB_KEY = "storage.tdb"; + public static final String STORAGE_LOCAL_KEY = "storage.local"; + // The Portal configuration properties file. Order of checking: // 1. Check "config.file" servlet context parameter // 2. Check ${user.home}/.wings/portal.properties if ${user.home} is present // 3. Check /etc/wings/portal.properties private String configFile; + // Configuration objects + private MainConfig mainConfig; + private OntologyConfig ontologyConfig; + // The following are loaded from the config file private String storageDirectory; private String tdbDirectory; private String logsDirectory; - private String dotFile; - private String serverUrl; - private OntologyConfig ontologyConfig; private boolean deleteRunOutputs; @@ -79,9 +96,6 @@ public class Config { private PlannerConfig plannerConfig = new PlannerConfig(); - // Comma separated list of spellbook client hosts - private String clients; - // The following are set from the "request" variable private String viewerId; private String userId; @@ -244,7 +258,8 @@ private void initializeUserConfig(HttpServletRequest request, String userid, Str if (!this.checkUser(null)) return; - this.exportUserUrl = serverUrl + contextRootPath + exportServletPath + "/" + usersRelativeDir + this.exportUserUrl = this.mainConfig.getServerUrl() + contextRootPath + exportServletPath + "/" + + usersRelativeDir + "/" + userId; this.userDir = storageDirectory + File.separator + usersRelativeDir + File.separator + userId; @@ -318,15 +333,12 @@ public Publisher getPublisher() { private void initializePortalConfig(HttpServletRequest request) { this.contextRootPath = request.getContextPath(); - PropertyListConfiguration serverConfig = getPortalConfiguration(request); - this.storageDirectory = serverConfig.getString("storage.local"); - this.tdbDirectory = serverConfig.getString("storage.tdb"); - this.serverUrl = serverConfig.getString("server"); - this.dotFile = serverConfig.getString("graphviz"); - this.clients = serverConfig.getString("clients"); - this.ontologyConfig = new OntologyConfig(serverConfig); + this.storageDirectory = serverConfig.getString(STORAGE_LOCAL_KEY); + this.tdbDirectory = serverConfig.getString(STORAGE_TDB_KEY); + this.mainConfig = new MainConfig(serverConfig); + this.ontologyConfig = new OntologyConfig(serverConfig); if (serverConfig.containsKey("metaworkflows")) this.hasMetaWorkflows = serverConfig.getBoolean("metaworkflows"); @@ -365,7 +377,7 @@ private void initializePortalConfig(HttpServletRequest request) { if (!logdir.exists() && !logdir.mkdirs()) System.err.println("Cannot create logs directory : " + logdir.getAbsolutePath()); - this.exportCommunityUrl = serverUrl + contextRootPath + exportServletPath + "/" + this.exportCommunityUrl = this.mainConfig.getServerUrl() + contextRootPath + exportServletPath + "/" + communityRelativeDir; this.communityPath = contextRootPath + "/" + usersRelativeDir + "/" + communityRelativeDir; @@ -479,9 +491,10 @@ private ExeEngine getExeEngine(HierarchicalConfiguration node) { public PropertyListConfiguration getPortalConfiguration(HttpServletRequest request) { ServletContext app = request.getSession().getServletContext(); + this.configFile = app.getInitParameter("config.file"); if (this.configFile == null) { - String home = System.getProperty("user.home"); + String home = System.getProperty(USER_HOME_KEY); if (home != null && !home.equals("")) this.configFile = home + File.separator + ".wings" + File.separator + "portal.properties"; @@ -512,35 +525,28 @@ public PropertyListConfiguration getPortalConfiguration(HttpServletRequest reque } private void createDefaultPortalConfig(HttpServletRequest request) { - String server = request.getScheme() + "://" + request.getServerName() + ":" - + request.getServerPort(); - String storageDir = null; - String home = System.getProperty("user.home"); - if (home != null && !home.equals("")) - storageDir = home + File.separator + ".wings" + File.separator + "storage"; - else - storageDir = System.getProperty("java.io.tmpdir") + - File.separator + "wings" + File.separator + "storage"; + PropertyListConfiguration config = new PropertyListConfiguration(); + String storageDir = createDefaultStorageDirectory(); + String defaultServer = obtainServerURL(request); - File storageDirFile = new File(storageDir); - if (!storageDirFile.exists() && !storageDirFile.mkdirs()) - System.err.println("Cannot create storage directory: " + storageDir); + OntologyConfig defaultOntologyConfig = new OntologyConfig(); + MainConfig defaultMainConfig = new MainConfig(); - PropertyListConfiguration config = new PropertyListConfiguration(); - config.addProperty("storage.local", storageDir); - config.addProperty("storage.tdb", storageDir + File.separator + "TDB"); - config.addProperty("server", server); + // Main section configuration + defaultMainConfig.setServerUrl(defaultServer); + config.addProperty(MAIN_SERVER_URL_KEY, mainConfig.getServerUrl()); + config.addProperty(MAIN_GRAPHVIZ_KEY, mainConfig.getDotFile()); - File loc1 = new File("/usr/bin/dot"); - File loc2 = new File("/usr/local/bin/dot"); - config.addProperty("graphviz", loc2.exists() ? loc2.getAbsolutePath() : loc1.getAbsolutePath()); + // Storage section configuration + config.addProperty(STORAGE_LOCAL_KEY, storageDir); + config.addProperty(STORAGE_TDB_KEY, storageDir + File.separator + "TDB"); - OntologyConfig ontConfig = new OntologyConfig(); - config.addProperty("ontology.data", ontConfig.getDataOntologyUrl()); - config.addProperty("ontology.component", ontConfig.getComponentOntologyUrl()); - config.addProperty("ontology.workflow", ontConfig.getWorkflowOntologyUrl()); - config.addProperty("ontology.execution", ontConfig.getExecutionOntologyUrl()); - config.addProperty("ontology.resource", ontConfig.getResourceOntologyUrl()); + // Ontology section configuration + config.addProperty(ONTOLOGY_DATA_KEY, defaultOntologyConfig.getDataOntologyUrl()); + config.addProperty(ONTOLOGY_COMPONENT_KEY, defaultOntologyConfig.getComponentOntologyUrl()); + config.addProperty(ONTOLOGY_WORKFLOW_KEY, defaultOntologyConfig.getWorkflowOntologyUrl()); + config.addProperty(ONTOLOGY_EXECUTION_KEY, defaultOntologyConfig.getExecutionOntologyUrl()); + config.addProperty(ONTOLOGY_RESOURCE_KEY, defaultOntologyConfig.getResourceOntologyUrl()); this.addEngineConfig(config, new ExeEngine("Local", LocalExecutionEngine.class.getCanonicalName(), ExeEngine.Type.BOTH)); @@ -555,11 +561,11 @@ private void createDefaultPortalConfig(HttpServletRequest request) { } private void addEngineConfig(PropertyListConfiguration config, ExeEngine engine) { - config.addProperty("execution.engine(-1).name", engine.getName()); - config.addProperty("execution.engine.implementation", engine.getImplementation()); - config.addProperty("execution.engine.type", engine.getType()); + config.addProperty(EXECUTION_ENGINE_1_NAME, engine.getName()); + config.addProperty(EXECUTION_ENGINE_IMPLEMENTATION, engine.getImplementation()); + config.addProperty(EXECUTION_ENGINE_TYPE, engine.getType()); for (Entry entry : engine.getProperties().entrySet()) - config.addProperty("execution.engine.properties." + entry.getKey(), entry.getValue()); + config.addProperty(EXECUTION_ENGINE_PROPERTIES + entry.getKey(), entry.getValue()); } public Properties getProperties() { @@ -594,7 +600,7 @@ public Properties getProperties(Domain domain) { props.setProperty("tdb.repository.dir", this.getTripleStoreDir()); } props.setProperty("logs.dir", this.getLogsDirectory()); - props.setProperty("dot.path", this.getDotFile()); + props.setProperty("dot.path", this.getMainConfig().getDotFile()); props.setProperty("lib.resource.url", this.getExportCommunityUrl() + "/resource/library.owl"); @@ -720,14 +726,6 @@ public void setExportUserUrl(String exportUserUrl) { this.exportUserUrl = exportUserUrl; } - public String getServerUrl() { - return serverUrl; - } - - public void setServerUrl(String serverUrl) { - this.serverUrl = serverUrl; - } - public String getCommunityPath() { return communityPath; } @@ -820,14 +818,6 @@ public void setSandboxed(boolean isSandboxed) { this.isSandboxed = isSandboxed; } - public String getDotFile() { - return dotFile; - } - - public void setDotFile(String dotFile) { - this.dotFile = dotFile; - } - public boolean isDeleteRunOutputs() { return deleteRunOutputs; } @@ -836,14 +826,6 @@ public void setDeleteRunOutputs(boolean deleteRunOutputs) { this.deleteRunOutputs = deleteRunOutputs; } - public String getClients() { - return clients; - } - - public void setClients(String clients) { - this.clients = clients; - } - public String getStorageDirectory() { return storageDirectory; } @@ -872,4 +854,29 @@ public void setOntologyConfig(OntologyConfig ontologyConfig) { this.ontologyConfig = ontologyConfig; } + public MainConfig getMainConfig() { + return mainConfig; + } + + private static String createDefaultStorageDirectory() { + String storageDir = null; + String DEFAULT_WINGS_HIDDEN_STORAGE_DIR = ".wings" + File.separator + "storage"; + String DEFAULT_WINGS_STORAGE_DIR = "wings" + File.separator + "storage"; + String home = System.getProperty(USER_HOME_KEY); + if (home != null && !home.equals("")) + storageDir = home + File.separator + DEFAULT_WINGS_HIDDEN_STORAGE_DIR; + else + storageDir = System.getProperty("java.io.tmpdir") + DEFAULT_WINGS_STORAGE_DIR; + + File storageDirFile = new File(storageDir); + if (!storageDirFile.exists() && !storageDirFile.mkdirs()) + System.err.println("Cannot create storage directory: " + storageDir); + return storageDir; + } + + private String obtainServerURL(HttpServletRequest request) { + String defaultServer = request.getScheme() + "://" + request.getServerName() + ":" + + request.getServerPort(); + return defaultServer; + } } diff --git a/portal/src/main/java/edu/isi/wings/portal/classes/config/MainConfig.java b/portal/src/main/java/edu/isi/wings/portal/classes/config/MainConfig.java new file mode 100644 index 00000000..f078f548 --- /dev/null +++ b/portal/src/main/java/edu/isi/wings/portal/classes/config/MainConfig.java @@ -0,0 +1,54 @@ +package edu.isi.wings.portal.classes.config; + +import java.io.File; + +import org.apache.commons.configuration.plist.PropertyListConfiguration; + +public class MainConfig { + + // Comma separated list of spellbook client hosts + private String clients; + private String dotFile = "/usr/bin/dot"; + private String serverUrl; + + public MainConfig() { + } + + public MainConfig(PropertyListConfiguration serverConfig) { + this.clients = serverConfig.getString(Config.MAIN_CLIENTS_KEY); + this.dotFile = serverConfig.getString(Config.MAIN_GRAPHVIZ_KEY); + this.serverUrl = serverConfig.getString(Config.MAIN_SERVER_URL_KEY); + File localDotFile = new File(this.dotFile); + File localDotAlternative1 = new File("/usr/local/bin/dot"); + File localDotAlternative2 = new File("/usr/bin/dot"); + if (!localDotFile.exists() && localDotAlternative1.exists()) { + this.dotFile = localDotAlternative1.getAbsolutePath(); + } else if (!localDotFile.exists() && localDotAlternative2.exists()) { + this.dotFile = localDotAlternative2.getAbsolutePath(); + } + } + + public String getClients() { + return clients; + } + + public void setClients(String clients) { + this.clients = clients; + } + + public String getDotFile() { + return dotFile; + } + + public void setDotFile(String dotFile) { + this.dotFile = dotFile; + } + + public String getServerUrl() { + return serverUrl; + } + + public void setServerUrl(String serverUrl) { + this.serverUrl = serverUrl; + } +} diff --git a/portal/src/main/java/edu/isi/wings/portal/controllers/DataController.java b/portal/src/main/java/edu/isi/wings/portal/controllers/DataController.java index f054286e..e9069b8d 100644 --- a/portal/src/main/java/edu/isi/wings/portal/controllers/DataController.java +++ b/portal/src/main/java/edu/isi/wings/portal/controllers/DataController.java @@ -83,8 +83,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.asynchttpclient.Dsl.basicAuthRealm; - -public class DataController { +public class DataController { public String dcns; public String domns; public String libns; @@ -94,7 +93,7 @@ public class DataController { public ComponentCreationAPI cc; public ProvenanceAPI prov; public RunController rc; - + public boolean loadExternal; public Config config; public Properties props; @@ -109,21 +108,21 @@ public DataController(Config config, boolean load_external) { this.props = config.getProperties(); dc = DataFactory.getCreationAPI(props); - if(this.loadExternal) - dc = dc.getExternalCatalog(); - + if (this.loadExternal) + dc = dc.getExternalCatalog(); + cc = ComponentFactory.getCreationAPI(props); tc = TemplateFactory.getCreationAPI(props); - + prov = ProvenanceFactory.getAPI(props); - + this.rc = new RunController(config); - this.dcns = (String) props.get("ont.data.url") + "#"; - this.domns = (String) props.get("ont.domain.data.url") + "#"; - this.libns = (String) props.get("lib.domain.data.url") + "#"; - - this.trustAllCertificates(); + this.dcns = (String) props.get("ont.data.url") + "#"; + this.domns = (String) props.get("ont.domain.data.url") + "#"; + this.libns = (String) props.get("lib.domain.data.url") + "#"; + + this.trustAllCertificates(); } /* @@ -132,19 +131,19 @@ public DataController(Config config, boolean load_external) { public String getDataJSON(String dataid) { if (this.dc == null) return "{}"; - + String location = this.dc.getDataLocation(dataid); DataItem dtype = this.dc.getDatatypeForData(dataid); - if(dtype == null) - return null; - + if (dtype == null) + return null; + ArrayList props = this.dc.getMetadataProperties(dtype.getID(), false); ArrayList propids = new ArrayList(); for (MetadataProperty prop : props) propids.add(prop.getID()); ArrayList vals = this.dc.getMetadataValues(dataid, propids); String sensor = this.dc.getTypeSensor(dtype.getID()); - + HashMap details = new HashMap(); details.put("dtype", dtype.getID()); details.put("location", location); @@ -172,106 +171,103 @@ public String getDatatypeJSON(String dtype) { public String getDataHierarchyJSON() { DataTree tree = dc.getDataHierarchy(); String dtree = null; - if(tree != null) + if (tree != null) dtree = json.toJson(tree.getRoot()); return dtree; } - - public String getSensorWorkflowsListJSON() { - ArrayList list = new ArrayList(); - for(String tid : this.tc.getTemplateList()) { - if(tid.contains("#Extractor_")) { - list.add(tid); - } - } - return json.toJson(list); - } - - public String getSensorComponentsListJSON() { - ArrayList list = new ArrayList(); - ComponentTree tree = this.cc.getComponentHierarchy(false); - ComponentTreeNode root = tree.getRoot(); - ComponentTreeNode extractorRoot = null; - for (ComponentTreeNode child : root.getChildren()) { - if (child.getCls().getID().contains("#MetadataExtraction")) { - extractorRoot = child; - break; - } - } - if(extractorRoot != null) { - for(ComponentTreeNode extractorNode : tree.flatten(extractorRoot)) { - Component c = extractorNode.getCls().getComponent(); - if(c != null && c.getType() == Component.CONCRETE) { - list.add(c.getID()); - } - } - } - return json.toJson(list); - } - + + public String getSensorWorkflowsListJSON() { + ArrayList list = new ArrayList(); + for (String tid : this.tc.getTemplateList()) { + if (tid.contains("#Extractor_")) { + list.add(tid); + } + } + return json.toJson(list); + } + + public String getSensorComponentsListJSON() { + ArrayList list = new ArrayList(); + ComponentTree tree = this.cc.getComponentHierarchy(false); + ComponentTreeNode root = tree.getRoot(); + ComponentTreeNode extractorRoot = null; + for (ComponentTreeNode child : root.getChildren()) { + if (child.getCls().getID().contains("#MetadataExtraction")) { + extractorRoot = child; + break; + } + } + if (extractorRoot != null) { + for (ComponentTreeNode extractorNode : tree.flatten(extractorRoot)) { + Component c = extractorNode.getCls().getComponent(); + if (c != null && c.getType() == Component.CONCRETE) { + list.add(c.getID()); + } + } + } + return json.toJson(list); + } + private HashMap convertNodeToUINode(DataTreeNode node) { - if(node == null) - return null; - - DataItem item = node.getItem(); - HashMap treeNode = new HashMap(); - - treeNode.put("text", item.getName()); - treeNode.put("id", item.getID()); - treeNode.put("isClass", (item.getType() == 1 ? true : false)); - treeNode.put("iconCls", (item.getType() == 1 ? "icon-folder fa fa-yellow": - "icon-file-alt fa fa-blue")); - treeNode.put("expIconCls", (item.getType() == 1 ? "icon-folder-open fa fa-yellow": - "icon-file-alt fa fa-blue")); - treeNode.put("expanded", false); - treeNode.put("leaf", (item.getType() == 1 ? false : true)); - treeNode.put("draggable", (item.getType() == 1)); - return treeNode; - } - + if (node == null) + return null; + + DataItem item = node.getItem(); + HashMap treeNode = new HashMap(); + + treeNode.put("text", item.getName()); + treeNode.put("id", item.getID()); + treeNode.put("isClass", (item.getType() == 1 ? true : false)); + treeNode.put("iconCls", (item.getType() == 1 ? "icon-folder fa fa-yellow" : "icon-file-alt fa fa-blue")); + treeNode.put("expIconCls", (item.getType() == 1 ? "icon-folder-open fa fa-yellow" : "icon-file-alt fa fa-blue")); + treeNode.put("expanded", false); + treeNode.put("leaf", (item.getType() == 1 ? false : true)); + treeNode.put("draggable", (item.getType() == 1)); + return treeNode; + } + public String getNodeDataHierarchyJSON(String nodeid) { - DataTree tree = dc.getNodeDataHierarchy(nodeid); - String dtree = null; - if(tree != null) { - HashMap treeNode = new HashMap(); - ArrayList children = tree.getRoot().getChildren(); - if (children != null && children.size() > 0) { - ArrayList> uichildren = new ArrayList>(); - for(DataTreeNode childnode : children) { - uichildren.add(this.convertNodeToUINode(childnode)); - } - treeNode.put("children", uichildren); - } - dtree = json.toJson(treeNode); - } - return dtree; + DataTree tree = dc.getNodeDataHierarchy(nodeid); + String dtree = null; + if (tree != null) { + HashMap treeNode = new HashMap(); + ArrayList children = tree.getRoot().getChildren(); + if (children != null && children.size() > 0) { + ArrayList> uichildren = new ArrayList>(); + for (DataTreeNode childnode : children) { + uichildren.add(this.convertNodeToUINode(childnode)); + } + treeNode.put("children", uichildren); + } + dtree = json.toJson(treeNode); + } + return dtree; } public String getDataListJSON() { - HashMap> typeInstances = dc.getAllDatatypeDatasets(); - return json.toJson(typeInstances); + HashMap> typeInstances = dc.getAllDatatypeDatasets(); + return json.toJson(typeInstances); } - + public String getMetricsHierarchyJSON() { DataTree tree = dc.getMetricsHierarchy(); String mtree = null; - if(tree != null) + if (tree != null) mtree = json.toJson(tree.getRoot()); return mtree; } public String publishData(String dataid) { String location = dc.getDataLocation(dataid); - if(location != null) { + if (location != null) { File datafile = new File(location); - if(config.getPublisher() != null) { + if (config.getPublisher() != null) { return this.uploadFile(config.getPublisher().getUploadServer(), datafile); - } - else { + } else { return null; } } - return null; + return null; } private String uploadFile(ServerDetails server, File datafile) { @@ -279,10 +275,10 @@ private String uploadFile(ServerDetails server, File datafile) { String username = server.getUsername(); String password = server.getPassword(); - if (username == null || password == null){ + if (username == null || password == null) { return "missing username or password " + upUrl + " " + username + " " + password; } - if(datafile.exists()) { + if (datafile.exists()) { AsyncHttpClient client = Dsl.asyncHttpClient(); InputStream inputStream; @@ -291,11 +287,10 @@ private String uploadFile(ServerDetails server, File datafile) { try { org.asynchttpclient.Response response = client.preparePost(upUrl) .setRealm(basicAuthRealm(username, password).setUsePreemptiveAuth(true)) - .addBodyPart(new - InputStreamPart( - datafile.getName(), inputStream, datafile.getName(), -1, - "application/octet-stream", UTF_8) - ).execute().get(); + .addBodyPart(new InputStreamPart( + datafile.getName(), inputStream, datafile.getName(), -1, + "application/octet-stream", UTF_8)) + .execute().get(); return response.getResponseBody(); } catch (InterruptedException e) { return null; @@ -304,16 +299,16 @@ private String uploadFile(ServerDetails server, File datafile) { } } catch (FileNotFoundException e) { - return null; + return null; } } return null; } - + public Response streamData(String dataid, ServletContext context) { String location = dc.getDataLocation(dataid); - - if(location != null) { + + if (location != null) { // Check if this is a file url File f = null; try { @@ -323,10 +318,10 @@ public Response streamData(String dataid, ServletContext context) { // Do nothing } // Else assume it's a file path - if(f == null) + if (f == null) f = new File(location); - return StorageHandler.streamFile(f.getAbsolutePath(), context); + return StorageHandler.streamFile(f.getAbsolutePath(), context); } return null; } @@ -339,141 +334,138 @@ public synchronized boolean saveDataJSON(String dataid, String propvals_json) { return false; try { - JsonParser parser = new JsonParser(); - JsonElement propvals = parser.parse(propvals_json); - - DataItem dtype = dc.getDatatypeForData(dataid); - ArrayList props = dc.getMetadataProperties(dtype.getID(), false); - - ArrayList propids = new ArrayList(); - HashMap pinfos = new HashMap(); - for (MetadataProperty prop : props) { - propids.add(prop.getID()); - pinfos.put(prop.getID(), prop); - } - - dc.start_write(); - dc.start_batch_operation(); - - dc.removeAllPropertyValues(dataid, propids); - for (JsonElement propval : propvals.getAsJsonArray()) { - JsonObject pval = propval.getAsJsonObject(); - String propid = pval.get("name").getAsString(); - String value = pval.get("value").getAsString(); - MetadataProperty pinfo = pinfos.get(propid); - if(pinfo != null) { - if (pinfo.isDatatypeProperty()) { - if (value.equals("") && !pinfo.getRange().contains("string")) - continue; - dc.addDatatypePropertyValue(dataid, propid, value, pinfo.getRange()); - } else { - dc.addObjectPropertyValue(dataid, propid, this.domns + value.toString()); - } - } - } - dc.stop_batch_operation(); - - String provlog = "Updating metadata"; - Provenance p = new Provenance(dataid); - p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); - return - dc.save() && dc.end() && - prov.addProvenance(p); - } - catch (Exception e) { - e.printStackTrace(); - dc.stop_batch_operation(); - dc.end(); - return false; + JsonParser parser = new JsonParser(); + JsonElement propvals = parser.parse(propvals_json); + + DataItem dtype = dc.getDatatypeForData(dataid); + ArrayList props = dc.getMetadataProperties(dtype.getID(), false); + + ArrayList propids = new ArrayList(); + HashMap pinfos = new HashMap(); + for (MetadataProperty prop : props) { + propids.add(prop.getID()); + pinfos.put(prop.getID(), prop); + } + + dc.start_write(); + dc.start_batch_operation(); + + dc.removeAllPropertyValues(dataid, propids); + for (JsonElement propval : propvals.getAsJsonArray()) { + JsonObject pval = propval.getAsJsonObject(); + String propid = pval.get("name").getAsString(); + String value = pval.get("value").getAsString(); + MetadataProperty pinfo = pinfos.get(propid); + if (pinfo != null) { + if (pinfo.isDatatypeProperty()) { + if (value.equals("") && !pinfo.getRange().contains("string")) + continue; + dc.addDatatypePropertyValue(dataid, propid, value, pinfo.getRange()); + } else { + dc.addObjectPropertyValue(dataid, propid, this.domns + value.toString()); + } + } + } + dc.stop_batch_operation(); + + String provlog = "Updating metadata"; + Provenance p = new Provenance(dataid); + p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); + return dc.save() && dc.end() && + prov.addProvenance(p); + } catch (Exception e) { + e.printStackTrace(); + dc.stop_batch_operation(); + dc.end(); + return false; } } - + public synchronized String registerData(String dataid, String newname, String metadata_json) { - try { - JsonParser parser = new JsonParser(); - JsonElement propvals = parser.parse(metadata_json); - JsonObject pvals = propvals.getAsJsonObject(); - if(pvals.get("type") == null) - return "Datatype not known"; - - // Choose the most specific type - DataTree tree = dc.getDatatypeHierarchy(); - String dtypeid = null; - for(JsonElement el : pvals.get("type").getAsJsonArray()) { - String tmpid = el.getAsString(); - if(dtypeid == null) - dtypeid = tmpid; - else { - DataTreeNode dnode = tree.findNode(dtypeid); - if(dnode.hasChild(tree.findNode(tmpid), false)) - dtypeid = tmpid; - } - } - - String dloc = dc.getDataLocation(dataid); - if(dloc == null) - return "Existing data not found on server"; - - String newid = this.libns + newname; - String newloc = dc.getDataLocation(newid); - if(!dc.addData(newid, dtypeid)) - return "Could not add data"; - - if(!dataid.equals(newid)) { - File origf = new File(dloc); - File newf = new File(origf.getParentFile().getAbsolutePath()+File.separator+newname); - newloc = newf.getAbsolutePath(); - if(origf.exists() && !newf.exists()) - FileUtils.copyFile(origf, newf); - } - if(newloc == null) - return "Cannot find location for new data"; - - if(!dc.setDataLocation(newid, newloc)) - return "Could not set data location"; - - ArrayList props = dc.getMetadataProperties(dtypeid, false); - HashMap pinfos = new HashMap(); - for (MetadataProperty prop : props) { - pinfos.put(prop.getName(), prop); - } - - for(Entry entry : pvals.entrySet()) { - String pname = entry.getKey(); - for(JsonElement el : entry.getValue().getAsJsonArray()) { - String value = el.getAsString(); - MetadataProperty pinfo = pinfos.get(pname); - if(pinfo != null) { - if (pinfo.isDatatypeProperty()) { - if (value.equals("") && !pinfo.getRange().contains("string")) - continue; - dc.addDatatypePropertyValue(newid, pinfo.getID(), value, pinfo.getRange()); - } else { - dc.addObjectPropertyValue(newid, pinfo.getID(), this.domns + value.toString()); - } - } - } - } - - Provenance p = new Provenance(newid); - p.addActivity(new ProvActivity(ProvActivity.CREATE, "Saving data from a run")); - - if(prov.addProvenance(p)) { - return "OK"; - } - return ""; - } - catch(Exception e) { - e.printStackTrace(); - return e.getMessage(); - } + try { + JsonParser parser = new JsonParser(); + JsonElement propvals = parser.parse(metadata_json); + JsonObject pvals = propvals.getAsJsonObject(); + if (pvals.get("type") == null) + return "Datatype not known"; + + // Choose the most specific type + DataTree tree = dc.getDatatypeHierarchy(); + String dtypeid = null; + for (JsonElement el : pvals.get("type").getAsJsonArray()) { + String tmpid = el.getAsString(); + if (dtypeid == null) + dtypeid = tmpid; + else { + DataTreeNode dnode = tree.findNode(dtypeid); + if (dnode.hasChild(tree.findNode(tmpid), false)) + dtypeid = tmpid; + } + } + + String dloc = dc.getDataLocation(dataid); + if (dloc == null) + return "Existing data not found on server"; + + String newid = this.libns + newname; + String newloc = dc.getDataLocation(newid); + if (!dc.addData(newid, dtypeid)) + return "Could not add data"; + + if (!dataid.equals(newid)) { + File origf = new File(dloc); + File newf = new File(origf.getParentFile().getAbsolutePath() + File.separator + newname); + newloc = newf.getAbsolutePath(); + if (origf.exists() && !newf.exists()) + FileUtils.copyFile(origf, newf); + } + if (newloc == null) + return "Cannot find location for new data"; + + if (!dc.setDataLocation(newid, newloc)) + return "Could not set data location"; + + ArrayList props = dc.getMetadataProperties(dtypeid, false); + HashMap pinfos = new HashMap(); + for (MetadataProperty prop : props) { + pinfos.put(prop.getName(), prop); + } + + for (Entry entry : pvals.entrySet()) { + String pname = entry.getKey(); + for (JsonElement el : entry.getValue().getAsJsonArray()) { + String value = el.getAsString(); + MetadataProperty pinfo = pinfos.get(pname); + if (pinfo != null) { + if (pinfo.isDatatypeProperty()) { + if (value.equals("") && !pinfo.getRange().contains("string")) + continue; + dc.addDatatypePropertyValue(newid, pinfo.getID(), value, pinfo.getRange()); + } else { + dc.addObjectPropertyValue(newid, pinfo.getID(), this.domns + value.toString()); + } + } + } + } + + Provenance p = new Provenance(newid); + p.addActivity(new ProvActivity(ProvActivity.CREATE, "Saving data from a run")); + + if (prov.addProvenance(p)) { + return "OK"; + } + return ""; + } catch (Exception e) { + e.printStackTrace(); + return e.getMessage(); + } } public synchronized String saveDatatypeJSON(String dtypeid, String props_json) { ArrayList errors = new ArrayList(); ArrayList warnings = new ArrayList(); - - if(dtypeid == null || props_json == null) + + if (dtypeid == null || props_json == null) return "[\"Null inputs\"]"; try { @@ -483,19 +475,19 @@ public synchronized String saveDatatypeJSON(String dtypeid, String props_json) { dc.start_write(); dc.start_batch_operation(); - + // Save datatype's name format String fmt = null; String sensor = null; if (ops.get("format") != null && !ops.get("format").isJsonNull()) { fmt = ops.get("format").getAsString(); } - // Same datatype's sensing component - if (ops.get("sensor") != null && !ops.get("sensor").isJsonNull()) { - sensor = ops.get("sensor").getAsString(); - } - this.dc.setTypeAnnotations(dtypeid, fmt, sensor); - + // Same datatype's sensing component + if (ops.get("sensor") != null && !ops.get("sensor").isJsonNull()) { + sensor = ops.get("sensor").getAsString(); + } + this.dc.setTypeAnnotations(dtypeid, fmt, sensor); + // Check all properties being added JsonObject addops = ops.get("add").getAsJsonObject(); for (Map.Entry entry : addops.entrySet()) { @@ -508,35 +500,33 @@ public synchronized String saveDatatypeJSON(String dtypeid, String props_json) { String range = prop.get("range").getAsString(); MetadataProperty eprop = dc.getMetadataProperty(propid); if (eprop != null) { - // A property with this id already exists. + // A property with this id already exists. // Add dtypeid as a domain of the property this.dc.addMetadataPropertyDomain(propid, dtypeid); - } - else { + } else { this.dc.addMetadataProperty(propid, dtypeid, range); } } - /* - dc.removeMetadataPropertyInLibrary(String propid); - dc.renameMetadataPropertyInLibrary(String oldid, String newid); - */ - + /* + * dc.removeMetadataPropertyInLibrary(String propid); + * dc.renameMetadataPropertyInLibrary(String oldid, String newid); + */ + // Check all properties being deleted JsonObject delops = ops.get("del").getAsJsonObject(); for (Map.Entry entry : delops.entrySet()) { String propid = entry.getKey(); MetadataProperty eprop = dc.getMetadataProperty(propid); - if(eprop.getDomains().size() > 1) { + if (eprop.getDomains().size() > 1) { // There are more than one datatypes using this property // just remove the property from this datatype this.dc.removeMetadataPropertyDomain(propid, dtypeid); eprop.getDomains().remove(dtypeid); warnings.add("Note that the property you deleted currently also " - + "exists for other datatypes: " + + "exists for other datatypes: " + eprop.getDomains()); - } - else { + } else { this.dc.removeMetadataProperty(propid); this.dc.removeMetadataPropertyInLibrary(propid); } @@ -555,55 +545,53 @@ public synchronized String saveDatatypeJSON(String dtypeid, String props_json) { String npropid = prop.get("pid").getAsString(); MetadataProperty eprop = this.dc.getMetadataProperty(propid); MetadataProperty enprop = this.dc.getMetadataProperty(npropid); - if(enprop != null && !propid.equals(npropid) && - !range.equals(enprop.getRange())) { - errors.add("Property "+enprop.getName() - + " already exists with a different range: " - + enprop.getRange()); - continue; + if (enprop != null && !propid.equals(npropid) && + !range.equals(enprop.getRange())) { + errors.add("Property " + enprop.getName() + + " already exists with a different range: " + + enprop.getRange()); + continue; } if (!eprop.getRange().equals(range)) { this.dc.removeMetadataProperty(propid); this.dc.addMetadataProperty(npropid, dtypeid, range); // Re-add any other domains that the property might have had - for(String domid : eprop.getDomains()) { - if(!domid.equals(dtypeid)) + for (String domid : eprop.getDomains()) { + if (!domid.equals(dtypeid)) this.dc.addMetadataPropertyDomain(npropid, domid); } } else if (!propid.equals(npropid)) { this.dc.renameMetadataProperty(propid, npropid); this.dc.renamePropertyInLibrary(propid, npropid); } - if(eprop.getDomains().size() > 1) { + if (eprop.getDomains().size() > 1) { eprop.getDomains().remove(dtypeid); warnings.add("Note that the property you modified also " - + "exists for other datatypes, and would have been modified for them as well : " + + "exists for other datatypes, and would have been modified for them as well : " + eprop.getDomains()); } } this.dc.stop_batch_operation(); - - if(errors.size() == 0) { - if(!dc.save() || !dc.end()) { - errors.add("Could not save Data catalog"); - } - } - else { - dc.end(); - } - - if(errors.size() == 0) { - String provlog = "Updating datatype properties"; - Provenance p = new Provenance(dtypeid); - p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); - if(!prov.addProvenance(p)) { - errors.add("Could not add provenance"); - } + + if (errors.size() == 0) { + if (!dc.save() || !dc.end()) { + errors.add("Could not save Data catalog"); + } + } else { + dc.end(); } - } - catch(Exception e) { - this.dc.stop_batch_operation(); - this.dc.end(); + + if (errors.size() == 0) { + String provlog = "Updating datatype properties"; + Provenance p = new Provenance(dtypeid); + p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); + if (!prov.addProvenance(p)) { + errors.add("Could not add provenance"); + } + } + } catch (Exception e) { + this.dc.stop_batch_operation(); + this.dc.end(); e.printStackTrace(); errors.add(e.getMessage()); } @@ -615,51 +603,48 @@ public synchronized String saveDatatypeJSON(String dtypeid, String props_json) { } public synchronized boolean addDatatype(String ptype, String dtype) { - if(ptype == null || dtype == null) + if (ptype == null || dtype == null) return false; - String provlog = "Creating datatype with parent "+KBUtils.getLocalName(ptype); - Provenance p = new Provenance(dtype); - p.addActivity(new ProvActivity(ProvActivity.CREATE, provlog)); - - return - dc.addDatatype(dtype, ptype) && - prov.addProvenance(p); + String provlog = "Creating datatype with parent " + KBUtils.getLocalName(ptype); + Provenance p = new Provenance(dtype); + p.addActivity(new ProvActivity(ProvActivity.CREATE, provlog)); + + return dc.addDatatype(dtype, ptype) && + prov.addProvenance(p); } public synchronized boolean moveDatatypeTo(String dtypeid, String fromtype, String totype) { - if(dtypeid == null || fromtype == null || totype == null) + if (dtypeid == null || fromtype == null || totype == null) return false; - String provlog = "Moving Datatype from " + KBUtils.getLocalName(fromtype) + - " to " + KBUtils.getLocalName(totype); - Provenance p = new Provenance(dtypeid); - p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); - - return - dc.moveDatatypeParent(dtypeid, fromtype, totype) && - dc.moveDatatypeParentInLibrary(dtypeid, fromtype, totype) && - prov.addProvenance(p); - } - - public synchronized boolean moveDataTo(String dataid, String fromtype, String totype) { - if(dataid == null || fromtype == null || totype == null) - return false; - - String provlog = "Moving Data from " + KBUtils.getLocalName(fromtype) + - " to " + KBUtils.getLocalName(totype); - Provenance p = new Provenance(dataid); - p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); - - return - dc.moveDataParent(dataid, fromtype, totype) && - prov.addProvenance(p); - } - + String provlog = "Moving Datatype from " + KBUtils.getLocalName(fromtype) + + " to " + KBUtils.getLocalName(totype); + Provenance p = new Provenance(dtypeid); + p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); + + return dc.moveDatatypeParent(dtypeid, fromtype, totype) && + dc.moveDatatypeParentInLibrary(dtypeid, fromtype, totype) && + prov.addProvenance(p); + } + + public synchronized boolean moveDataTo(String dataid, String fromtype, String totype) { + if (dataid == null || fromtype == null || totype == null) + return false; + + String provlog = "Moving Data from " + KBUtils.getLocalName(fromtype) + + " to " + KBUtils.getLocalName(totype); + Provenance p = new Provenance(dataid); + p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); + + return dc.moveDataParent(dataid, fromtype, totype) && + prov.addProvenance(p); + } + public synchronized boolean delDatatypes(String[] dtypes) { for (String dtypeid : dtypes) { - if(!dc.removeDatatype(dtypeid) || - !prov.removeAllProvenance(dtypeid)) { + if (!dc.removeDatatype(dtypeid) || + !prov.removeAllProvenance(dtypeid)) { return false; } } @@ -667,230 +652,222 @@ public synchronized boolean delDatatypes(String[] dtypes) { } public synchronized boolean addDataForDatatype(String dataid, String dtypeid) { - String provlog = "Creating data of type "+KBUtils.getLocalName(dtypeid); - Provenance p = new Provenance(dataid); - p.addActivity(new ProvActivity(ProvActivity.CREATE, provlog)); - - return - dc.addData(dataid, dtypeid) && - prov.addProvenance(p); - } - - public synchronized String addRemoteDataForType(String dataLocationUrl, String dtypeid) { - try { - String provlog = "Downloading from " + dataLocationUrl + " and creating data of type "+KBUtils.getLocalName(dtypeid); - - // Check if it doesn't already exist - URL url = new URL(dataLocationUrl); - String filename = KBUtils.sanitizeID(new File(url.getFile()).getName()); - String dataid = this.libns + filename; - //System.out.println("Check if data exists"); - if(dc.getDataLocation(dataid) != null) { - // Dataset already exists, so do not overwrite (FIXME : Make it configurable) - return dataid; - } - - Provenance p = new Provenance(dataid); - p.addActivity(new ProvActivity(ProvActivity.CREATE, provlog)); - //System.out.println("Adding new data"); - - String location = dc.getDefaultDataLocation(dataid); - //System.out.println("Copying "+ url + " to " + location); - FileUtils.copyURLToFile(url, new File(location)); - //System.out.println("Add data id: " + dataid); - dc.addData(dataid, dtypeid); - //System.out.println("Set data location of " + dataid); - if(dc.setDataLocation(dataid, location) && prov.addProvenance(p)) { - return dataid; - } - } - catch (Exception e) { - e.printStackTrace(); - } - return null; - } - - public synchronized boolean addBatchData(String dtypeid, String[] dids, String[] locations, - HttpServletRequest request, ServletContext context) { - for(int i=0; i i && locations[i] != null) - if(!dc.setDataLocation(dids[i], locations[i])) + if (locations.length > i && locations[i] != null) + if (!dc.setDataLocation(dids[i], locations[i])) return false; this.runSensorWorkflow(dids[i], request, context); } return true; } - - - public String runSensorWorkflow(String dataid, HttpServletRequest request, ServletContext context) { - try { - DataItem dtype = this.dc.getDatatypeForData(dataid); - if(dtype == null) { - System.err.println("No datatype for data id: " + dataid); - return null; - } - - String tplid = this.dc.getTypeSensor(dtype.getID()); - if(tplid == null) { - return null; - } - - Variable[] invars = this.tc.getTemplate(tplid).getInputVariables(); - if(invars.length != 1) { - System.err.println("Template should have exactly 1 input variable. Has " + invars.length); - return null; - } - HashMap> dataBindings = new HashMap>(); - ArrayList inputfiles = new ArrayList(); - inputfiles.add(dataid); - dataBindings.put(invars[0].getID(), inputfiles); - - TemplateBindings tbindings = new TemplateBindings(); - tbindings.setCallbackUrl(this.config.getServerUrl() + - this.config.getUserDomainUrl() +"/data/setMetadataFromSensorOutput?data_id=" + - URLEncoder.encode(dataid, "UTF-8")); - tbindings.setCallbackCookies(CookieHandler.httpCookiesFromServlet(request)); - tbindings.setDataBindings(dataBindings); - tbindings.setParameterBindings(new HashMap>()); - tbindings.setParameterTypes(new HashMap()); - tbindings.setComponentBindings(new HashMap()); - tbindings.setTemplateId(tplid); - - ArrayList runids = this.rc.expandAndRunTemplate(tbindings, context); - if(runids != null && runids.size() > 0) { - return runids.get(0); - } - } - catch (Exception e) { - e.printStackTrace(); - this.cc.end(); - this.rc.end(); - this.dc.end(); - } - return null; - } - - public String runSensorComponent(String dataid, HttpServletRequest request, ServletContext context) { - try { - DataItem dtype = this.dc.getDatatypeForData(dataid); - if(dtype == null) { - System.err.println("No datatype for data id: " + dataid); - return null; - } - - String cid = this.dc.getTypeSensor(dtype.getID()); - if(cid == null) { - return null; - } - Component c = this.cc.getComponent(cid, true); - ArrayList inroles = c.getInputs(); - ArrayList outroles = c.getOutputs(); - if(inroles.size() != 1) { - System.err.println("Sensor component should have exactly 1 input. Has " + inroles.size()); - return null; - } - if(outroles.size() != 1) { - System.err.println("Sensor component should have exactly 1 output. Has " + outroles.size()); - return null; - } - - HashMap role_bindings = new HashMap(); - Binding b = new Binding(dataid); - b.setLocation(this.dc.getDataLocation(dataid)); - - Binding bout = new Binding(dataid + ExecutionFile.metaExtension); - bout.setLocation(b.getLocation() + ExecutionFile.metaExtension); - - role_bindings.put(inroles.get(0).getRoleName(), b); - role_bindings.put(outroles.get(0).getRoleName(), bout); - - String callbackUrl = request.getScheme() + "://" + request.getServerName() + ":" + request.getServerPort() + - this.config.getUserDomainUrl() +"/data/setMetadataFromSensorOutput?data_id=" + - URLEncoder.encode(dataid, "UTF-8"); - - this.rc.runComponent(cid, role_bindings, callbackUrl, CookieHandler.httpCookiesFromServlet(request), context); - } - catch (Exception e) { - e.printStackTrace(); - this.rc.end(); - this.dc.end(); - this.cc.end(); - } - return null; - } - - public String setMetadataFromSensorOutput(String data_id, ExecutionPlan plan) { - //System.out.println(plan); - for(ExecutionStep step : plan.getAllExecutionSteps()) { - for(ExecutionFile file : step.getOutputFiles()) { - file.loadMetadataFromFileContents(); - this.dc.setMetadataForDataObject(data_id, file.getMetadata()); - } - } - return plan.getID(); - } + + public String runSensorWorkflow(String dataid, HttpServletRequest request, ServletContext context) { + try { + DataItem dtype = this.dc.getDatatypeForData(dataid); + if (dtype == null) { + System.err.println("No datatype for data id: " + dataid); + return null; + } + + String tplid = this.dc.getTypeSensor(dtype.getID()); + if (tplid == null) { + return null; + } + + Variable[] invars = this.tc.getTemplate(tplid).getInputVariables(); + if (invars.length != 1) { + System.err.println("Template should have exactly 1 input variable. Has " + invars.length); + return null; + } + HashMap> dataBindings = new HashMap>(); + ArrayList inputfiles = new ArrayList(); + inputfiles.add(dataid); + dataBindings.put(invars[0].getID(), inputfiles); + + TemplateBindings tbindings = new TemplateBindings(); + tbindings.setCallbackUrl(this.config.getMainConfig().getServerUrl() + + this.config.getUserDomainUrl() + "/data/setMetadataFromSensorOutput?data_id=" + + URLEncoder.encode(dataid, "UTF-8")); + tbindings.setCallbackCookies(CookieHandler.httpCookiesFromServlet(request)); + tbindings.setDataBindings(dataBindings); + tbindings.setParameterBindings(new HashMap>()); + tbindings.setParameterTypes(new HashMap()); + tbindings.setComponentBindings(new HashMap()); + tbindings.setTemplateId(tplid); + + ArrayList runids = this.rc.expandAndRunTemplate(tbindings, context); + if (runids != null && runids.size() > 0) { + return runids.get(0); + } + } catch (Exception e) { + e.printStackTrace(); + this.cc.end(); + this.rc.end(); + this.dc.end(); + } + return null; + } + + public String runSensorComponent(String dataid, HttpServletRequest request, ServletContext context) { + try { + DataItem dtype = this.dc.getDatatypeForData(dataid); + if (dtype == null) { + System.err.println("No datatype for data id: " + dataid); + return null; + } + + String cid = this.dc.getTypeSensor(dtype.getID()); + if (cid == null) { + return null; + } + Component c = this.cc.getComponent(cid, true); + ArrayList inroles = c.getInputs(); + ArrayList outroles = c.getOutputs(); + if (inroles.size() != 1) { + System.err.println("Sensor component should have exactly 1 input. Has " + inroles.size()); + return null; + } + if (outroles.size() != 1) { + System.err.println("Sensor component should have exactly 1 output. Has " + outroles.size()); + return null; + } + + HashMap role_bindings = new HashMap(); + Binding b = new Binding(dataid); + b.setLocation(this.dc.getDataLocation(dataid)); + + Binding bout = new Binding(dataid + ExecutionFile.metaExtension); + bout.setLocation(b.getLocation() + ExecutionFile.metaExtension); + + role_bindings.put(inroles.get(0).getRoleName(), b); + role_bindings.put(outroles.get(0).getRoleName(), bout); + + String callbackUrl = request.getScheme() + "://" + request.getServerName() + ":" + request.getServerPort() + + this.config.getUserDomainUrl() + "/data/setMetadataFromSensorOutput?data_id=" + + URLEncoder.encode(dataid, "UTF-8"); + + this.rc.runComponent(cid, role_bindings, callbackUrl, CookieHandler.httpCookiesFromServlet(request), context); + } catch (Exception e) { + e.printStackTrace(); + this.rc.end(); + this.dc.end(); + this.cc.end(); + } + return null; + } + + public String setMetadataFromSensorOutput(String data_id, ExecutionPlan plan) { + // System.out.println(plan); + for (ExecutionStep step : plan.getAllExecutionSteps()) { + for (ExecutionFile file : step.getOutputFiles()) { + file.loadMetadataFromFileContents(); + this.dc.setMetadataForDataObject(data_id, file.getMetadata()); + } + } + return plan.getID(); + } public synchronized boolean delData(String dataid) { - return - dc.removeData(dataid) && - prov.removeAllProvenance(dataid); + return dc.removeData(dataid) && + prov.removeAllProvenance(dataid); } - + public synchronized boolean renameData(String dataid, String newid) { - String provlog = "Renaming " + KBUtils.getLocalName(dataid) - + " to " + KBUtils.getLocalName(newid); - Provenance p = new Provenance(dataid); - p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); - - return - dc.renameData(newid, dataid) && - prov.addProvenance(p); - } - + String provlog = "Renaming " + KBUtils.getLocalName(dataid) + + " to " + KBUtils.getLocalName(newid); + Provenance p = new Provenance(dataid); + p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); + + return dc.renameData(newid, dataid) && + prov.addProvenance(p); + } + public synchronized boolean setDataLocation(String dataid, String location) { - String provlog = "Setting location"; - Provenance p = new Provenance(dataid); - p.addActivity(new ProvActivity(ProvActivity.UPLOAD, provlog)); - - return - dc.setDataLocation(dataid, location) && - prov.addProvenance(p); - } - + String provlog = "Setting location"; + Provenance p = new Provenance(dataid); + p.addActivity(new ProvActivity(ProvActivity.UPLOAD, provlog)); + + return dc.setDataLocation(dataid, location) && + prov.addProvenance(p); + } + public synchronized boolean renameDataType(String dtypeid, String newid) { - String provlog = "Renaming " + KBUtils.getLocalName(dtypeid) - + " to " + KBUtils.getLocalName(newid); - Provenance p = new Provenance(dtypeid); - p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); - - return - dc.renameDatatype(newid, dtypeid) && - dc.renameDatatypeInLibrary(newid, dtypeid) && - prov.addProvenance(p); - } - + String provlog = "Renaming " + KBUtils.getLocalName(dtypeid) + + " to " + KBUtils.getLocalName(newid); + Provenance p = new Provenance(dtypeid); + p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); + + return dc.renameDatatype(newid, dtypeid) && + dc.renameDatatypeInLibrary(newid, dtypeid) && + prov.addProvenance(p); + } + public void end() { - if(dc != null) { - dc.end(); - } - if(prov != null) { - prov.end(); - } - } - - public synchronized boolean importFromExternalCatalog(String dataid, String dtypeid, + if (dc != null) { + dc.end(); + } + if (prov != null) { + prov.end(); + } + } + + public synchronized boolean importFromExternalCatalog(String dataid, String dtypeid, String propvals_json, String location) { if (dc == null) return false; - + try { JsonParser parser = new JsonParser(); JsonElement propvals = parser.parse(propvals_json); DataItem dtype = dc.getDatatypeForData(dataid); - if(dtype == null) { + if (dtype == null) { dc.addData(dataid, dtypeid); } for (Map.Entry entry : propvals.getAsJsonObject().entrySet()) { @@ -903,49 +880,49 @@ public synchronized boolean importFromExternalCatalog(String dataid, String dtyp if (eprop == null) { // Property doesn't exist this.dc.addMetadataProperty(propid, dtypeid, range); + } else { + if (!eprop.getDomains().contains(dtypeid)) { + // Property exists for another class. Add this class as domain + this.dc.addMetadataPropertyDomain(propid, dtypeid); + } + range = eprop.getRange(); } - else { - if(!eprop.getDomains().contains(dtypeid)) { - // Property exists for another class. Add this class as domain - this.dc.addMetadataPropertyDomain(propid, dtypeid); - } - range = eprop.getRange(); - } - + this.dc.addDatatypePropertyValue(dataid, propid, value, range); } dc.setDataLocation(dataid, location); return true; - } - catch(Exception e) { + } catch (Exception e) { e.printStackTrace(); return false; } } - + private void trustAllCertificates() { - // Create a new trust manager that trust all certificates - TrustManager[] trustAllCerts = new TrustManager[]{ - new X509TrustManager() { - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return null; - } - public void checkClientTrusted( - java.security.cert.X509Certificate[] certs, String authType) { - } - public void checkServerTrusted( - java.security.cert.X509Certificate[] certs, String authType) { - } - } - }; - - // Activate the new trust manager - try { - SSLContext sc = SSLContext.getInstance("SSL"); - sc.init(null, trustAllCerts, new java.security.SecureRandom()); - HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); - } catch (Exception e) { - } + // Create a new trust manager that trust all certificates + TrustManager[] trustAllCerts = new TrustManager[] { + new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + + public void checkClientTrusted( + java.security.cert.X509Certificate[] certs, String authType) { + } + + public void checkServerTrusted( + java.security.cert.X509Certificate[] certs, String authType) { + } + } + }; + + // Activate the new trust manager + try { + SSLContext sc = SSLContext.getInstance("SSL"); + sc.init(null, trustAllCerts, new java.security.SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + } catch (Exception e) { + } } } diff --git a/portal/src/main/java/edu/isi/wings/portal/resources/TemplateResource.java b/portal/src/main/java/edu/isi/wings/portal/resources/TemplateResource.java index ebfa90b9..36ebf332 100644 --- a/portal/src/main/java/edu/isi/wings/portal/resources/TemplateResource.java +++ b/portal/src/main/java/edu/isi/wings/portal/resources/TemplateResource.java @@ -24,58 +24,59 @@ public class TemplateResource extends WingsResource { TemplateController tc; - @PathParam("edit") String edit; - @PathParam("tell") String tell; + @PathParam("edit") + String edit; + @PathParam("tell") + String tell; - HashMap options; + HashMap options; boolean editor; boolean tellme; - + @PostConstruct public void init() { super.init(); - + this.editor = ("/edit".equals(edit)); this.tellme = ("/tellme".equals(tell)); - - if(this.hasPermissions() && !this.isPage("intro")) + + if (this.hasPermissions() && !this.isPage("intro")) this.tc = new TemplateController(config); this.options = new HashMap(); @SuppressWarnings("unchecked") Enumeration pnames = request.getParameterNames(); - while(pnames.hasMoreElements()) { + while (pnames.hasMoreElements()) { String pname = pnames.nextElement(); - if(pname.matches("^hide_.*")) + if (pname.matches("^hide_.*")) this.options.put(pname, Boolean.parseBoolean(request.getParameter(pname))); } } - + @PreDestroy public void destroy() { System.out.println("Template Resource Destroyed"); - if(this.tc != null) { + if (this.tc != null) { this.tc.end(); } - } - + } + @GET @Path("intro") public void getIntroduction() { String intropage = "Template"; - if(!editor) + if (!editor) intropage += "Browser"; - else + else intropage += "Editor" + (tellme ? "Tellme" : ""); this.loadIntroduction(intropage); } - - + @GET @Produces(MediaType.TEXT_HTML) public String getHTML( @QueryParam("template_id") final String template_id) { - if(this.tc != null) { + if (this.tc != null) { request.setAttribute("controller", this.tc); request.setAttribute("options", this.options); request.setAttribute("editor", this.editor); @@ -84,17 +85,16 @@ public String getHTML( } return null; } - @GET @Path("{template_name}.owl") @Produces(MediaType.TEXT_HTML) public String getTemplateHTML( @PathParam("template_name") String template_name) { - if(this.tc != null) { - config.setScriptPath(config.getScriptPath().replaceAll("/"+template_name+".owl", "")); + if (this.tc != null) { + config.setScriptPath(config.getScriptPath().replaceAll("/" + template_name + ".owl", "")); this.options.put("hide_selector", true); - + request.setAttribute("controller", this.tc); request.setAttribute("options", this.options); request.setAttribute("editor", this.editor); @@ -104,42 +104,42 @@ public String getTemplateHTML( } return null; } - + @GET @Path("getTemplatesListJSON") @Produces(MediaType.APPLICATION_JSON) public String getTemplatesListJSON() { - if(this.tc != null) + if (this.tc != null) return this.tc.getTemplatesListJSON(); return null; } - + @GET @Path("getViewerJSON") @Produces(MediaType.APPLICATION_JSON) public String getViewerJSON( @QueryParam("template_id") String template_id) { - if(this.tc != null) + if (this.tc != null) return this.tc.getViewerJSON(template_id); return null; } - + @GET @Path("getInputsJSON") @Produces(MediaType.APPLICATION_JSON) public String getInputsJSON( @QueryParam("template_id") String template_id) { - if(this.tc != null) + if (this.tc != null) return this.tc.getInputsJSON(template_id); return null; } - + @GET @Path("getEditorJSON") @Produces(MediaType.APPLICATION_JSON) public String getEditorJSON( @QueryParam("template_id") String template_id) { - if(this.tc != null) + if (this.tc != null) return this.tc.getEditorJSON(template_id); return null; } @@ -148,8 +148,8 @@ public String getEditorJSON( @Path("layoutTemplate") @Produces(MediaType.APPLICATION_JSON) public String layoutTemplate(@JsonProperty("json") String json) { - String dotexe = config.getDotFile(); - if(this.tc != null) + String dotexe = config.getMainConfig().getDotFile(); + if (this.tc != null) try { return this.tc.layoutTemplate(json, dotexe); } catch (IOException e) { @@ -165,29 +165,29 @@ public String saveTemplateJSON( @FormParam("template_id") String template_id, @FormParam("json") String json, @FormParam("constraints_json") String constraints_json) { - if(this.tc != null) + if (this.tc != null) return this.tc.saveTemplateJSON(template_id, json, constraints_json); return null; } - + @POST @Path("newTemplate") @Produces(MediaType.TEXT_PLAIN) public String newTemplate( @FormParam("template_id") String template_id) { - if(this.tc != null) { + if (this.tc != null) { RunController.invalidateCachedAPIs(); return this.tc.newTemplate(template_id); } return null; } - + @POST @Path("deleteTemplate") @Produces(MediaType.TEXT_PLAIN) public String deleteTemplate( @FormParam("template_id") String template_id) { - if(this.tc != null && this.isOwner() && !config.isSandboxed()) { + if (this.tc != null && this.isOwner() && !config.isSandboxed()) { RunController.invalidateCachedAPIs(); return this.tc.deleteTemplate(template_id); } diff --git a/portal/src/main/java/edu/isi/wings/portal/servlets/ExportGraph.java b/portal/src/main/java/edu/isi/wings/portal/servlets/ExportGraph.java index 76700b20..80992de4 100644 --- a/portal/src/main/java/edu/isi/wings/portal/servlets/ExportGraph.java +++ b/portal/src/main/java/edu/isi/wings/portal/servlets/ExportGraph.java @@ -37,7 +37,7 @@ */ public class ExportGraph extends HttpServlet { private static final long serialVersionUID = 1L; - + /** * @see HttpServlet#HttpServlet() */ @@ -53,49 +53,47 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { PrintWriter out = response.getWriter(); Config config = new Config(request, null, null); - //if(!config.checkDomain(request, response)) - // return; - - response.addHeader("Access-Control-Allow-Origin", config.getClients()); + // if(!config.checkDomain(request, response)) + // return; + + response.addHeader("Access-Control-Allow-Origin", config.getMainConfig().getClients()); response.addHeader("Access-Control-Allow-Credentials", "true"); response.addHeader("Access-Control-Allow-Methods", "GET, POST, DELETE, PUT"); response.addHeader("Access-Control-Allow-Headers", - "X-Requested-With, Content-Type, X-HTTP-Method-Override"); - - String format = request.getParameter("format"); - String uri = config.getServerUrl() + request.getRequestURI(); + "X-Requested-With, Content-Type, X-HTTP-Method-Override"); + + String format = request.getParameter("format"); + String uri = config.getMainConfig().getServerUrl() + request.getRequestURI(); OntFactory tdbfac = new OntFactory(OntFactory.JENA, config.getTripleStoreDir()); try { - Pattern complibpat = Pattern.compile(".+\\/components\\/(.+)\\.owl"); + Pattern complibpat = Pattern.compile(".+\\/components\\/(.+)\\.owl"); KBAPI kb = tdbfac.getKB(uri, OntSpec.PLAIN); - + Matcher mat = complibpat.matcher(uri); - if(mat.find()) { - String abslib = "abstract"; - String complib = mat.group(1); - if(!complib.equals(abslib)) { - String absuri = uri.replace(complib, abslib); - tdbfac.start_write_transaction(); - kb.copyFrom(tdbfac.getKB(absuri, OntSpec.PLAIN)); - tdbfac.end_transaction(); - } + if (mat.find()) { + String abslib = "abstract"; + String complib = mat.group(1); + if (!complib.equals(abslib)) { + String absuri = uri.replace(complib, abslib); + tdbfac.start_write_transaction(); + kb.copyFrom(tdbfac.getKB(absuri, OntSpec.PLAIN)); + tdbfac.end_transaction(); + } } - - tdbfac.start_read_transaction(); - if(kb.getAllTriples().size() > 0) { + + tdbfac.start_read_transaction(); + if (kb.getAllTriples().size() > 0) { response.setContentType("application/rdf+xml"); - if(format != null) { - if(format.equals("json")) - out.println(kb.toJson()); - else if(format.equals("n3")) - out.println(kb.toN3()); - } - else - out.println(kb.toAbbrevRdf(true)); + if (format != null) { + if (format.equals("json")) + out.println(kb.toJson()); + else if (format.equals("n3")) + out.println(kb.toN3()); + } else + out.println(kb.toAbbrevRdf(true)); } tdbfac.end_transaction(); - } - catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); } } diff --git a/portal/src/main/java/edu/isi/wings/portal/servlets/SparqlEndpoint.java b/portal/src/main/java/edu/isi/wings/portal/servlets/SparqlEndpoint.java index ac5acebb..fae78b80 100644 --- a/portal/src/main/java/edu/isi/wings/portal/servlets/SparqlEndpoint.java +++ b/portal/src/main/java/edu/isi/wings/portal/servlets/SparqlEndpoint.java @@ -35,23 +35,24 @@ */ public class SparqlEndpoint extends HttpServlet { private static final long serialVersionUID = 1L; - + private PrintStream out; private SparqlAPI api; - - /** - * @see HttpServlet#HttpServlet() - */ - public SparqlEndpoint() { - super(); - } /** - * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) + * @see HttpServlet#HttpServlet() + */ + public SparqlEndpoint() { + super(); + } + + /** + * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse + * response) */ protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { this.out = new PrintStream(response.getOutputStream()); - + Config config = new Config(request, null, null); String queryString = request.getParameter("query"); String updateString = request.getParameter("update"); @@ -69,78 +70,76 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t + ""); return; } - + try { - if(queryString != null && !queryString.equals("")) + if (queryString != null && !queryString.equals("")) this.showQueryResults(queryString, request, response); - else if(updateString != null && !updateString.equals("")) + else if (updateString != null && !updateString.equals("")) this.updateDataset(updateString, request, response); - } - catch (Exception e) { + } catch (Exception e) { response.getOutputStream().print(e.getMessage()); } response.getOutputStream().flush(); } - private void showQueryResults(String queryString, HttpServletRequest request, HttpServletResponse response) + private void showQueryResults(String queryString, HttpServletRequest request, HttpServletResponse response) throws IOException { - Config config = new Config(request, null, null); - String tdbdir = config.getTripleStoreDir(); - String format = request.getParameter("format"); - - this.api = new SparqlAPI(tdbdir); - this.api.showQueryResults(queryString, format, out); + Config config = new Config(request, null, null); + String tdbdir = config.getTripleStoreDir(); + String format = request.getParameter("format"); + + this.api = new SparqlAPI(tdbdir); + this.api.showQueryResults(queryString, format, out); } - - private void updateDataset(String updateString, HttpServletRequest request, HttpServletResponse response) + + private void updateDataset(String updateString, HttpServletRequest request, HttpServletResponse response) throws IOException { Config config = new Config(request, null, null); - if(!config.checkDomain(request, response)) + if (!config.checkDomain(request, response)) return; - String tdbdir = config.getTripleStoreDir(); + String tdbdir = config.getTripleStoreDir(); this.api = new SparqlAPI(tdbdir); - - if(updateString.startsWith("__SERVER_RENAME:")) { - String newurl = updateString.substring(updateString.indexOf(":")+1); - this.updateServerURL(newurl, config); - } - else { - this.api.executeUpdateQuery(updateString, out); + + if (updateString.startsWith("__SERVER_RENAME:")) { + String newurl = updateString.substring(updateString.indexOf(":") + 1); + this.updateServerURL(newurl, config); + } else { + this.api.executeUpdateQuery(updateString, out); } } - + private void updateServerURL(String newurl, Config config) { - String cururl = config.getServerUrl(); - - // Update all graphs in the triple store - this.api.updateGraphURLs(cururl, newurl, out); - - // Update all User domains - for(String userid : config.getUsersList()) { - config.setUserId(userid); - config.setViewerId(userid); - DomainController dc = new DomainController(config); - for(String domname : dc.getDomainsList()) { - DomainInfo dominfo = dc.getDomainInfo(domname); - String url = dominfo.getUrl(); - if(url.startsWith(cururl)) { - try { - out.println("* changing "+userid+"'s domain url for "+domname); - } - catch(Exception e) { - System.out.println("* changing "+userid+"'s domain url for "+domname); - } - url = url.replace(cururl, newurl); - dc.setDomainURL(domname, url); - } - } - } + String cururl = config.getMainConfig().getServerUrl(); + + // Update all graphs in the triple store + this.api.updateGraphURLs(cururl, newurl, out); + + // Update all User domains + for (String userid : config.getUsersList()) { + config.setUserId(userid); + config.setViewerId(userid); + DomainController dc = new DomainController(config); + for (String domname : dc.getDomainsList()) { + DomainInfo dominfo = dc.getDomainInfo(domname); + String url = dominfo.getUrl(); + if (url.startsWith(cururl)) { + try { + out.println("* changing " + userid + "'s domain url for " + domname); + } catch (Exception e) { + System.out.println("* changing " + userid + "'s domain url for " + domname); + } + url = url.replace(cururl, newurl); + dc.setDomainURL(domname, url); + } + } + } } /** - * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response) + * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse + * response) */ protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { // TODO Auto-generated method stub diff --git a/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java b/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java index 90e63725..2c14eba3 100644 --- a/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java +++ b/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java @@ -54,9 +54,9 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) Config config = new Config(request, null, null); Gson json = JsonHandler.createPrettyGson(); HashMap props = new HashMap(); - props.put("internal_server", config.getServerUrl()); + props.put("internal_server", config.getMainConfig().getServerUrl()); props.put("storage", config.getStorageDirectory()); - props.put("dotpath", config.getDotFile()); + props.put("dotpath", config.getMainConfig().getDotFile()); props.put("ontology", config.getOntologyConfig().getWorkflowOntologyUrl()); props.put("planner", config.getPlannerConfig()); out.println(json.toJson(props)); From 7a74f70ed721d1d22083f867238ba6b531a07a62 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Wed, 12 Jul 2023 13:09:16 -0400 Subject: [PATCH 04/11] Revert "improve: configuration main config (#161)" This reverts commit 382d5f2fc8c37fcd755ef6cb96c0639bee1b35dd. --- .../wings/portal/classes/config/Config.java | 147 ++- .../portal/classes/config/MainConfig.java | 54 - .../portal/controllers/DataController.java | 1091 +++++++++-------- .../portal/resources/TemplateResource.java | 74 +- .../wings/portal/servlets/ExportGraph.java | 64 +- .../wings/portal/servlets/SparqlEndpoint.java | 119 +- .../isi/wings/portal/servlets/ViewConfig.java | 4 +- 7 files changed, 759 insertions(+), 794 deletions(-) delete mode 100644 portal/src/main/java/edu/isi/wings/portal/classes/config/MainConfig.java diff --git a/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java b/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java index 3ff4c045..67576005 100644 --- a/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java +++ b/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java @@ -52,36 +52,19 @@ import edu.isi.wings.portal.controllers.DomainController; public class Config { - private static final String USER_HOME_KEY = "user.home"; - public static final String MAIN_SERVER_URL_KEY = "server"; - public static final String MAIN_GRAPHVIZ_KEY = "graphviz"; - public static final String MAIN_CLIENTS_KEY = "clients"; - public static final String EXECUTION_ENGINE_PROPERTIES = "execution.engine.properties."; - public static final String EXECUTION_ENGINE_TYPE = "execution.engine.type"; - public static final String EXECUTION_ENGINE_IMPLEMENTATION = "execution.engine.implementation"; - public static final String EXECUTION_ENGINE_1_NAME = "execution.engine(-1).name"; - public static final String ONTOLOGY_RESOURCE_KEY = "ontology.resource"; - public static final String ONTOLOGY_EXECUTION_KEY = "ontology.execution"; - public static final String ONTOLOGY_WORKFLOW_KEY = "ontology.workflow"; - public static final String ONTOLOGY_COMPONENT_KEY = "ontology.component"; - public static final String ONTOLOGY_DATA_KEY = "ontology.data"; - public static final String STORAGE_TDB_KEY = "storage.tdb"; - public static final String STORAGE_LOCAL_KEY = "storage.local"; - // The Portal configuration properties file. Order of checking: // 1. Check "config.file" servlet context parameter // 2. Check ${user.home}/.wings/portal.properties if ${user.home} is present // 3. Check /etc/wings/portal.properties private String configFile; - // Configuration objects - private MainConfig mainConfig; - private OntologyConfig ontologyConfig; - // The following are loaded from the config file private String storageDirectory; private String tdbDirectory; private String logsDirectory; + private String dotFile; + private String serverUrl; + private OntologyConfig ontologyConfig; private boolean deleteRunOutputs; @@ -96,6 +79,9 @@ public class Config { private PlannerConfig plannerConfig = new PlannerConfig(); + // Comma separated list of spellbook client hosts + private String clients; + // The following are set from the "request" variable private String viewerId; private String userId; @@ -258,8 +244,7 @@ private void initializeUserConfig(HttpServletRequest request, String userid, Str if (!this.checkUser(null)) return; - this.exportUserUrl = this.mainConfig.getServerUrl() + contextRootPath + exportServletPath + "/" - + usersRelativeDir + this.exportUserUrl = serverUrl + contextRootPath + exportServletPath + "/" + usersRelativeDir + "/" + userId; this.userDir = storageDirectory + File.separator + usersRelativeDir + File.separator + userId; @@ -333,13 +318,16 @@ public Publisher getPublisher() { private void initializePortalConfig(HttpServletRequest request) { this.contextRootPath = request.getContextPath(); - PropertyListConfiguration serverConfig = getPortalConfiguration(request); - this.storageDirectory = serverConfig.getString(STORAGE_LOCAL_KEY); - this.tdbDirectory = serverConfig.getString(STORAGE_TDB_KEY); - this.mainConfig = new MainConfig(serverConfig); + PropertyListConfiguration serverConfig = getPortalConfiguration(request); + this.storageDirectory = serverConfig.getString("storage.local"); + this.tdbDirectory = serverConfig.getString("storage.tdb"); + this.serverUrl = serverConfig.getString("server"); + this.dotFile = serverConfig.getString("graphviz"); + this.clients = serverConfig.getString("clients"); this.ontologyConfig = new OntologyConfig(serverConfig); + if (serverConfig.containsKey("metaworkflows")) this.hasMetaWorkflows = serverConfig.getBoolean("metaworkflows"); @@ -377,7 +365,7 @@ private void initializePortalConfig(HttpServletRequest request) { if (!logdir.exists() && !logdir.mkdirs()) System.err.println("Cannot create logs directory : " + logdir.getAbsolutePath()); - this.exportCommunityUrl = this.mainConfig.getServerUrl() + contextRootPath + exportServletPath + "/" + this.exportCommunityUrl = serverUrl + contextRootPath + exportServletPath + "/" + communityRelativeDir; this.communityPath = contextRootPath + "/" + usersRelativeDir + "/" + communityRelativeDir; @@ -491,10 +479,9 @@ private ExeEngine getExeEngine(HierarchicalConfiguration node) { public PropertyListConfiguration getPortalConfiguration(HttpServletRequest request) { ServletContext app = request.getSession().getServletContext(); - this.configFile = app.getInitParameter("config.file"); if (this.configFile == null) { - String home = System.getProperty(USER_HOME_KEY); + String home = System.getProperty("user.home"); if (home != null && !home.equals("")) this.configFile = home + File.separator + ".wings" + File.separator + "portal.properties"; @@ -525,28 +512,35 @@ public PropertyListConfiguration getPortalConfiguration(HttpServletRequest reque } private void createDefaultPortalConfig(HttpServletRequest request) { - PropertyListConfiguration config = new PropertyListConfiguration(); - String storageDir = createDefaultStorageDirectory(); - String defaultServer = obtainServerURL(request); + String server = request.getScheme() + "://" + request.getServerName() + ":" + + request.getServerPort(); + String storageDir = null; + String home = System.getProperty("user.home"); + if (home != null && !home.equals("")) + storageDir = home + File.separator + ".wings" + File.separator + "storage"; + else + storageDir = System.getProperty("java.io.tmpdir") + + File.separator + "wings" + File.separator + "storage"; - OntologyConfig defaultOntologyConfig = new OntologyConfig(); - MainConfig defaultMainConfig = new MainConfig(); + File storageDirFile = new File(storageDir); + if (!storageDirFile.exists() && !storageDirFile.mkdirs()) + System.err.println("Cannot create storage directory: " + storageDir); - // Main section configuration - defaultMainConfig.setServerUrl(defaultServer); - config.addProperty(MAIN_SERVER_URL_KEY, mainConfig.getServerUrl()); - config.addProperty(MAIN_GRAPHVIZ_KEY, mainConfig.getDotFile()); + PropertyListConfiguration config = new PropertyListConfiguration(); + config.addProperty("storage.local", storageDir); + config.addProperty("storage.tdb", storageDir + File.separator + "TDB"); + config.addProperty("server", server); - // Storage section configuration - config.addProperty(STORAGE_LOCAL_KEY, storageDir); - config.addProperty(STORAGE_TDB_KEY, storageDir + File.separator + "TDB"); + File loc1 = new File("/usr/bin/dot"); + File loc2 = new File("/usr/local/bin/dot"); + config.addProperty("graphviz", loc2.exists() ? loc2.getAbsolutePath() : loc1.getAbsolutePath()); - // Ontology section configuration - config.addProperty(ONTOLOGY_DATA_KEY, defaultOntologyConfig.getDataOntologyUrl()); - config.addProperty(ONTOLOGY_COMPONENT_KEY, defaultOntologyConfig.getComponentOntologyUrl()); - config.addProperty(ONTOLOGY_WORKFLOW_KEY, defaultOntologyConfig.getWorkflowOntologyUrl()); - config.addProperty(ONTOLOGY_EXECUTION_KEY, defaultOntologyConfig.getExecutionOntologyUrl()); - config.addProperty(ONTOLOGY_RESOURCE_KEY, defaultOntologyConfig.getResourceOntologyUrl()); + OntologyConfig ontConfig = new OntologyConfig(); + config.addProperty("ontology.data", ontConfig.getDataOntologyUrl()); + config.addProperty("ontology.component", ontConfig.getComponentOntologyUrl()); + config.addProperty("ontology.workflow", ontConfig.getWorkflowOntologyUrl()); + config.addProperty("ontology.execution", ontConfig.getExecutionOntologyUrl()); + config.addProperty("ontology.resource", ontConfig.getResourceOntologyUrl()); this.addEngineConfig(config, new ExeEngine("Local", LocalExecutionEngine.class.getCanonicalName(), ExeEngine.Type.BOTH)); @@ -561,11 +555,11 @@ private void createDefaultPortalConfig(HttpServletRequest request) { } private void addEngineConfig(PropertyListConfiguration config, ExeEngine engine) { - config.addProperty(EXECUTION_ENGINE_1_NAME, engine.getName()); - config.addProperty(EXECUTION_ENGINE_IMPLEMENTATION, engine.getImplementation()); - config.addProperty(EXECUTION_ENGINE_TYPE, engine.getType()); + config.addProperty("execution.engine(-1).name", engine.getName()); + config.addProperty("execution.engine.implementation", engine.getImplementation()); + config.addProperty("execution.engine.type", engine.getType()); for (Entry entry : engine.getProperties().entrySet()) - config.addProperty(EXECUTION_ENGINE_PROPERTIES + entry.getKey(), entry.getValue()); + config.addProperty("execution.engine.properties." + entry.getKey(), entry.getValue()); } public Properties getProperties() { @@ -600,7 +594,7 @@ public Properties getProperties(Domain domain) { props.setProperty("tdb.repository.dir", this.getTripleStoreDir()); } props.setProperty("logs.dir", this.getLogsDirectory()); - props.setProperty("dot.path", this.getMainConfig().getDotFile()); + props.setProperty("dot.path", this.getDotFile()); props.setProperty("lib.resource.url", this.getExportCommunityUrl() + "/resource/library.owl"); @@ -726,6 +720,14 @@ public void setExportUserUrl(String exportUserUrl) { this.exportUserUrl = exportUserUrl; } + public String getServerUrl() { + return serverUrl; + } + + public void setServerUrl(String serverUrl) { + this.serverUrl = serverUrl; + } + public String getCommunityPath() { return communityPath; } @@ -818,6 +820,14 @@ public void setSandboxed(boolean isSandboxed) { this.isSandboxed = isSandboxed; } + public String getDotFile() { + return dotFile; + } + + public void setDotFile(String dotFile) { + this.dotFile = dotFile; + } + public boolean isDeleteRunOutputs() { return deleteRunOutputs; } @@ -826,6 +836,14 @@ public void setDeleteRunOutputs(boolean deleteRunOutputs) { this.deleteRunOutputs = deleteRunOutputs; } + public String getClients() { + return clients; + } + + public void setClients(String clients) { + this.clients = clients; + } + public String getStorageDirectory() { return storageDirectory; } @@ -854,29 +872,4 @@ public void setOntologyConfig(OntologyConfig ontologyConfig) { this.ontologyConfig = ontologyConfig; } - public MainConfig getMainConfig() { - return mainConfig; - } - - private static String createDefaultStorageDirectory() { - String storageDir = null; - String DEFAULT_WINGS_HIDDEN_STORAGE_DIR = ".wings" + File.separator + "storage"; - String DEFAULT_WINGS_STORAGE_DIR = "wings" + File.separator + "storage"; - String home = System.getProperty(USER_HOME_KEY); - if (home != null && !home.equals("")) - storageDir = home + File.separator + DEFAULT_WINGS_HIDDEN_STORAGE_DIR; - else - storageDir = System.getProperty("java.io.tmpdir") + DEFAULT_WINGS_STORAGE_DIR; - - File storageDirFile = new File(storageDir); - if (!storageDirFile.exists() && !storageDirFile.mkdirs()) - System.err.println("Cannot create storage directory: " + storageDir); - return storageDir; - } - - private String obtainServerURL(HttpServletRequest request) { - String defaultServer = request.getScheme() + "://" + request.getServerName() + ":" - + request.getServerPort(); - return defaultServer; - } } diff --git a/portal/src/main/java/edu/isi/wings/portal/classes/config/MainConfig.java b/portal/src/main/java/edu/isi/wings/portal/classes/config/MainConfig.java deleted file mode 100644 index f078f548..00000000 --- a/portal/src/main/java/edu/isi/wings/portal/classes/config/MainConfig.java +++ /dev/null @@ -1,54 +0,0 @@ -package edu.isi.wings.portal.classes.config; - -import java.io.File; - -import org.apache.commons.configuration.plist.PropertyListConfiguration; - -public class MainConfig { - - // Comma separated list of spellbook client hosts - private String clients; - private String dotFile = "/usr/bin/dot"; - private String serverUrl; - - public MainConfig() { - } - - public MainConfig(PropertyListConfiguration serverConfig) { - this.clients = serverConfig.getString(Config.MAIN_CLIENTS_KEY); - this.dotFile = serverConfig.getString(Config.MAIN_GRAPHVIZ_KEY); - this.serverUrl = serverConfig.getString(Config.MAIN_SERVER_URL_KEY); - File localDotFile = new File(this.dotFile); - File localDotAlternative1 = new File("/usr/local/bin/dot"); - File localDotAlternative2 = new File("/usr/bin/dot"); - if (!localDotFile.exists() && localDotAlternative1.exists()) { - this.dotFile = localDotAlternative1.getAbsolutePath(); - } else if (!localDotFile.exists() && localDotAlternative2.exists()) { - this.dotFile = localDotAlternative2.getAbsolutePath(); - } - } - - public String getClients() { - return clients; - } - - public void setClients(String clients) { - this.clients = clients; - } - - public String getDotFile() { - return dotFile; - } - - public void setDotFile(String dotFile) { - this.dotFile = dotFile; - } - - public String getServerUrl() { - return serverUrl; - } - - public void setServerUrl(String serverUrl) { - this.serverUrl = serverUrl; - } -} diff --git a/portal/src/main/java/edu/isi/wings/portal/controllers/DataController.java b/portal/src/main/java/edu/isi/wings/portal/controllers/DataController.java index e9069b8d..f054286e 100644 --- a/portal/src/main/java/edu/isi/wings/portal/controllers/DataController.java +++ b/portal/src/main/java/edu/isi/wings/portal/controllers/DataController.java @@ -83,7 +83,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.asynchttpclient.Dsl.basicAuthRealm; -public class DataController { + +public class DataController { public String dcns; public String domns; public String libns; @@ -93,7 +94,7 @@ public class DataController { public ComponentCreationAPI cc; public ProvenanceAPI prov; public RunController rc; - + public boolean loadExternal; public Config config; public Properties props; @@ -108,21 +109,21 @@ public DataController(Config config, boolean load_external) { this.props = config.getProperties(); dc = DataFactory.getCreationAPI(props); - if (this.loadExternal) - dc = dc.getExternalCatalog(); - + if(this.loadExternal) + dc = dc.getExternalCatalog(); + cc = ComponentFactory.getCreationAPI(props); tc = TemplateFactory.getCreationAPI(props); - + prov = ProvenanceFactory.getAPI(props); - + this.rc = new RunController(config); - this.dcns = (String) props.get("ont.data.url") + "#"; - this.domns = (String) props.get("ont.domain.data.url") + "#"; - this.libns = (String) props.get("lib.domain.data.url") + "#"; - - this.trustAllCertificates(); + this.dcns = (String) props.get("ont.data.url") + "#"; + this.domns = (String) props.get("ont.domain.data.url") + "#"; + this.libns = (String) props.get("lib.domain.data.url") + "#"; + + this.trustAllCertificates(); } /* @@ -131,19 +132,19 @@ public DataController(Config config, boolean load_external) { public String getDataJSON(String dataid) { if (this.dc == null) return "{}"; - + String location = this.dc.getDataLocation(dataid); DataItem dtype = this.dc.getDatatypeForData(dataid); - if (dtype == null) - return null; - + if(dtype == null) + return null; + ArrayList props = this.dc.getMetadataProperties(dtype.getID(), false); ArrayList propids = new ArrayList(); for (MetadataProperty prop : props) propids.add(prop.getID()); ArrayList vals = this.dc.getMetadataValues(dataid, propids); String sensor = this.dc.getTypeSensor(dtype.getID()); - + HashMap details = new HashMap(); details.put("dtype", dtype.getID()); details.put("location", location); @@ -171,103 +172,106 @@ public String getDatatypeJSON(String dtype) { public String getDataHierarchyJSON() { DataTree tree = dc.getDataHierarchy(); String dtree = null; - if (tree != null) + if(tree != null) dtree = json.toJson(tree.getRoot()); return dtree; } - - public String getSensorWorkflowsListJSON() { - ArrayList list = new ArrayList(); - for (String tid : this.tc.getTemplateList()) { - if (tid.contains("#Extractor_")) { - list.add(tid); - } - } - return json.toJson(list); - } - - public String getSensorComponentsListJSON() { - ArrayList list = new ArrayList(); - ComponentTree tree = this.cc.getComponentHierarchy(false); - ComponentTreeNode root = tree.getRoot(); - ComponentTreeNode extractorRoot = null; - for (ComponentTreeNode child : root.getChildren()) { - if (child.getCls().getID().contains("#MetadataExtraction")) { - extractorRoot = child; - break; - } - } - if (extractorRoot != null) { - for (ComponentTreeNode extractorNode : tree.flatten(extractorRoot)) { - Component c = extractorNode.getCls().getComponent(); - if (c != null && c.getType() == Component.CONCRETE) { - list.add(c.getID()); - } - } - } - return json.toJson(list); - } - + + public String getSensorWorkflowsListJSON() { + ArrayList list = new ArrayList(); + for(String tid : this.tc.getTemplateList()) { + if(tid.contains("#Extractor_")) { + list.add(tid); + } + } + return json.toJson(list); + } + + public String getSensorComponentsListJSON() { + ArrayList list = new ArrayList(); + ComponentTree tree = this.cc.getComponentHierarchy(false); + ComponentTreeNode root = tree.getRoot(); + ComponentTreeNode extractorRoot = null; + for (ComponentTreeNode child : root.getChildren()) { + if (child.getCls().getID().contains("#MetadataExtraction")) { + extractorRoot = child; + break; + } + } + if(extractorRoot != null) { + for(ComponentTreeNode extractorNode : tree.flatten(extractorRoot)) { + Component c = extractorNode.getCls().getComponent(); + if(c != null && c.getType() == Component.CONCRETE) { + list.add(c.getID()); + } + } + } + return json.toJson(list); + } + private HashMap convertNodeToUINode(DataTreeNode node) { - if (node == null) - return null; - - DataItem item = node.getItem(); - HashMap treeNode = new HashMap(); - - treeNode.put("text", item.getName()); - treeNode.put("id", item.getID()); - treeNode.put("isClass", (item.getType() == 1 ? true : false)); - treeNode.put("iconCls", (item.getType() == 1 ? "icon-folder fa fa-yellow" : "icon-file-alt fa fa-blue")); - treeNode.put("expIconCls", (item.getType() == 1 ? "icon-folder-open fa fa-yellow" : "icon-file-alt fa fa-blue")); - treeNode.put("expanded", false); - treeNode.put("leaf", (item.getType() == 1 ? false : true)); - treeNode.put("draggable", (item.getType() == 1)); - return treeNode; - } - + if(node == null) + return null; + + DataItem item = node.getItem(); + HashMap treeNode = new HashMap(); + + treeNode.put("text", item.getName()); + treeNode.put("id", item.getID()); + treeNode.put("isClass", (item.getType() == 1 ? true : false)); + treeNode.put("iconCls", (item.getType() == 1 ? "icon-folder fa fa-yellow": + "icon-file-alt fa fa-blue")); + treeNode.put("expIconCls", (item.getType() == 1 ? "icon-folder-open fa fa-yellow": + "icon-file-alt fa fa-blue")); + treeNode.put("expanded", false); + treeNode.put("leaf", (item.getType() == 1 ? false : true)); + treeNode.put("draggable", (item.getType() == 1)); + return treeNode; + } + public String getNodeDataHierarchyJSON(String nodeid) { - DataTree tree = dc.getNodeDataHierarchy(nodeid); - String dtree = null; - if (tree != null) { - HashMap treeNode = new HashMap(); - ArrayList children = tree.getRoot().getChildren(); - if (children != null && children.size() > 0) { - ArrayList> uichildren = new ArrayList>(); - for (DataTreeNode childnode : children) { - uichildren.add(this.convertNodeToUINode(childnode)); - } - treeNode.put("children", uichildren); - } - dtree = json.toJson(treeNode); - } - return dtree; + DataTree tree = dc.getNodeDataHierarchy(nodeid); + String dtree = null; + if(tree != null) { + HashMap treeNode = new HashMap(); + ArrayList children = tree.getRoot().getChildren(); + if (children != null && children.size() > 0) { + ArrayList> uichildren = new ArrayList>(); + for(DataTreeNode childnode : children) { + uichildren.add(this.convertNodeToUINode(childnode)); + } + treeNode.put("children", uichildren); + } + dtree = json.toJson(treeNode); + } + return dtree; } public String getDataListJSON() { - HashMap> typeInstances = dc.getAllDatatypeDatasets(); - return json.toJson(typeInstances); + HashMap> typeInstances = dc.getAllDatatypeDatasets(); + return json.toJson(typeInstances); } - + public String getMetricsHierarchyJSON() { DataTree tree = dc.getMetricsHierarchy(); String mtree = null; - if (tree != null) + if(tree != null) mtree = json.toJson(tree.getRoot()); return mtree; } public String publishData(String dataid) { String location = dc.getDataLocation(dataid); - if (location != null) { + if(location != null) { File datafile = new File(location); - if (config.getPublisher() != null) { + if(config.getPublisher() != null) { return this.uploadFile(config.getPublisher().getUploadServer(), datafile); - } else { + } + else { return null; } } - return null; + return null; } private String uploadFile(ServerDetails server, File datafile) { @@ -275,10 +279,10 @@ private String uploadFile(ServerDetails server, File datafile) { String username = server.getUsername(); String password = server.getPassword(); - if (username == null || password == null) { + if (username == null || password == null){ return "missing username or password " + upUrl + " " + username + " " + password; } - if (datafile.exists()) { + if(datafile.exists()) { AsyncHttpClient client = Dsl.asyncHttpClient(); InputStream inputStream; @@ -287,10 +291,11 @@ private String uploadFile(ServerDetails server, File datafile) { try { org.asynchttpclient.Response response = client.preparePost(upUrl) .setRealm(basicAuthRealm(username, password).setUsePreemptiveAuth(true)) - .addBodyPart(new InputStreamPart( - datafile.getName(), inputStream, datafile.getName(), -1, - "application/octet-stream", UTF_8)) - .execute().get(); + .addBodyPart(new + InputStreamPart( + datafile.getName(), inputStream, datafile.getName(), -1, + "application/octet-stream", UTF_8) + ).execute().get(); return response.getResponseBody(); } catch (InterruptedException e) { return null; @@ -299,16 +304,16 @@ private String uploadFile(ServerDetails server, File datafile) { } } catch (FileNotFoundException e) { - return null; + return null; } } return null; } - + public Response streamData(String dataid, ServletContext context) { String location = dc.getDataLocation(dataid); - - if (location != null) { + + if(location != null) { // Check if this is a file url File f = null; try { @@ -318,10 +323,10 @@ public Response streamData(String dataid, ServletContext context) { // Do nothing } // Else assume it's a file path - if (f == null) + if(f == null) f = new File(location); - return StorageHandler.streamFile(f.getAbsolutePath(), context); + return StorageHandler.streamFile(f.getAbsolutePath(), context); } return null; } @@ -334,138 +339,141 @@ public synchronized boolean saveDataJSON(String dataid, String propvals_json) { return false; try { - JsonParser parser = new JsonParser(); - JsonElement propvals = parser.parse(propvals_json); - - DataItem dtype = dc.getDatatypeForData(dataid); - ArrayList props = dc.getMetadataProperties(dtype.getID(), false); - - ArrayList propids = new ArrayList(); - HashMap pinfos = new HashMap(); - for (MetadataProperty prop : props) { - propids.add(prop.getID()); - pinfos.put(prop.getID(), prop); - } - - dc.start_write(); - dc.start_batch_operation(); - - dc.removeAllPropertyValues(dataid, propids); - for (JsonElement propval : propvals.getAsJsonArray()) { - JsonObject pval = propval.getAsJsonObject(); - String propid = pval.get("name").getAsString(); - String value = pval.get("value").getAsString(); - MetadataProperty pinfo = pinfos.get(propid); - if (pinfo != null) { - if (pinfo.isDatatypeProperty()) { - if (value.equals("") && !pinfo.getRange().contains("string")) - continue; - dc.addDatatypePropertyValue(dataid, propid, value, pinfo.getRange()); - } else { - dc.addObjectPropertyValue(dataid, propid, this.domns + value.toString()); - } - } - } - dc.stop_batch_operation(); - - String provlog = "Updating metadata"; - Provenance p = new Provenance(dataid); - p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); - return dc.save() && dc.end() && - prov.addProvenance(p); - } catch (Exception e) { - e.printStackTrace(); - dc.stop_batch_operation(); - dc.end(); - return false; + JsonParser parser = new JsonParser(); + JsonElement propvals = parser.parse(propvals_json); + + DataItem dtype = dc.getDatatypeForData(dataid); + ArrayList props = dc.getMetadataProperties(dtype.getID(), false); + + ArrayList propids = new ArrayList(); + HashMap pinfos = new HashMap(); + for (MetadataProperty prop : props) { + propids.add(prop.getID()); + pinfos.put(prop.getID(), prop); + } + + dc.start_write(); + dc.start_batch_operation(); + + dc.removeAllPropertyValues(dataid, propids); + for (JsonElement propval : propvals.getAsJsonArray()) { + JsonObject pval = propval.getAsJsonObject(); + String propid = pval.get("name").getAsString(); + String value = pval.get("value").getAsString(); + MetadataProperty pinfo = pinfos.get(propid); + if(pinfo != null) { + if (pinfo.isDatatypeProperty()) { + if (value.equals("") && !pinfo.getRange().contains("string")) + continue; + dc.addDatatypePropertyValue(dataid, propid, value, pinfo.getRange()); + } else { + dc.addObjectPropertyValue(dataid, propid, this.domns + value.toString()); + } + } + } + dc.stop_batch_operation(); + + String provlog = "Updating metadata"; + Provenance p = new Provenance(dataid); + p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); + return + dc.save() && dc.end() && + prov.addProvenance(p); + } + catch (Exception e) { + e.printStackTrace(); + dc.stop_batch_operation(); + dc.end(); + return false; } } - + public synchronized String registerData(String dataid, String newname, String metadata_json) { - try { - JsonParser parser = new JsonParser(); - JsonElement propvals = parser.parse(metadata_json); - JsonObject pvals = propvals.getAsJsonObject(); - if (pvals.get("type") == null) - return "Datatype not known"; - - // Choose the most specific type - DataTree tree = dc.getDatatypeHierarchy(); - String dtypeid = null; - for (JsonElement el : pvals.get("type").getAsJsonArray()) { - String tmpid = el.getAsString(); - if (dtypeid == null) - dtypeid = tmpid; - else { - DataTreeNode dnode = tree.findNode(dtypeid); - if (dnode.hasChild(tree.findNode(tmpid), false)) - dtypeid = tmpid; - } - } - - String dloc = dc.getDataLocation(dataid); - if (dloc == null) - return "Existing data not found on server"; - - String newid = this.libns + newname; - String newloc = dc.getDataLocation(newid); - if (!dc.addData(newid, dtypeid)) - return "Could not add data"; - - if (!dataid.equals(newid)) { - File origf = new File(dloc); - File newf = new File(origf.getParentFile().getAbsolutePath() + File.separator + newname); - newloc = newf.getAbsolutePath(); - if (origf.exists() && !newf.exists()) - FileUtils.copyFile(origf, newf); - } - if (newloc == null) - return "Cannot find location for new data"; - - if (!dc.setDataLocation(newid, newloc)) - return "Could not set data location"; - - ArrayList props = dc.getMetadataProperties(dtypeid, false); - HashMap pinfos = new HashMap(); - for (MetadataProperty prop : props) { - pinfos.put(prop.getName(), prop); - } - - for (Entry entry : pvals.entrySet()) { - String pname = entry.getKey(); - for (JsonElement el : entry.getValue().getAsJsonArray()) { - String value = el.getAsString(); - MetadataProperty pinfo = pinfos.get(pname); - if (pinfo != null) { - if (pinfo.isDatatypeProperty()) { - if (value.equals("") && !pinfo.getRange().contains("string")) - continue; - dc.addDatatypePropertyValue(newid, pinfo.getID(), value, pinfo.getRange()); - } else { - dc.addObjectPropertyValue(newid, pinfo.getID(), this.domns + value.toString()); - } - } - } - } - - Provenance p = new Provenance(newid); - p.addActivity(new ProvActivity(ProvActivity.CREATE, "Saving data from a run")); - - if (prov.addProvenance(p)) { - return "OK"; - } - return ""; - } catch (Exception e) { - e.printStackTrace(); - return e.getMessage(); - } + try { + JsonParser parser = new JsonParser(); + JsonElement propvals = parser.parse(metadata_json); + JsonObject pvals = propvals.getAsJsonObject(); + if(pvals.get("type") == null) + return "Datatype not known"; + + // Choose the most specific type + DataTree tree = dc.getDatatypeHierarchy(); + String dtypeid = null; + for(JsonElement el : pvals.get("type").getAsJsonArray()) { + String tmpid = el.getAsString(); + if(dtypeid == null) + dtypeid = tmpid; + else { + DataTreeNode dnode = tree.findNode(dtypeid); + if(dnode.hasChild(tree.findNode(tmpid), false)) + dtypeid = tmpid; + } + } + + String dloc = dc.getDataLocation(dataid); + if(dloc == null) + return "Existing data not found on server"; + + String newid = this.libns + newname; + String newloc = dc.getDataLocation(newid); + if(!dc.addData(newid, dtypeid)) + return "Could not add data"; + + if(!dataid.equals(newid)) { + File origf = new File(dloc); + File newf = new File(origf.getParentFile().getAbsolutePath()+File.separator+newname); + newloc = newf.getAbsolutePath(); + if(origf.exists() && !newf.exists()) + FileUtils.copyFile(origf, newf); + } + if(newloc == null) + return "Cannot find location for new data"; + + if(!dc.setDataLocation(newid, newloc)) + return "Could not set data location"; + + ArrayList props = dc.getMetadataProperties(dtypeid, false); + HashMap pinfos = new HashMap(); + for (MetadataProperty prop : props) { + pinfos.put(prop.getName(), prop); + } + + for(Entry entry : pvals.entrySet()) { + String pname = entry.getKey(); + for(JsonElement el : entry.getValue().getAsJsonArray()) { + String value = el.getAsString(); + MetadataProperty pinfo = pinfos.get(pname); + if(pinfo != null) { + if (pinfo.isDatatypeProperty()) { + if (value.equals("") && !pinfo.getRange().contains("string")) + continue; + dc.addDatatypePropertyValue(newid, pinfo.getID(), value, pinfo.getRange()); + } else { + dc.addObjectPropertyValue(newid, pinfo.getID(), this.domns + value.toString()); + } + } + } + } + + Provenance p = new Provenance(newid); + p.addActivity(new ProvActivity(ProvActivity.CREATE, "Saving data from a run")); + + if(prov.addProvenance(p)) { + return "OK"; + } + return ""; + } + catch(Exception e) { + e.printStackTrace(); + return e.getMessage(); + } } public synchronized String saveDatatypeJSON(String dtypeid, String props_json) { ArrayList errors = new ArrayList(); ArrayList warnings = new ArrayList(); - - if (dtypeid == null || props_json == null) + + if(dtypeid == null || props_json == null) return "[\"Null inputs\"]"; try { @@ -475,19 +483,19 @@ public synchronized String saveDatatypeJSON(String dtypeid, String props_json) { dc.start_write(); dc.start_batch_operation(); - + // Save datatype's name format String fmt = null; String sensor = null; if (ops.get("format") != null && !ops.get("format").isJsonNull()) { fmt = ops.get("format").getAsString(); } - // Same datatype's sensing component - if (ops.get("sensor") != null && !ops.get("sensor").isJsonNull()) { - sensor = ops.get("sensor").getAsString(); - } - this.dc.setTypeAnnotations(dtypeid, fmt, sensor); - + // Same datatype's sensing component + if (ops.get("sensor") != null && !ops.get("sensor").isJsonNull()) { + sensor = ops.get("sensor").getAsString(); + } + this.dc.setTypeAnnotations(dtypeid, fmt, sensor); + // Check all properties being added JsonObject addops = ops.get("add").getAsJsonObject(); for (Map.Entry entry : addops.entrySet()) { @@ -500,33 +508,35 @@ public synchronized String saveDatatypeJSON(String dtypeid, String props_json) { String range = prop.get("range").getAsString(); MetadataProperty eprop = dc.getMetadataProperty(propid); if (eprop != null) { - // A property with this id already exists. + // A property with this id already exists. // Add dtypeid as a domain of the property this.dc.addMetadataPropertyDomain(propid, dtypeid); - } else { + } + else { this.dc.addMetadataProperty(propid, dtypeid, range); } } - /* - * dc.removeMetadataPropertyInLibrary(String propid); - * dc.renameMetadataPropertyInLibrary(String oldid, String newid); - */ - + /* + dc.removeMetadataPropertyInLibrary(String propid); + dc.renameMetadataPropertyInLibrary(String oldid, String newid); + */ + // Check all properties being deleted JsonObject delops = ops.get("del").getAsJsonObject(); for (Map.Entry entry : delops.entrySet()) { String propid = entry.getKey(); MetadataProperty eprop = dc.getMetadataProperty(propid); - if (eprop.getDomains().size() > 1) { + if(eprop.getDomains().size() > 1) { // There are more than one datatypes using this property // just remove the property from this datatype this.dc.removeMetadataPropertyDomain(propid, dtypeid); eprop.getDomains().remove(dtypeid); warnings.add("Note that the property you deleted currently also " - + "exists for other datatypes: " + + "exists for other datatypes: " + eprop.getDomains()); - } else { + } + else { this.dc.removeMetadataProperty(propid); this.dc.removeMetadataPropertyInLibrary(propid); } @@ -545,53 +555,55 @@ public synchronized String saveDatatypeJSON(String dtypeid, String props_json) { String npropid = prop.get("pid").getAsString(); MetadataProperty eprop = this.dc.getMetadataProperty(propid); MetadataProperty enprop = this.dc.getMetadataProperty(npropid); - if (enprop != null && !propid.equals(npropid) && - !range.equals(enprop.getRange())) { - errors.add("Property " + enprop.getName() - + " already exists with a different range: " - + enprop.getRange()); - continue; + if(enprop != null && !propid.equals(npropid) && + !range.equals(enprop.getRange())) { + errors.add("Property "+enprop.getName() + + " already exists with a different range: " + + enprop.getRange()); + continue; } if (!eprop.getRange().equals(range)) { this.dc.removeMetadataProperty(propid); this.dc.addMetadataProperty(npropid, dtypeid, range); // Re-add any other domains that the property might have had - for (String domid : eprop.getDomains()) { - if (!domid.equals(dtypeid)) + for(String domid : eprop.getDomains()) { + if(!domid.equals(dtypeid)) this.dc.addMetadataPropertyDomain(npropid, domid); } } else if (!propid.equals(npropid)) { this.dc.renameMetadataProperty(propid, npropid); this.dc.renamePropertyInLibrary(propid, npropid); } - if (eprop.getDomains().size() > 1) { + if(eprop.getDomains().size() > 1) { eprop.getDomains().remove(dtypeid); warnings.add("Note that the property you modified also " - + "exists for other datatypes, and would have been modified for them as well : " + + "exists for other datatypes, and would have been modified for them as well : " + eprop.getDomains()); } } this.dc.stop_batch_operation(); - - if (errors.size() == 0) { - if (!dc.save() || !dc.end()) { - errors.add("Could not save Data catalog"); - } - } else { - dc.end(); + + if(errors.size() == 0) { + if(!dc.save() || !dc.end()) { + errors.add("Could not save Data catalog"); + } + } + else { + dc.end(); + } + + if(errors.size() == 0) { + String provlog = "Updating datatype properties"; + Provenance p = new Provenance(dtypeid); + p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); + if(!prov.addProvenance(p)) { + errors.add("Could not add provenance"); + } } - - if (errors.size() == 0) { - String provlog = "Updating datatype properties"; - Provenance p = new Provenance(dtypeid); - p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); - if (!prov.addProvenance(p)) { - errors.add("Could not add provenance"); - } - } - } catch (Exception e) { - this.dc.stop_batch_operation(); - this.dc.end(); + } + catch(Exception e) { + this.dc.stop_batch_operation(); + this.dc.end(); e.printStackTrace(); errors.add(e.getMessage()); } @@ -603,48 +615,51 @@ public synchronized String saveDatatypeJSON(String dtypeid, String props_json) { } public synchronized boolean addDatatype(String ptype, String dtype) { - if (ptype == null || dtype == null) + if(ptype == null || dtype == null) return false; - String provlog = "Creating datatype with parent " + KBUtils.getLocalName(ptype); - Provenance p = new Provenance(dtype); - p.addActivity(new ProvActivity(ProvActivity.CREATE, provlog)); - - return dc.addDatatype(dtype, ptype) && - prov.addProvenance(p); + String provlog = "Creating datatype with parent "+KBUtils.getLocalName(ptype); + Provenance p = new Provenance(dtype); + p.addActivity(new ProvActivity(ProvActivity.CREATE, provlog)); + + return + dc.addDatatype(dtype, ptype) && + prov.addProvenance(p); } public synchronized boolean moveDatatypeTo(String dtypeid, String fromtype, String totype) { - if (dtypeid == null || fromtype == null || totype == null) + if(dtypeid == null || fromtype == null || totype == null) return false; - String provlog = "Moving Datatype from " + KBUtils.getLocalName(fromtype) + - " to " + KBUtils.getLocalName(totype); - Provenance p = new Provenance(dtypeid); - p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); - - return dc.moveDatatypeParent(dtypeid, fromtype, totype) && - dc.moveDatatypeParentInLibrary(dtypeid, fromtype, totype) && - prov.addProvenance(p); - } - - public synchronized boolean moveDataTo(String dataid, String fromtype, String totype) { - if (dataid == null || fromtype == null || totype == null) - return false; - - String provlog = "Moving Data from " + KBUtils.getLocalName(fromtype) + - " to " + KBUtils.getLocalName(totype); - Provenance p = new Provenance(dataid); - p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); - - return dc.moveDataParent(dataid, fromtype, totype) && - prov.addProvenance(p); - } - + String provlog = "Moving Datatype from " + KBUtils.getLocalName(fromtype) + + " to " + KBUtils.getLocalName(totype); + Provenance p = new Provenance(dtypeid); + p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); + + return + dc.moveDatatypeParent(dtypeid, fromtype, totype) && + dc.moveDatatypeParentInLibrary(dtypeid, fromtype, totype) && + prov.addProvenance(p); + } + + public synchronized boolean moveDataTo(String dataid, String fromtype, String totype) { + if(dataid == null || fromtype == null || totype == null) + return false; + + String provlog = "Moving Data from " + KBUtils.getLocalName(fromtype) + + " to " + KBUtils.getLocalName(totype); + Provenance p = new Provenance(dataid); + p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); + + return + dc.moveDataParent(dataid, fromtype, totype) && + prov.addProvenance(p); + } + public synchronized boolean delDatatypes(String[] dtypes) { for (String dtypeid : dtypes) { - if (!dc.removeDatatype(dtypeid) || - !prov.removeAllProvenance(dtypeid)) { + if(!dc.removeDatatype(dtypeid) || + !prov.removeAllProvenance(dtypeid)) { return false; } } @@ -652,222 +667,230 @@ public synchronized boolean delDatatypes(String[] dtypes) { } public synchronized boolean addDataForDatatype(String dataid, String dtypeid) { - String provlog = "Creating data of type " + KBUtils.getLocalName(dtypeid); - Provenance p = new Provenance(dataid); - p.addActivity(new ProvActivity(ProvActivity.CREATE, provlog)); - - return dc.addData(dataid, dtypeid) && - prov.addProvenance(p); - } - - public synchronized String addRemoteDataForType(String dataLocationUrl, String dtypeid) { - try { - String provlog = "Downloading from " + dataLocationUrl + " and creating data of type " - + KBUtils.getLocalName(dtypeid); - - // Check if it doesn't already exist - URL url = new URL(dataLocationUrl); - String filename = KBUtils.sanitizeID(new File(url.getFile()).getName()); - String dataid = this.libns + filename; - // System.out.println("Check if data exists"); - if (dc.getDataLocation(dataid) != null) { - // Dataset already exists, so do not overwrite (FIXME : Make it configurable) - return dataid; - } - - Provenance p = new Provenance(dataid); - p.addActivity(new ProvActivity(ProvActivity.CREATE, provlog)); - // System.out.println("Adding new data"); - - String location = dc.getDefaultDataLocation(dataid); - // System.out.println("Copying "+ url + " to " + location); - FileUtils.copyURLToFile(url, new File(location)); - // System.out.println("Add data id: " + dataid); - dc.addData(dataid, dtypeid); - // System.out.println("Set data location of " + dataid); - if (dc.setDataLocation(dataid, location) && prov.addProvenance(p)) { - return dataid; - } - } catch (Exception e) { - e.printStackTrace(); - } - return null; - } - - public synchronized boolean addBatchData(String dtypeid, String[] dids, String[] locations, - HttpServletRequest request, ServletContext context) { - for (int i = 0; i < dids.length; i++) { - if (!dc.addData(dids[i], dtypeid)) + String provlog = "Creating data of type "+KBUtils.getLocalName(dtypeid); + Provenance p = new Provenance(dataid); + p.addActivity(new ProvActivity(ProvActivity.CREATE, provlog)); + + return + dc.addData(dataid, dtypeid) && + prov.addProvenance(p); + } + + public synchronized String addRemoteDataForType(String dataLocationUrl, String dtypeid) { + try { + String provlog = "Downloading from " + dataLocationUrl + " and creating data of type "+KBUtils.getLocalName(dtypeid); + + // Check if it doesn't already exist + URL url = new URL(dataLocationUrl); + String filename = KBUtils.sanitizeID(new File(url.getFile()).getName()); + String dataid = this.libns + filename; + //System.out.println("Check if data exists"); + if(dc.getDataLocation(dataid) != null) { + // Dataset already exists, so do not overwrite (FIXME : Make it configurable) + return dataid; + } + + Provenance p = new Provenance(dataid); + p.addActivity(new ProvActivity(ProvActivity.CREATE, provlog)); + //System.out.println("Adding new data"); + + String location = dc.getDefaultDataLocation(dataid); + //System.out.println("Copying "+ url + " to " + location); + FileUtils.copyURLToFile(url, new File(location)); + //System.out.println("Add data id: " + dataid); + dc.addData(dataid, dtypeid); + //System.out.println("Set data location of " + dataid); + if(dc.setDataLocation(dataid, location) && prov.addProvenance(p)) { + return dataid; + } + } + catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public synchronized boolean addBatchData(String dtypeid, String[] dids, String[] locations, + HttpServletRequest request, ServletContext context) { + for(int i=0; i i && locations[i] != null) - if (!dc.setDataLocation(dids[i], locations[i])) + if(locations.length > i && locations[i] != null) + if(!dc.setDataLocation(dids[i], locations[i])) return false; this.runSensorWorkflow(dids[i], request, context); } return true; } - - public String runSensorWorkflow(String dataid, HttpServletRequest request, ServletContext context) { - try { - DataItem dtype = this.dc.getDatatypeForData(dataid); - if (dtype == null) { - System.err.println("No datatype for data id: " + dataid); - return null; - } - - String tplid = this.dc.getTypeSensor(dtype.getID()); - if (tplid == null) { - return null; - } - - Variable[] invars = this.tc.getTemplate(tplid).getInputVariables(); - if (invars.length != 1) { - System.err.println("Template should have exactly 1 input variable. Has " + invars.length); - return null; - } - HashMap> dataBindings = new HashMap>(); - ArrayList inputfiles = new ArrayList(); - inputfiles.add(dataid); - dataBindings.put(invars[0].getID(), inputfiles); - - TemplateBindings tbindings = new TemplateBindings(); - tbindings.setCallbackUrl(this.config.getMainConfig().getServerUrl() + - this.config.getUserDomainUrl() + "/data/setMetadataFromSensorOutput?data_id=" + - URLEncoder.encode(dataid, "UTF-8")); - tbindings.setCallbackCookies(CookieHandler.httpCookiesFromServlet(request)); - tbindings.setDataBindings(dataBindings); - tbindings.setParameterBindings(new HashMap>()); - tbindings.setParameterTypes(new HashMap()); - tbindings.setComponentBindings(new HashMap()); - tbindings.setTemplateId(tplid); - - ArrayList runids = this.rc.expandAndRunTemplate(tbindings, context); - if (runids != null && runids.size() > 0) { - return runids.get(0); - } - } catch (Exception e) { - e.printStackTrace(); - this.cc.end(); - this.rc.end(); - this.dc.end(); - } - return null; - } - - public String runSensorComponent(String dataid, HttpServletRequest request, ServletContext context) { - try { - DataItem dtype = this.dc.getDatatypeForData(dataid); - if (dtype == null) { - System.err.println("No datatype for data id: " + dataid); - return null; - } - - String cid = this.dc.getTypeSensor(dtype.getID()); - if (cid == null) { - return null; - } - Component c = this.cc.getComponent(cid, true); - ArrayList inroles = c.getInputs(); - ArrayList outroles = c.getOutputs(); - if (inroles.size() != 1) { - System.err.println("Sensor component should have exactly 1 input. Has " + inroles.size()); - return null; - } - if (outroles.size() != 1) { - System.err.println("Sensor component should have exactly 1 output. Has " + outroles.size()); - return null; - } - - HashMap role_bindings = new HashMap(); - Binding b = new Binding(dataid); - b.setLocation(this.dc.getDataLocation(dataid)); - - Binding bout = new Binding(dataid + ExecutionFile.metaExtension); - bout.setLocation(b.getLocation() + ExecutionFile.metaExtension); - - role_bindings.put(inroles.get(0).getRoleName(), b); - role_bindings.put(outroles.get(0).getRoleName(), bout); - - String callbackUrl = request.getScheme() + "://" + request.getServerName() + ":" + request.getServerPort() + - this.config.getUserDomainUrl() + "/data/setMetadataFromSensorOutput?data_id=" + - URLEncoder.encode(dataid, "UTF-8"); - - this.rc.runComponent(cid, role_bindings, callbackUrl, CookieHandler.httpCookiesFromServlet(request), context); - } catch (Exception e) { - e.printStackTrace(); - this.rc.end(); - this.dc.end(); - this.cc.end(); - } - return null; - } - - public String setMetadataFromSensorOutput(String data_id, ExecutionPlan plan) { - // System.out.println(plan); - for (ExecutionStep step : plan.getAllExecutionSteps()) { - for (ExecutionFile file : step.getOutputFiles()) { - file.loadMetadataFromFileContents(); - this.dc.setMetadataForDataObject(data_id, file.getMetadata()); - } - } - return plan.getID(); - } + + + public String runSensorWorkflow(String dataid, HttpServletRequest request, ServletContext context) { + try { + DataItem dtype = this.dc.getDatatypeForData(dataid); + if(dtype == null) { + System.err.println("No datatype for data id: " + dataid); + return null; + } + + String tplid = this.dc.getTypeSensor(dtype.getID()); + if(tplid == null) { + return null; + } + + Variable[] invars = this.tc.getTemplate(tplid).getInputVariables(); + if(invars.length != 1) { + System.err.println("Template should have exactly 1 input variable. Has " + invars.length); + return null; + } + HashMap> dataBindings = new HashMap>(); + ArrayList inputfiles = new ArrayList(); + inputfiles.add(dataid); + dataBindings.put(invars[0].getID(), inputfiles); + + TemplateBindings tbindings = new TemplateBindings(); + tbindings.setCallbackUrl(this.config.getServerUrl() + + this.config.getUserDomainUrl() +"/data/setMetadataFromSensorOutput?data_id=" + + URLEncoder.encode(dataid, "UTF-8")); + tbindings.setCallbackCookies(CookieHandler.httpCookiesFromServlet(request)); + tbindings.setDataBindings(dataBindings); + tbindings.setParameterBindings(new HashMap>()); + tbindings.setParameterTypes(new HashMap()); + tbindings.setComponentBindings(new HashMap()); + tbindings.setTemplateId(tplid); + + ArrayList runids = this.rc.expandAndRunTemplate(tbindings, context); + if(runids != null && runids.size() > 0) { + return runids.get(0); + } + } + catch (Exception e) { + e.printStackTrace(); + this.cc.end(); + this.rc.end(); + this.dc.end(); + } + return null; + } + + public String runSensorComponent(String dataid, HttpServletRequest request, ServletContext context) { + try { + DataItem dtype = this.dc.getDatatypeForData(dataid); + if(dtype == null) { + System.err.println("No datatype for data id: " + dataid); + return null; + } + + String cid = this.dc.getTypeSensor(dtype.getID()); + if(cid == null) { + return null; + } + Component c = this.cc.getComponent(cid, true); + ArrayList inroles = c.getInputs(); + ArrayList outroles = c.getOutputs(); + if(inroles.size() != 1) { + System.err.println("Sensor component should have exactly 1 input. Has " + inroles.size()); + return null; + } + if(outroles.size() != 1) { + System.err.println("Sensor component should have exactly 1 output. Has " + outroles.size()); + return null; + } + + HashMap role_bindings = new HashMap(); + Binding b = new Binding(dataid); + b.setLocation(this.dc.getDataLocation(dataid)); + + Binding bout = new Binding(dataid + ExecutionFile.metaExtension); + bout.setLocation(b.getLocation() + ExecutionFile.metaExtension); + + role_bindings.put(inroles.get(0).getRoleName(), b); + role_bindings.put(outroles.get(0).getRoleName(), bout); + + String callbackUrl = request.getScheme() + "://" + request.getServerName() + ":" + request.getServerPort() + + this.config.getUserDomainUrl() +"/data/setMetadataFromSensorOutput?data_id=" + + URLEncoder.encode(dataid, "UTF-8"); + + this.rc.runComponent(cid, role_bindings, callbackUrl, CookieHandler.httpCookiesFromServlet(request), context); + } + catch (Exception e) { + e.printStackTrace(); + this.rc.end(); + this.dc.end(); + this.cc.end(); + } + return null; + } + + public String setMetadataFromSensorOutput(String data_id, ExecutionPlan plan) { + //System.out.println(plan); + for(ExecutionStep step : plan.getAllExecutionSteps()) { + for(ExecutionFile file : step.getOutputFiles()) { + file.loadMetadataFromFileContents(); + this.dc.setMetadataForDataObject(data_id, file.getMetadata()); + } + } + return plan.getID(); + } public synchronized boolean delData(String dataid) { - return dc.removeData(dataid) && - prov.removeAllProvenance(dataid); + return + dc.removeData(dataid) && + prov.removeAllProvenance(dataid); } - + public synchronized boolean renameData(String dataid, String newid) { - String provlog = "Renaming " + KBUtils.getLocalName(dataid) - + " to " + KBUtils.getLocalName(newid); - Provenance p = new Provenance(dataid); - p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); - - return dc.renameData(newid, dataid) && - prov.addProvenance(p); - } - + String provlog = "Renaming " + KBUtils.getLocalName(dataid) + + " to " + KBUtils.getLocalName(newid); + Provenance p = new Provenance(dataid); + p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); + + return + dc.renameData(newid, dataid) && + prov.addProvenance(p); + } + public synchronized boolean setDataLocation(String dataid, String location) { - String provlog = "Setting location"; - Provenance p = new Provenance(dataid); - p.addActivity(new ProvActivity(ProvActivity.UPLOAD, provlog)); - - return dc.setDataLocation(dataid, location) && - prov.addProvenance(p); - } - + String provlog = "Setting location"; + Provenance p = new Provenance(dataid); + p.addActivity(new ProvActivity(ProvActivity.UPLOAD, provlog)); + + return + dc.setDataLocation(dataid, location) && + prov.addProvenance(p); + } + public synchronized boolean renameDataType(String dtypeid, String newid) { - String provlog = "Renaming " + KBUtils.getLocalName(dtypeid) - + " to " + KBUtils.getLocalName(newid); - Provenance p = new Provenance(dtypeid); - p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); - - return dc.renameDatatype(newid, dtypeid) && - dc.renameDatatypeInLibrary(newid, dtypeid) && - prov.addProvenance(p); - } - + String provlog = "Renaming " + KBUtils.getLocalName(dtypeid) + + " to " + KBUtils.getLocalName(newid); + Provenance p = new Provenance(dtypeid); + p.addActivity(new ProvActivity(ProvActivity.UPDATE, provlog)); + + return + dc.renameDatatype(newid, dtypeid) && + dc.renameDatatypeInLibrary(newid, dtypeid) && + prov.addProvenance(p); + } + public void end() { - if (dc != null) { - dc.end(); - } - if (prov != null) { - prov.end(); - } - } - - public synchronized boolean importFromExternalCatalog(String dataid, String dtypeid, + if(dc != null) { + dc.end(); + } + if(prov != null) { + prov.end(); + } + } + + public synchronized boolean importFromExternalCatalog(String dataid, String dtypeid, String propvals_json, String location) { if (dc == null) return false; - + try { JsonParser parser = new JsonParser(); JsonElement propvals = parser.parse(propvals_json); DataItem dtype = dc.getDatatypeForData(dataid); - if (dtype == null) { + if(dtype == null) { dc.addData(dataid, dtypeid); } for (Map.Entry entry : propvals.getAsJsonObject().entrySet()) { @@ -880,49 +903,49 @@ public synchronized boolean importFromExternalCatalog(String dataid, String dtyp if (eprop == null) { // Property doesn't exist this.dc.addMetadataProperty(propid, dtypeid, range); - } else { - if (!eprop.getDomains().contains(dtypeid)) { - // Property exists for another class. Add this class as domain - this.dc.addMetadataPropertyDomain(propid, dtypeid); - } - range = eprop.getRange(); } - + else { + if(!eprop.getDomains().contains(dtypeid)) { + // Property exists for another class. Add this class as domain + this.dc.addMetadataPropertyDomain(propid, dtypeid); + } + range = eprop.getRange(); + } + this.dc.addDatatypePropertyValue(dataid, propid, value, range); } dc.setDataLocation(dataid, location); return true; - } catch (Exception e) { + } + catch(Exception e) { e.printStackTrace(); return false; } } - + private void trustAllCertificates() { - // Create a new trust manager that trust all certificates - TrustManager[] trustAllCerts = new TrustManager[] { - new X509TrustManager() { - public java.security.cert.X509Certificate[] getAcceptedIssuers() { - return null; - } - - public void checkClientTrusted( - java.security.cert.X509Certificate[] certs, String authType) { - } - - public void checkServerTrusted( - java.security.cert.X509Certificate[] certs, String authType) { - } - } - }; - - // Activate the new trust manager - try { - SSLContext sc = SSLContext.getInstance("SSL"); - sc.init(null, trustAllCerts, new java.security.SecureRandom()); - HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); - } catch (Exception e) { - } + // Create a new trust manager that trust all certificates + TrustManager[] trustAllCerts = new TrustManager[]{ + new X509TrustManager() { + public java.security.cert.X509Certificate[] getAcceptedIssuers() { + return null; + } + public void checkClientTrusted( + java.security.cert.X509Certificate[] certs, String authType) { + } + public void checkServerTrusted( + java.security.cert.X509Certificate[] certs, String authType) { + } + } + }; + + // Activate the new trust manager + try { + SSLContext sc = SSLContext.getInstance("SSL"); + sc.init(null, trustAllCerts, new java.security.SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + } catch (Exception e) { + } } } diff --git a/portal/src/main/java/edu/isi/wings/portal/resources/TemplateResource.java b/portal/src/main/java/edu/isi/wings/portal/resources/TemplateResource.java index 36ebf332..ebfa90b9 100644 --- a/portal/src/main/java/edu/isi/wings/portal/resources/TemplateResource.java +++ b/portal/src/main/java/edu/isi/wings/portal/resources/TemplateResource.java @@ -24,59 +24,58 @@ public class TemplateResource extends WingsResource { TemplateController tc; - @PathParam("edit") - String edit; - @PathParam("tell") - String tell; + @PathParam("edit") String edit; + @PathParam("tell") String tell; - HashMap options; + HashMap options; boolean editor; boolean tellme; - + @PostConstruct public void init() { super.init(); - + this.editor = ("/edit".equals(edit)); this.tellme = ("/tellme".equals(tell)); - - if (this.hasPermissions() && !this.isPage("intro")) + + if(this.hasPermissions() && !this.isPage("intro")) this.tc = new TemplateController(config); this.options = new HashMap(); @SuppressWarnings("unchecked") Enumeration pnames = request.getParameterNames(); - while (pnames.hasMoreElements()) { + while(pnames.hasMoreElements()) { String pname = pnames.nextElement(); - if (pname.matches("^hide_.*")) + if(pname.matches("^hide_.*")) this.options.put(pname, Boolean.parseBoolean(request.getParameter(pname))); } } - + @PreDestroy public void destroy() { System.out.println("Template Resource Destroyed"); - if (this.tc != null) { + if(this.tc != null) { this.tc.end(); } - } - + } + @GET @Path("intro") public void getIntroduction() { String intropage = "Template"; - if (!editor) + if(!editor) intropage += "Browser"; - else + else intropage += "Editor" + (tellme ? "Tellme" : ""); this.loadIntroduction(intropage); } - + + @GET @Produces(MediaType.TEXT_HTML) public String getHTML( @QueryParam("template_id") final String template_id) { - if (this.tc != null) { + if(this.tc != null) { request.setAttribute("controller", this.tc); request.setAttribute("options", this.options); request.setAttribute("editor", this.editor); @@ -85,16 +84,17 @@ public String getHTML( } return null; } + @GET @Path("{template_name}.owl") @Produces(MediaType.TEXT_HTML) public String getTemplateHTML( @PathParam("template_name") String template_name) { - if (this.tc != null) { - config.setScriptPath(config.getScriptPath().replaceAll("/" + template_name + ".owl", "")); + if(this.tc != null) { + config.setScriptPath(config.getScriptPath().replaceAll("/"+template_name+".owl", "")); this.options.put("hide_selector", true); - + request.setAttribute("controller", this.tc); request.setAttribute("options", this.options); request.setAttribute("editor", this.editor); @@ -104,42 +104,42 @@ public String getTemplateHTML( } return null; } - + @GET @Path("getTemplatesListJSON") @Produces(MediaType.APPLICATION_JSON) public String getTemplatesListJSON() { - if (this.tc != null) + if(this.tc != null) return this.tc.getTemplatesListJSON(); return null; } - + @GET @Path("getViewerJSON") @Produces(MediaType.APPLICATION_JSON) public String getViewerJSON( @QueryParam("template_id") String template_id) { - if (this.tc != null) + if(this.tc != null) return this.tc.getViewerJSON(template_id); return null; } - + @GET @Path("getInputsJSON") @Produces(MediaType.APPLICATION_JSON) public String getInputsJSON( @QueryParam("template_id") String template_id) { - if (this.tc != null) + if(this.tc != null) return this.tc.getInputsJSON(template_id); return null; } - + @GET @Path("getEditorJSON") @Produces(MediaType.APPLICATION_JSON) public String getEditorJSON( @QueryParam("template_id") String template_id) { - if (this.tc != null) + if(this.tc != null) return this.tc.getEditorJSON(template_id); return null; } @@ -148,8 +148,8 @@ public String getEditorJSON( @Path("layoutTemplate") @Produces(MediaType.APPLICATION_JSON) public String layoutTemplate(@JsonProperty("json") String json) { - String dotexe = config.getMainConfig().getDotFile(); - if (this.tc != null) + String dotexe = config.getDotFile(); + if(this.tc != null) try { return this.tc.layoutTemplate(json, dotexe); } catch (IOException e) { @@ -165,29 +165,29 @@ public String saveTemplateJSON( @FormParam("template_id") String template_id, @FormParam("json") String json, @FormParam("constraints_json") String constraints_json) { - if (this.tc != null) + if(this.tc != null) return this.tc.saveTemplateJSON(template_id, json, constraints_json); return null; } - + @POST @Path("newTemplate") @Produces(MediaType.TEXT_PLAIN) public String newTemplate( @FormParam("template_id") String template_id) { - if (this.tc != null) { + if(this.tc != null) { RunController.invalidateCachedAPIs(); return this.tc.newTemplate(template_id); } return null; } - + @POST @Path("deleteTemplate") @Produces(MediaType.TEXT_PLAIN) public String deleteTemplate( @FormParam("template_id") String template_id) { - if (this.tc != null && this.isOwner() && !config.isSandboxed()) { + if(this.tc != null && this.isOwner() && !config.isSandboxed()) { RunController.invalidateCachedAPIs(); return this.tc.deleteTemplate(template_id); } diff --git a/portal/src/main/java/edu/isi/wings/portal/servlets/ExportGraph.java b/portal/src/main/java/edu/isi/wings/portal/servlets/ExportGraph.java index 80992de4..76700b20 100644 --- a/portal/src/main/java/edu/isi/wings/portal/servlets/ExportGraph.java +++ b/portal/src/main/java/edu/isi/wings/portal/servlets/ExportGraph.java @@ -37,7 +37,7 @@ */ public class ExportGraph extends HttpServlet { private static final long serialVersionUID = 1L; - + /** * @see HttpServlet#HttpServlet() */ @@ -53,47 +53,49 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { PrintWriter out = response.getWriter(); Config config = new Config(request, null, null); - // if(!config.checkDomain(request, response)) - // return; - - response.addHeader("Access-Control-Allow-Origin", config.getMainConfig().getClients()); + //if(!config.checkDomain(request, response)) + // return; + + response.addHeader("Access-Control-Allow-Origin", config.getClients()); response.addHeader("Access-Control-Allow-Credentials", "true"); response.addHeader("Access-Control-Allow-Methods", "GET, POST, DELETE, PUT"); response.addHeader("Access-Control-Allow-Headers", - "X-Requested-With, Content-Type, X-HTTP-Method-Override"); - - String format = request.getParameter("format"); - String uri = config.getMainConfig().getServerUrl() + request.getRequestURI(); + "X-Requested-With, Content-Type, X-HTTP-Method-Override"); + + String format = request.getParameter("format"); + String uri = config.getServerUrl() + request.getRequestURI(); OntFactory tdbfac = new OntFactory(OntFactory.JENA, config.getTripleStoreDir()); try { - Pattern complibpat = Pattern.compile(".+\\/components\\/(.+)\\.owl"); + Pattern complibpat = Pattern.compile(".+\\/components\\/(.+)\\.owl"); KBAPI kb = tdbfac.getKB(uri, OntSpec.PLAIN); - + Matcher mat = complibpat.matcher(uri); - if (mat.find()) { - String abslib = "abstract"; - String complib = mat.group(1); - if (!complib.equals(abslib)) { - String absuri = uri.replace(complib, abslib); - tdbfac.start_write_transaction(); - kb.copyFrom(tdbfac.getKB(absuri, OntSpec.PLAIN)); - tdbfac.end_transaction(); - } + if(mat.find()) { + String abslib = "abstract"; + String complib = mat.group(1); + if(!complib.equals(abslib)) { + String absuri = uri.replace(complib, abslib); + tdbfac.start_write_transaction(); + kb.copyFrom(tdbfac.getKB(absuri, OntSpec.PLAIN)); + tdbfac.end_transaction(); + } } - - tdbfac.start_read_transaction(); - if (kb.getAllTriples().size() > 0) { + + tdbfac.start_read_transaction(); + if(kb.getAllTriples().size() > 0) { response.setContentType("application/rdf+xml"); - if (format != null) { - if (format.equals("json")) - out.println(kb.toJson()); - else if (format.equals("n3")) - out.println(kb.toN3()); - } else - out.println(kb.toAbbrevRdf(true)); + if(format != null) { + if(format.equals("json")) + out.println(kb.toJson()); + else if(format.equals("n3")) + out.println(kb.toN3()); + } + else + out.println(kb.toAbbrevRdf(true)); } tdbfac.end_transaction(); - } catch (Exception e) { + } + catch (Exception e) { e.printStackTrace(); } } diff --git a/portal/src/main/java/edu/isi/wings/portal/servlets/SparqlEndpoint.java b/portal/src/main/java/edu/isi/wings/portal/servlets/SparqlEndpoint.java index fae78b80..ac5acebb 100644 --- a/portal/src/main/java/edu/isi/wings/portal/servlets/SparqlEndpoint.java +++ b/portal/src/main/java/edu/isi/wings/portal/servlets/SparqlEndpoint.java @@ -35,24 +35,23 @@ */ public class SparqlEndpoint extends HttpServlet { private static final long serialVersionUID = 1L; - + private PrintStream out; private SparqlAPI api; + + /** + * @see HttpServlet#HttpServlet() + */ + public SparqlEndpoint() { + super(); + } /** - * @see HttpServlet#HttpServlet() - */ - public SparqlEndpoint() { - super(); - } - - /** - * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse - * response) + * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) */ protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { this.out = new PrintStream(response.getOutputStream()); - + Config config = new Config(request, null, null); String queryString = request.getParameter("query"); String updateString = request.getParameter("update"); @@ -70,76 +69,78 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t + ""); return; } - + try { - if (queryString != null && !queryString.equals("")) + if(queryString != null && !queryString.equals("")) this.showQueryResults(queryString, request, response); - else if (updateString != null && !updateString.equals("")) + else if(updateString != null && !updateString.equals("")) this.updateDataset(updateString, request, response); - } catch (Exception e) { + } + catch (Exception e) { response.getOutputStream().print(e.getMessage()); } response.getOutputStream().flush(); } - private void showQueryResults(String queryString, HttpServletRequest request, HttpServletResponse response) + private void showQueryResults(String queryString, HttpServletRequest request, HttpServletResponse response) throws IOException { - Config config = new Config(request, null, null); - String tdbdir = config.getTripleStoreDir(); - String format = request.getParameter("format"); - - this.api = new SparqlAPI(tdbdir); - this.api.showQueryResults(queryString, format, out); + Config config = new Config(request, null, null); + String tdbdir = config.getTripleStoreDir(); + String format = request.getParameter("format"); + + this.api = new SparqlAPI(tdbdir); + this.api.showQueryResults(queryString, format, out); } - - private void updateDataset(String updateString, HttpServletRequest request, HttpServletResponse response) + + private void updateDataset(String updateString, HttpServletRequest request, HttpServletResponse response) throws IOException { Config config = new Config(request, null, null); - if (!config.checkDomain(request, response)) + if(!config.checkDomain(request, response)) return; - String tdbdir = config.getTripleStoreDir(); + String tdbdir = config.getTripleStoreDir(); this.api = new SparqlAPI(tdbdir); - - if (updateString.startsWith("__SERVER_RENAME:")) { - String newurl = updateString.substring(updateString.indexOf(":") + 1); - this.updateServerURL(newurl, config); - } else { - this.api.executeUpdateQuery(updateString, out); + + if(updateString.startsWith("__SERVER_RENAME:")) { + String newurl = updateString.substring(updateString.indexOf(":")+1); + this.updateServerURL(newurl, config); + } + else { + this.api.executeUpdateQuery(updateString, out); } } - + private void updateServerURL(String newurl, Config config) { - String cururl = config.getMainConfig().getServerUrl(); - - // Update all graphs in the triple store - this.api.updateGraphURLs(cururl, newurl, out); - - // Update all User domains - for (String userid : config.getUsersList()) { - config.setUserId(userid); - config.setViewerId(userid); - DomainController dc = new DomainController(config); - for (String domname : dc.getDomainsList()) { - DomainInfo dominfo = dc.getDomainInfo(domname); - String url = dominfo.getUrl(); - if (url.startsWith(cururl)) { - try { - out.println("* changing " + userid + "'s domain url for " + domname); - } catch (Exception e) { - System.out.println("* changing " + userid + "'s domain url for " + domname); - } - url = url.replace(cururl, newurl); - dc.setDomainURL(domname, url); - } - } - } + String cururl = config.getServerUrl(); + + // Update all graphs in the triple store + this.api.updateGraphURLs(cururl, newurl, out); + + // Update all User domains + for(String userid : config.getUsersList()) { + config.setUserId(userid); + config.setViewerId(userid); + DomainController dc = new DomainController(config); + for(String domname : dc.getDomainsList()) { + DomainInfo dominfo = dc.getDomainInfo(domname); + String url = dominfo.getUrl(); + if(url.startsWith(cururl)) { + try { + out.println("* changing "+userid+"'s domain url for "+domname); + } + catch(Exception e) { + System.out.println("* changing "+userid+"'s domain url for "+domname); + } + url = url.replace(cururl, newurl); + dc.setDomainURL(domname, url); + } + } + } } /** - * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse - * response) + * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response) */ protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { // TODO Auto-generated method stub diff --git a/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java b/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java index 2c14eba3..90e63725 100644 --- a/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java +++ b/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java @@ -54,9 +54,9 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) Config config = new Config(request, null, null); Gson json = JsonHandler.createPrettyGson(); HashMap props = new HashMap(); - props.put("internal_server", config.getMainConfig().getServerUrl()); + props.put("internal_server", config.getServerUrl()); props.put("storage", config.getStorageDirectory()); - props.put("dotpath", config.getMainConfig().getDotFile()); + props.put("dotpath", config.getDotFile()); props.put("ontology", config.getOntologyConfig().getWorkflowOntologyUrl()); props.put("planner", config.getPlannerConfig()); out.println(json.toJson(props)); From ef9f204230b0db12ab7150ccd5efbe18b1ffac89 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Wed, 12 Jul 2023 13:09:25 -0400 Subject: [PATCH 05/11] Revert "Improve ontology configuration handling (#160)" This reverts commit 7101a3aacddb68326d447349b36f930338ed4e00. --- .../wings/portal/classes/config/Config.java | 101 +++++++++++++----- .../portal/classes/config/OntologyConfig.java | 73 ------------- .../isi/wings/portal/servlets/ViewConfig.java | 4 +- 3 files changed, 78 insertions(+), 100 deletions(-) delete mode 100644 portal/src/main/java/edu/isi/wings/portal/classes/config/OntologyConfig.java diff --git a/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java b/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java index 67576005..882b9662 100644 --- a/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java +++ b/portal/src/main/java/edu/isi/wings/portal/classes/config/Config.java @@ -64,10 +64,15 @@ public class Config { private String logsDirectory; private String dotFile; private String serverUrl; - private OntologyConfig ontologyConfig; - + private String workflowOntologyUrl; + private String dataOntologyUrl; + private String componentOntologyUrl; + private String executionOntologyUrl; + private String resourceOntologyUrl; private boolean deleteRunOutputs; + private String ontdirurl = "http://www.wings-workflows.org/ontology"; + private HashMap engines; private Publisher publisher; @@ -325,8 +330,11 @@ private void initializePortalConfig(HttpServletRequest request) { this.serverUrl = serverConfig.getString("server"); this.dotFile = serverConfig.getString("graphviz"); this.clients = serverConfig.getString("clients"); - this.ontologyConfig = new OntologyConfig(serverConfig); - + this.dataOntologyUrl = serverConfig.getString("ontology.data"); + this.componentOntologyUrl = serverConfig.getString("ontology.component"); + this.workflowOntologyUrl = serverConfig.getString("ontology.workflow"); + this.executionOntologyUrl = serverConfig.getString("ontology.execution"); + this.resourceOntologyUrl = serverConfig.getString("ontology.resource"); if (serverConfig.containsKey("metaworkflows")) this.hasMetaWorkflows = serverConfig.getBoolean("metaworkflows"); @@ -534,19 +542,25 @@ private void createDefaultPortalConfig(HttpServletRequest request) { File loc1 = new File("/usr/bin/dot"); File loc2 = new File("/usr/local/bin/dot"); config.addProperty("graphviz", loc2.exists() ? loc2.getAbsolutePath() : loc1.getAbsolutePath()); - - OntologyConfig ontConfig = new OntologyConfig(); - config.addProperty("ontology.data", ontConfig.getDataOntologyUrl()); - config.addProperty("ontology.component", ontConfig.getComponentOntologyUrl()); - config.addProperty("ontology.workflow", ontConfig.getWorkflowOntologyUrl()); - config.addProperty("ontology.execution", ontConfig.getExecutionOntologyUrl()); - config.addProperty("ontology.resource", ontConfig.getResourceOntologyUrl()); + config.addProperty("ontology.data", ontdirurl + "/data.owl"); + config.addProperty("ontology.component", ontdirurl + "/component.owl"); + config.addProperty("ontology.workflow", ontdirurl + "/workflow.owl"); + config.addProperty("ontology.execution", ontdirurl + "/execution.owl"); + config.addProperty("ontology.resource", ontdirurl + "/resource.owl"); this.addEngineConfig(config, new ExeEngine("Local", LocalExecutionEngine.class.getCanonicalName(), ExeEngine.Type.BOTH)); this.addEngineConfig(config, new ExeEngine("Distributed", DistributedExecutionEngine.class.getCanonicalName(), ExeEngine.Type.BOTH)); + /* + * this.addEngineConfig(config, new ExeEngine("OODT", + * OODTExecutionEngine.class.getCanonicalName(), ExeEngine.Type.PLAN)); + * + * this.addEngineConfig(config, new ExeEngine("Pegasus", + * PegasusExecutionEngine.class.getCanonicalName(), ExeEngine.Type.PLAN)); + */ + try { config.save(this.configFile); } catch (Exception e) { @@ -574,15 +588,15 @@ public Properties getProperties(Domain domain) { if (domain.isLegacy()) return props; + props.setProperty("ont.dir.url", this.ontdirurl); if (!domain.getUseSharedTripleStore()) props.setProperty("ont.dir.map", "file:" + domain.getDomainDirectory() + File.separator + "ontology"); - props.setProperty("ont.dir.url", this.ontologyConfig.getOntologyDefaultUrl()); - props.setProperty("ont.data.url", this.ontologyConfig.getDataOntologyUrl()); - props.setProperty("ont.component.url", this.ontologyConfig.getComponentOntologyUrl()); - props.setProperty("ont.workflow.url", this.ontologyConfig.getWorkflowOntologyUrl()); - props.setProperty("ont.execution.url", this.ontologyConfig.getExecutionOntologyUrl()); + props.setProperty("ont.data.url", this.getDataOntologyUrl()); + props.setProperty("ont.component.url", this.getComponentOntologyUrl()); + props.setProperty("ont.workflow.url", this.getWorkflowOntologyUrl()); + props.setProperty("ont.execution.url", this.getExecutionOntologyUrl()); if (domain.getUseSharedTripleStore()) props.setProperty("tdb.repository.dir", this.getTripleStoreDir()); @@ -595,6 +609,11 @@ public Properties getProperties(Domain domain) { } props.setProperty("logs.dir", this.getLogsDirectory()); props.setProperty("dot.path", this.getDotFile()); + + if (this.getResourceOntologyUrl() == null) + this.setResourceOntologyUrl(ontdirurl + "/resource.owl"); + props.setProperty("ont.resource.url", this.getResourceOntologyUrl()); + props.setProperty("lib.resource.url", this.getExportCommunityUrl() + "/resource/library.owl"); @@ -650,6 +669,47 @@ public ExecutionMonitorAPI getDomainExecutionMonitor() { return ExecutionToolsFactory.createMonitor(this.getProperties()); } + // Getters and Setters + public String getWorkflowOntologyUrl() { + return this.workflowOntologyUrl; + } + + public String getComponentOntologyUrl() { + return this.componentOntologyUrl; + } + + public String getDataOntologyUrl() { + return this.dataOntologyUrl; + } + + public void setWorkflowOntologyUrl(String workflowOntologyUrl) { + this.workflowOntologyUrl = workflowOntologyUrl; + } + + public void setDataOntologyUrl(String dataOntologyUrl) { + this.dataOntologyUrl = dataOntologyUrl; + } + + public void setComponentOntologyUrl(String componentOntologyUrl) { + this.componentOntologyUrl = componentOntologyUrl; + } + + public String getExecutionOntologyUrl() { + return executionOntologyUrl; + } + + public void setExecutionOntologyUrl(String executionOntologyUrl) { + this.executionOntologyUrl = executionOntologyUrl; + } + + public String getResourceOntologyUrl() { + return resourceOntologyUrl; + } + + public void setResourceOntologyUrl(String resourceOntologyUrl) { + this.resourceOntologyUrl = resourceOntologyUrl; + } + public String getConfigFile() { return configFile; } @@ -863,13 +923,4 @@ public void setPlannerConfig(PlannerConfig plannerConfig) { public Set getEnginesList() { return this.engines.keySet(); } - - public OntologyConfig getOntologyConfig() { - return ontologyConfig; - } - - public void setOntologyConfig(OntologyConfig ontologyConfig) { - this.ontologyConfig = ontologyConfig; - } - } diff --git a/portal/src/main/java/edu/isi/wings/portal/classes/config/OntologyConfig.java b/portal/src/main/java/edu/isi/wings/portal/classes/config/OntologyConfig.java deleted file mode 100644 index ef3fb12c..00000000 --- a/portal/src/main/java/edu/isi/wings/portal/classes/config/OntologyConfig.java +++ /dev/null @@ -1,73 +0,0 @@ -package edu.isi.wings.portal.classes.config; - -import org.apache.commons.configuration.plist.PropertyListConfiguration; - -public class OntologyConfig { - - private String ontologyDefaultUrl = "http://www.wings-workflows.org/ontology"; - private String resourceOntologyUrl = ontologyDefaultUrl + "/resource.owl"; - private String workflowOntologyUrl = ontologyDefaultUrl + "/workflow.owl"; - private String dataOntologyUrl = ontologyDefaultUrl + "/data.owl"; - private String componentOntologyUrl = ontologyDefaultUrl + "/component.owl"; - private String executionOntologyUrl = ontologyDefaultUrl + "/execution.owl"; - - public OntologyConfig(PropertyListConfiguration serverConfig) { - dataOntologyUrl = serverConfig.getString("ontology.data"); - componentOntologyUrl = serverConfig.getString("ontology.component"); - workflowOntologyUrl = serverConfig.getString("ontology.workflow"); - executionOntologyUrl = serverConfig.getString("ontology.execution"); - resourceOntologyUrl = serverConfig.getString("ontology.resource"); - } - - public OntologyConfig() { - } - - public String getWorkflowOntologyUrl() { - return workflowOntologyUrl; - } - - public String getDataOntologyUrl() { - return dataOntologyUrl; - } - - public String getComponentOntologyUrl() { - return componentOntologyUrl; - } - - public String getExecutionOntologyUrl() { - return executionOntologyUrl; - } - - public String getResourceOntologyUrl() { - return resourceOntologyUrl; - } - - public void setWorkflowOntologyUrl(String workflowOntologyUrl) { - this.workflowOntologyUrl = workflowOntologyUrl; - } - - public void setDataOntologyUrl(String dataOntologyUrl) { - this.dataOntologyUrl = dataOntologyUrl; - } - - public void setComponentOntologyUrl(String componentOntologyUrl) { - this.componentOntologyUrl = componentOntologyUrl; - } - - public void setExecutionOntologyUrl(String executionOntologyUrl) { - this.executionOntologyUrl = executionOntologyUrl; - } - - public void setResourceOntologyUrl(String resourceOntologyUrl) { - this.resourceOntologyUrl = resourceOntologyUrl; - } - - public String getOntologyDefaultUrl() { - return ontologyDefaultUrl; - } - - public void setOntologyDefaultUrl(String ontologyDefaultUrl) { - this.ontologyDefaultUrl = ontologyDefaultUrl; - } - -} diff --git a/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java b/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java index 90e63725..bf74f893 100644 --- a/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java +++ b/portal/src/main/java/edu/isi/wings/portal/servlets/ViewConfig.java @@ -36,7 +36,7 @@ */ public class ViewConfig extends HttpServlet { private static final long serialVersionUID = 1L; - + /** * @see HttpServlet#HttpServlet() */ @@ -57,7 +57,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) props.put("internal_server", config.getServerUrl()); props.put("storage", config.getStorageDirectory()); props.put("dotpath", config.getDotFile()); - props.put("ontology", config.getOntologyConfig().getWorkflowOntologyUrl()); + props.put("ontology", config.getWorkflowOntologyUrl()); props.put("planner", config.getPlannerConfig()); out.println(json.toJson(props)); out.close(); From c2f02a3094b074e85fc224a1a19585b0a841a607 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Mon, 17 Jul 2023 18:51:55 -0400 Subject: [PATCH 06/11] add: prettier (#162) --- .github/workflows/prettier.yml | 26 + .prettierrc | 1 + .../api/impl/oodt/ComponentCreationFM.java | 258 +- .../data/api/impl/oodt/DataCreationFM.java | 1414 ++--- .../api/impl/oodt/OODTExecutionEngine.java | 249 +- .../api/impl/oodt/OODTWorkflowAdapter.java | 383 +- .../wings/util/oodt/CurationServiceAPI.java | 276 +- .../oodt/cas/workflow/misc/WingsTask.java | 177 +- .../wmservices/client/WmServicesClient.java | 55 +- .../repository/PackagedWorkflowManager.java | 102 +- .../cas/wmservices/resources/Resource.java | 38 +- .../resources/WorkflowResource.java | 43 +- .../servlets/WmServicesServlet.java | 50 +- .../impl/pegasus/PegasusExecutionEngine.java | 1019 ++-- .../impl/pegasus/PegasusWorkflowAdapter.java | 879 +-- .../catalog/component/ComponentFactory.java | 157 +- .../component/api/ComponentCreationAPI.java | 102 +- .../component/api/ComponentReasoningAPI.java | 18 +- .../api/impl/kb/ComponentCreationKB.java | 1060 ++-- .../component/api/impl/kb/ComponentKB.java | 1357 +++-- .../api/impl/kb/ComponentReasoningKB.java | 2495 ++++---- .../api/impl/kb/TemplateReasoningKB.java | 434 +- .../catalog/component/classes/Component.java | 27 +- .../component/classes/ComponentHolder.java | 24 +- .../classes/ComponentInvocation.java | 167 +- .../component/classes/ComponentPacket.java | 454 +- .../component/classes/ComponentRole.java | 121 +- .../component/classes/ComponentTree.java | 4 +- .../component/classes/ComponentTreeNode.java | 43 +- .../requirements/ComponentRequirement.java | 9 +- .../classes/requirements/Dependency.java | 51 +- .../classes/rules/ComponentRule.java | 76 +- .../component/classes/rules/Effect.java | 4 +- .../component/classes/rules/Precondition.java | 4 +- .../isi/wings/catalog/data/DataFactory.java | 90 +- .../catalog/data/api/DataCreationAPI.java | 151 +- .../catalog/data/api/DataReasoningAPI.java | 32 +- .../data/api/impl/kb/DataCreationKB.java | 1700 +++--- .../catalog/data/api/impl/kb/DataKB.java | 273 +- .../data/api/impl/kb/DataReasoningKB.java | 559 +- .../wings/catalog/data/classes/DataItem.java | 31 +- .../wings/catalog/data/classes/DataTree.java | 92 +- .../catalog/data/classes/DataTreeNode.java | 66 +- .../data/classes/MetadataProperty.java | 131 +- .../catalog/data/classes/MetadataValue.java | 73 +- .../data/classes/VariableBindings.java | 167 +- .../data/classes/VariableBindingsList.java | 3 +- .../data/classes/VariableBindingsListSet.java | 7 +- .../catalog/data/classes/metrics/Metric.java | 157 +- .../catalog/data/classes/metrics/Metrics.java | 70 +- .../catalog/provenance/ProvenanceFactory.java | 4 +- .../catalog/provenance/api/ProvenanceAPI.java | 22 +- .../provenance/api/impl/kb/ProvenanceKB.java | 239 +- .../provenance/classes/ProvActivity.java | 4 +- .../provenance/classes/Provenance.java | 1 + .../catalog/resource/ResourceFactory.java | 32 +- .../catalog/resource/api/ResourceAPI.java | 31 +- .../resource/api/impl/kb/ResourceKB.java | 715 +-- .../resource/classes/EnvironmentValue.java | 2 +- .../resource/classes/GridkitCloud.java | 95 +- .../catalog/resource/classes/Machine.java | 98 +- .../catalog/resource/classes/Resource.java | 3 +- .../catalog/resource/classes/Software.java | 5 +- .../resource/classes/SoftwareEnvironment.java | 1 + .../resource/classes/SoftwareVersion.java | 5 +- .../isi/wings/common/CollectionsHelper.java | 60 +- .../common/SerializableObjectCloner.java | 286 +- .../java/edu/isi/wings/common/URIEntity.java | 86 +- .../java/edu/isi/wings/common/UuidGen.java | 77 +- .../java/edu/isi/wings/common/kb/KBUtils.java | 367 +- .../isi/wings/common/kb/PropertiesHelper.java | 959 +-- .../edu/isi/wings/common/logging/Escape.java | 179 +- .../wings/common/logging/EventLogMessage.java | 736 +-- .../isi/wings/common/logging/LogEvent.java | 180 +- .../wings/common/logging/LoggerHelper.java | 120 +- .../isi/wings/common/logging/LoggingKeys.java | 287 +- .../execution/engine/ExecutionFactory.java | 33 +- .../engine/api/PlanExecutionEngine.java | 45 +- .../engine/api/StepExecutionEngine.java | 40 +- .../DistributedExecutionEngine.java | 494 +- .../api/impl/local/LocalExecutionEngine.java | 650 +- .../engine/classes/ExecutionQueue.java | 197 +- .../execution/engine/classes/RuntimeInfo.java | 91 +- .../execution/engine/classes/RuntimePlan.java | 287 +- .../execution/engine/classes/RuntimeStep.java | 182 +- .../tools/ExecutionToolsFactory.java | 57 +- .../tools/api/ExecutionLoggerAPI.java | 21 +- .../tools/api/ExecutionMonitorAPI.java | 57 +- .../tools/api/ExecutionResourceAPI.java | 6 +- .../api/impl/kb/ExecutionResourceKB.java | 15 +- .../execution/tools/api/impl/kb/RunKB.java | 1644 ++--- .../main/java/edu/isi/wings/planner/Test.java | 299 +- .../planner/api/WorkflowGenerationAPI.java | 44 +- .../api/impl/kb/WorkflowGenerationKB.java | 5364 +++++++++-------- .../edu/isi/wings/planner/cli/Arguments.java | 258 +- .../java/edu/isi/wings/planner/cli/Wings.java | 1061 ++-- .../isi/wings/workflow/plan/PlanFactory.java | 59 +- .../workflow/plan/api/ExecutionPlan.java | 59 +- .../workflow/plan/api/ExecutionStep.java | 98 +- .../workflow/plan/api/impl/pplan/PPlan.java | 139 +- .../plan/api/impl/pplan/PPlanStep.java | 243 +- .../workflow/plan/classes/ExecutionCode.java | 40 +- .../workflow/plan/classes/ExecutionFile.java | 85 +- .../workflow/template/TemplateFactory.java | 278 +- .../template/api/ConstraintEngine.java | 51 +- .../isi/wings/workflow/template/api/Seed.java | 26 +- .../wings/workflow/template/api/Template.java | 177 +- .../template/api/TemplateCreationAPI.java | 68 +- .../api/impl/kb/ConstraintEngineKB.java | 795 +-- .../workflow/template/api/impl/kb/SeedKB.java | 290 +- .../api/impl/kb/TemplateCreationKB.java | 738 +-- .../template/api/impl/kb/TemplateKB.java | 4508 +++++++------- .../template/classes/ConstraintProperty.java | 123 +- .../wings/workflow/template/classes/Link.java | 183 +- .../workflow/template/classes/Metadata.java | 100 +- .../wings/workflow/template/classes/Node.java | 212 +- .../wings/workflow/template/classes/Port.java | 41 +- .../wings/workflow/template/classes/Role.java | 81 +- .../workflow/template/classes/Rules.java | 33 +- .../template/classes/sets/Binding.java | 242 +- .../sets/ComponentSetCreationRule.java | 19 +- .../template/classes/sets/PortBinding.java | 31 +- .../classes/sets/PortBindingList.java | 74 +- .../classes/sets/PortSetCreationRule.java | 39 +- .../classes/sets/PortSetRuleHandler.java | 543 +- .../classes/sets/SetCreationRule.java | 52 +- .../template/classes/sets/SetExpression.java | 182 +- .../classes/sets/TemplateBinding.java | 43 +- .../template/classes/sets/ValueBinding.java | 157 +- .../template/classes/sets/WingsSet.java | 244 +- .../classes/variables/ComponentVariable.java | 49 +- .../classes/variables/DataVariable.java | 9 +- .../classes/variables/ParameterVariable.java | 9 +- .../template/classes/variables/Variable.java | 109 +- .../classes/variables/VariableType.java | 7 +- .../wings/workflows/tests/CollectionTest.java | 60 +- .../isi/wings/workflows/tests/SimpleTest.java | 60 +- .../edu/isi/wings/workflows/util/AWGUtil.java | 166 +- .../edu/isi/wings/portal/WingsServer.java | 11 +- .../isi/wings/portal/classes/JsonHandler.java | 553 +- .../wings/portal/classes/StorageHandler.java | 326 +- .../wings/portal/classes/config/Config.java | 1876 +++--- .../portal/classes/config/ExeEngine.java | 104 +- .../portal/classes/config/PlannerConfig.java | 12 +- .../portal/classes/config/Publisher.java | 110 +- .../portal/classes/config/ServerDetails.java | 142 +- .../wings/portal/classes/domains/Domain.java | 1682 +++--- .../portal/classes/domains/DomainInfo.java | 124 +- .../portal/classes/domains/DomainLibrary.java | 61 +- .../classes/domains/DomainOntology.java | 7 +- .../portal/classes/domains/Permission.java | 11 +- .../portal/classes/domains/UrlMapPrefix.java | 37 +- .../wings/portal/classes/html/CSSLoader.java | 123 +- .../wings/portal/classes/html/JSLoader.java | 354 +- .../isi/wings/portal/classes/users/User.java | 12 +- .../wings/portal/classes/users/UsersDB.java | 109 +- .../util/ComponentExecutingThread.java | 93 +- .../portal/classes/util/CookieHandler.java | 17 +- .../classes/util/PlanningAPIBindings.java | 20 +- .../util/PlanningAndExecutingThread.java | 188 +- .../portal/classes/util/TemplateBindings.java | 6 +- .../controllers/ComponentController.java | 743 ++- .../portal/controllers/DataController.java | 1511 ++--- .../portal/controllers/DomainController.java | 611 +- .../controllers/MetaworkflowController.java | 30 +- .../portal/controllers/PlanController.java | 991 +-- .../controllers/ResourceController.java | 25 +- .../portal/controllers/RunController.java | 629 +- .../controllers/TemplateController.java | 478 +- .../portal/controllers/UserController.java | 88 +- .../filters/resources/CORSResponseFilter.java | 24 +- .../resources/GZIPReaderInterceptor.java | 7 +- .../resources/GZIPWriterInterceptor.java | 29 +- .../portal/filters/servlets/CORSFilter.java | 33 +- .../filters/servlets/CompressionFilter.java | 36 +- .../filters/servlets/GZIPResponseStream.java | 12 +- .../filters/servlets/GZIPResponseWrapper.java | 18 +- .../portal/resources/ComponentResource.java | 363 +- .../wings/portal/resources/DataResource.java | 342 +- .../portal/resources/DomainResource.java | 97 +- .../resources/MetaworkflowResource.java | 18 +- .../portal/resources/PlannerResource.java | 77 +- .../wings/portal/resources/ProvResource.java | 38 +- .../wings/portal/resources/RunResource.java | 155 +- .../portal/resources/SoftwareResource.java | 203 +- .../portal/resources/TemplateResource.java | 142 +- .../portal/resources/UploadResource.java | 99 +- .../wings/portal/resources/UserResource.java | 54 +- .../wings/portal/resources/WingsResource.java | 75 +- .../wings/portal/servlets/ExportGraph.java | 153 +- .../edu/isi/wings/portal/servlets/Login.java | 63 +- .../wings/portal/servlets/SparqlEndpoint.java | 202 +- .../isi/wings/portal/servlets/ViewConfig.java | 19 +- 193 files changed, 28992 insertions(+), 25491 deletions(-) create mode 100644 .github/workflows/prettier.yml create mode 100644 .prettierrc diff --git a/.github/workflows/prettier.yml b/.github/workflows/prettier.yml new file mode 100644 index 00000000..e333a125 --- /dev/null +++ b/.github/workflows/prettier.yml @@ -0,0 +1,26 @@ +name: Prettier + +# This action works with pull requests and pushes +on: + pull_request: + push: + branches: + - main + +jobs: + prettier: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + # Make sure the actual branch is checked out when running on pull requests + ref: ${{ github.head_ref }} + + - name: Prettify code + uses: creyD/prettier_action@v4.3 + with: + prettier_options: --write **/**/*.java + prettier_version: "2.8.8" + prettier_plugins: "prettier-plugin-java" diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/.prettierrc @@ -0,0 +1 @@ +{} diff --git a/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/component/api/impl/oodt/ComponentCreationFM.java b/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/component/api/impl/oodt/ComponentCreationFM.java index 0f9208b4..bcaae829 100644 --- a/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/component/api/impl/oodt/ComponentCreationFM.java +++ b/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/component/api/impl/oodt/ComponentCreationFM.java @@ -17,6 +17,17 @@ package edu.isi.wings.catalog.component.api.impl.oodt; +import edu.isi.kcap.ontapi.KBAPI; +import edu.isi.kcap.ontapi.OntFactory; +import edu.isi.wings.catalog.component.api.ComponentCreationAPI; +import edu.isi.wings.catalog.component.classes.Component; +import edu.isi.wings.catalog.component.classes.ComponentHolder; +import edu.isi.wings.catalog.component.classes.ComponentRole; +import edu.isi.wings.catalog.component.classes.ComponentTree; +import edu.isi.wings.catalog.component.classes.ComponentTreeNode; +import edu.isi.wings.catalog.component.classes.requirements.ComponentRequirement; +import edu.isi.wings.common.URIEntity; +import edu.isi.wings.util.oodt.CurationServiceAPI; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; @@ -28,7 +39,6 @@ import java.util.Properties; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; - import org.apache.commons.lang.StringUtils; import org.apache.oodt.cas.filemgr.structs.Element; import org.apache.oodt.cas.filemgr.structs.Product; @@ -40,19 +50,9 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.ValidationLayerException; import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.metadata.Metadata; -import edu.isi.wings.catalog.component.api.ComponentCreationAPI; -import edu.isi.wings.catalog.component.classes.Component; -import edu.isi.wings.catalog.component.classes.ComponentHolder; -import edu.isi.wings.catalog.component.classes.ComponentRole; -import edu.isi.wings.catalog.component.classes.ComponentTree; -import edu.isi.wings.catalog.component.classes.ComponentTreeNode; -import edu.isi.wings.catalog.component.classes.requirements.ComponentRequirement; -import edu.isi.wings.common.URIEntity; -import edu.isi.kcap.ontapi.KBAPI; -import edu.isi.kcap.ontapi.OntFactory; -import edu.isi.wings.util.oodt.CurationServiceAPI; public class ComponentCreationFM implements ComponentCreationAPI { + XmlRpcFileManagerClient fmclient; CurationServiceAPI curatorApi; String policy; @@ -68,8 +68,10 @@ public class ComponentCreationFM implements ComponentCreationAPI { OntFactory ontologyFactory; - private static HashMap compHolders = new HashMap(); - private static HashMap compMeta = new HashMap(); + private static HashMap compHolders = + new HashMap(); + private static HashMap compMeta = + new HashMap(); public ComponentCreationFM(Properties props) { this.fmurl = props.getProperty("oodt.fmurl"); @@ -83,16 +85,14 @@ public ComponentCreationFM(Properties props) { ontologyFactory = new OntFactory(OntFactory.JENA); File f = new File(this.archivedir); - if(!f.exists()) f.mkdirs(); + if (!f.exists()) f.mkdirs(); try { this.curatorApi = new CurationServiceAPI(this.curatorurl, this.policy); this.fmclient = new XmlRpcFileManagerClient(new URL(this.fmurl)); - } - catch (MalformedURLException e) { + } catch (MalformedURLException e) { e.printStackTrace(); - } - catch (ConnectionException e) { + } catch (ConnectionException e) { e.printStackTrace(); } } @@ -102,17 +102,17 @@ public ComponentTree getComponentHierarchy(boolean details) { String rootid = this.absurl + "#Component"; ComponentHolder rootholder = new ComponentHolder(rootid); ComponentTreeNode rootnode = new ComponentTreeNode(rootholder); - HashMap tnmap = new HashMap(); + HashMap tnmap = + new HashMap(); HashMap pmap = this.curatorApi.getParentTypeMap(); try { - for(ProductType ptype : this.fmclient.getProductTypes()) { + for (ProductType ptype : this.fmclient.getProductTypes()) { String pid = ptype.getProductTypeId(); String parentid = pmap.get(pid); - while(parentid != null && !parentid.equals(rootid)) { + while (parentid != null && !parentid.equals(rootid)) { parentid = pmap.get(parentid); } - if(parentid == null) - continue; + if (parentid == null) continue; ComponentHolder pholder = new ComponentHolder(pid); ComponentTreeNode pnode = new ComponentTreeNode(pholder); for (Product p : this.fmclient.getProductsByProductType(ptype)) { @@ -121,18 +121,17 @@ public ComponentTree getComponentHierarchy(boolean details) { tnmap.put(ptype.getProductTypeId(), pnode); rootnode.addChild(pnode); } - for(String childid: pmap.keySet()) { + for (String childid : pmap.keySet()) { String parentid = pmap.get(childid); ComponentTreeNode ptn = tnmap.get(parentid); ComponentTreeNode ctn = tnmap.get(childid); - if(ptn != null && ctn != null) { + if (ptn != null && ctn != null) { rootnode.removeChild(ctn); ptn.addChild(ctn); } } return new ComponentTree(rootnode); - } - catch (CatalogException e) { + } catch (CatalogException e) { e.printStackTrace(); } catch (RepositoryManagerException e) { e.printStackTrace(); @@ -145,20 +144,22 @@ public Component getComponent(String cid, boolean details) { try { Product prod = this.fmclient.getProductById(cid); Metadata meta = this.fmclient.getMetadata(prod); - int comptype = Component.ABSTRACT; + int comptype = Component.ABSTRACT; String conc = meta.getMetadata("IsConcrete"); - if(conc != null && conc.equals("true")) - comptype = Component.CONCRETE; + if (conc != null && conc.equals("true")) comptype = Component.CONCRETE; Component comp = new Component(cid, comptype); - for(String input : meta.getAllMetadata("Inputs")) - comp.addInput(this.getRoleFromString(input)); - for(String output : meta.getAllMetadata("Outputs")) - comp.addOutput(this.getRoleFromString(output)); + for (String input : meta.getAllMetadata("Inputs")) comp.addInput( + this.getRoleFromString(input) + ); + for (String output : meta.getAllMetadata("Outputs")) comp.addOutput( + this.getRoleFromString(output) + ); comp.setDocumentation(meta.getMetadata("Documentation")); comp.setRules(ontologyFactory.parseRules(meta.getMetadata("Rules"))); - comp.setComponentRequirement(this.getRequirementFromString( - meta.getMetadata("Requirement"))); - String loc = this.getComponentLocation(cid); + comp.setComponentRequirement( + this.getRequirementFromString(meta.getMetadata("Requirement")) + ); + String loc = this.getComponentLocation(cid); comp.setLocation(loc); return comp; } catch (CatalogException e) { @@ -171,7 +172,7 @@ public Component getComponent(String cid, boolean details) { public String getComponentLocation(String cid) { try { Product prod = this.fmclient.getProductById(cid); - for(Reference ref : this.fmclient.getProductReferences(prod)) { + for (Reference ref : this.fmclient.getProductReferences(prod)) { return ref.getDataStoreReference(); } } catch (CatalogException e) { @@ -196,21 +197,31 @@ public boolean addComponent(Component comp, String pholderid) { Metadata meta = new Metadata(); ArrayList inputs = new ArrayList(); ArrayList outputs = new ArrayList(); - for(ComponentRole role : comp.getInputs()) - inputs.add(this.getRoleString(role)); - for(ComponentRole role : comp.getOutputs()) - outputs.add(this.getRoleString(role)); + for (ComponentRole role : comp.getInputs()) inputs.add( + this.getRoleString(role) + ); + for (ComponentRole role : comp.getOutputs()) outputs.add( + this.getRoleString(role) + ); meta.addMetadata("Inputs", inputs); meta.addMetadata("Outputs", outputs); - meta.addMetadata("IsConcrete", - (comp.getType() == Component.CONCRETE) ? "true" : "false"); - if(comp.getDocumentation() != null) - meta.addMetadata("Documentation", comp.getDocumentation()); - if(comp.getRules() != null) - meta.addMetadata("Rules", StringUtils.join(comp.getRules(), "\n")); + meta.addMetadata( + "IsConcrete", + (comp.getType() == Component.CONCRETE) ? "true" : "false" + ); + if (comp.getDocumentation() != null) meta.addMetadata( + "Documentation", + comp.getDocumentation() + ); + if (comp.getRules() != null) meta.addMetadata( + "Rules", + StringUtils.join(comp.getRules(), "\n") + ); ComponentRequirement compreq = comp.getComponentRequirement(); - if(compreq != null) - meta.addMetadata("Requirement", this.getRequirementString(compreq)); + if (compreq != null) meta.addMetadata( + "Requirement", + this.getRequirementString(compreq) + ); compHolders.put(comp.getID(), pholderid); compMeta.put(comp.getID(), meta); @@ -220,14 +231,16 @@ public boolean addComponent(Component comp, String pholderid) { @Override public boolean addComponentHolder(String holderid, String pholderid) { String holdername = new URIEntity(holderid).getName(); - String desc = "A product type for "+holdername+" component class"; + String desc = "A product type for " + holdername + " component class"; String ver = "org.apache.oodt.cas.filemgr.versioning.BasicVersioner"; - String repo = "file://"+this.archivedir; + String repo = "file://" + this.archivedir; ProductType type = new ProductType(holderid, holderid, desc, repo, ver); try { this.fmclient.addProductType(type); - if(pholderid != null) - return this.curatorApi.addParentForProductType(type, pholderid); + if (pholderid != null) return this.curatorApi.addParentForProductType( + type, + pholderid + ); } catch (RepositoryManagerException e) { e.printStackTrace(); } @@ -236,14 +249,14 @@ public boolean addComponentHolder(String holderid, String pholderid) { @Override public boolean updateComponent(Component comp) { - if(comp == null) return false; - + if (comp == null) return false; + String locuri = this.getComponentLocation(comp.getID()); // Remove existing component assertions and re-add the new component details boolean ok1 = this.removeComponent(comp.getID(), false, false); boolean ok2 = this.addComponent(comp, null); this.setComponentLocation(comp.getID(), locuri); - + // TODO: If abstract, update all components defined in all libraries ! return ok1 && ok2; } @@ -255,8 +268,11 @@ public boolean renameComponent(String oldid, String newid) { } @Override - public boolean removeComponent(String cid, boolean remove_holder, - boolean unlink) { + public boolean removeComponent( + String cid, + boolean remove_holder, + boolean unlink + ) { try { Product prod = this.fmclient.getProductById(cid); this.fmclient.removeProduct(prod); @@ -271,7 +287,7 @@ public boolean removeComponent(String cid, boolean remove_holder, public boolean removeComponentHolder(String holderid) { try { ProductType type = this.fmclient.getProductTypeById(holderid); - for(Product prod : this.fmclient.getProductsByProductType(type)) { + for (Product prod : this.fmclient.getProductsByProductType(type)) { this.fmclient.removeProduct(prod); } return this.curatorApi.removeProductType(type); @@ -285,25 +301,21 @@ public boolean removeComponentHolder(String holderid) { @Override public boolean setComponentLocation(String cid, String locuri) { - if(locuri == null) { + if (locuri == null) { try { File f = File.createTempFile("dummy-", "-abstract"); locuri = f.getAbsolutePath(); - } - catch (IOException e) { + } catch (IOException e) { return false; } } File locf = new File(locuri); // Zip component directory (if it is a directory) - if(locf.isDirectory()) - locf = this.zipDirectory(locf); - if(locf == null || !locf.exists()) - return false; + if (locf.isDirectory()) locf = this.zipDirectory(locf); + if (locf == null || !locf.exists()) return false; String holderid = compHolders.get(cid); - if(holderid == null) - return false; + if (holderid == null) return false; ProductType type; try { @@ -318,8 +330,13 @@ public boolean setComponentLocation(String cid, String locuri) { long filesize = locf.length(); refs.add(new Reference(locf.toURI().toString(), "", filesize)); - Product prod = new Product(compname, type, - Product.STRUCTURE_FLAT, Product.STATUS_TRANSFER, refs); + Product prod = new Product( + compname, + type, + Product.STRUCTURE_FLAT, + Product.STATUS_TRANSFER, + refs + ); prod.setProductId(cid); try { Metadata meta = compMeta.get(cid); @@ -359,12 +376,15 @@ public void setExternalCatalog(ComponentCreationAPI dc) { /* * Private functions */ - private void initialSync(ComponentCreationAPI cc, ComponentTreeNode node, - ComponentTreeNode parent) { + private void initialSync( + ComponentCreationAPI cc, + ComponentTreeNode node, + ComponentTreeNode parent + ) { // Replace top DataObject with local namespaced DataObject String holderid = node.getCls().getID(); - if(holderid.equals(this.ccurl + "#Component")) - holderid = this.absurl + "#Component"; + if (holderid.equals(this.ccurl + "#Component")) holderid = + this.absurl + "#Component"; ProductType ptype = null; try { @@ -375,30 +395,36 @@ private void initialSync(ComponentCreationAPI cc, ComponentTreeNode node, if (ptype == null) { String pholderid = parent != null ? parent.getCls().getID() : null; - if(pholderid != null && pholderid.equals(this.ccurl + "#Component")) - pholderid = this.absurl + "#Component"; + if ( + pholderid != null && pholderid.equals(this.ccurl + "#Component") + ) pholderid = this.absurl + "#Component"; this.addComponentHolder(holderid, pholderid); this.save(); - if(holderid.equals(this.absurl + "#Component")) { + if (holderid.equals(this.absurl + "#Component")) { try { ptype = this.fmclient.getProductTypeById(holderid); } catch (RepositoryManagerException e1) { e1.printStackTrace(); } - if(ptype == null) - return; - - String[] props = new String[] {"Inputs", "Outputs", "IsConcrete", - "Documentation", "Requirement", "Rules"}; + if (ptype == null) return; + + String[] props = new String[] { + "Inputs", + "Outputs", + "IsConcrete", + "Documentation", + "Requirement", + "Rules", + }; // MD5 ? (Also add it for DataCreationFM) ArrayList elementList = new ArrayList(); - for(String prop: props) { + for (String prop : props) { Element el = null; try { el = this.fmclient.getElementById(prop); } catch (ValidationLayerException e) { - el = new Element(prop, prop, "", "", "Element "+prop, ""); + el = new Element(prop, prop, "", "", "Element " + prop, ""); elementList.add(el); } } @@ -428,21 +454,27 @@ private void initialSync(ComponentCreationAPI cc, ComponentTreeNode node, private String getRoleString(ComponentRole role) { String sep = "|"; - return role.getID() + sep + - role.getRoleName() + sep + - role.getPrefix() + sep + - role.getType() + sep + - role.getDimensionality() + sep + - role.isParam() + sep + - role.getParamDefaultalue(); + return ( + role.getID() + + sep + + role.getRoleName() + + sep + + role.getPrefix() + + sep + + role.getType() + + sep + + role.getDimensionality() + + sep + + role.isParam() + + sep + + role.getParamDefaultalue() + ); } private ComponentRole getRoleFromString(String str) { - if(str == null) - return null; + if (str == null) return null; String[] vals = str.split("\\|"); - if(vals.length < 7) - return null; + if (vals.length < 7) return null; ComponentRole role = new ComponentRole(vals[0]); role.setRoleName(vals[1]); role.setPrefix(vals[2]); @@ -455,25 +487,28 @@ private ComponentRole getRoleFromString(String str) { private String getRequirementString(ComponentRequirement req) { String sep = "|"; - return req.getMemoryGB() + sep + - req.getStorageGB() + sep + - req.isNeed64bit() + sep + - StringUtils.join(req.getSoftwareIds(), ","); + return ( + req.getMemoryGB() + + sep + + req.getStorageGB() + + sep + + req.isNeed64bit() + + sep + + StringUtils.join(req.getSoftwareIds(), ",") + ); } private ComponentRequirement getRequirementFromString(String str) { - if(str == null) - return null; + if (str == null) return null; String[] vals = str.split("\\|"); - if(vals.length < 3) - return null; + if (vals.length < 3) return null; ComponentRequirement req = new ComponentRequirement(); req.setMemoryGB(Float.parseFloat(vals[0])); req.setStorageGB(Float.parseFloat(vals[1])); req.setNeed64bit(Boolean.parseBoolean(vals[2])); - if(vals.length > 3) - for(String softwareId: vals[3].split(",")) - req.addSoftwareId(softwareId); + if (vals.length > 3) for (String softwareId : vals[3].split( + "," + )) req.addSoftwareId(softwareId); return req; } @@ -485,8 +520,7 @@ private File zipDirectory(File srcFile) { this.addDirToArchive(zos, srcFile); zos.close(); return zipFile; - } - catch (IOException ioe) { + } catch (IOException ioe) { ioe.printStackTrace(); } return null; @@ -554,7 +588,7 @@ public boolean start_write() { @Override public void stop_batch_operation() { // TODO Auto-generated method stub - + } @Override @@ -568,5 +602,3 @@ public boolean delete() { return false; } } - - diff --git a/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/data/api/impl/oodt/DataCreationFM.java b/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/data/api/impl/oodt/DataCreationFM.java index 50af3195..712f3f65 100644 --- a/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/data/api/impl/oodt/DataCreationFM.java +++ b/engines/oodt/adapter/src/main/java/edu/isi/wings/catalog/data/api/impl/oodt/DataCreationFM.java @@ -17,13 +17,22 @@ package edu.isi.wings.catalog.data.api.impl.oodt; +import edu.isi.kcap.ontapi.KBAPI; +import edu.isi.wings.catalog.data.api.DataCreationAPI; +import edu.isi.wings.catalog.data.classes.DataItem; +import edu.isi.wings.catalog.data.classes.DataTree; +import edu.isi.wings.catalog.data.classes.DataTreeNode; +import edu.isi.wings.catalog.data.classes.MetadataProperty; +import edu.isi.wings.catalog.data.classes.MetadataValue; +import edu.isi.wings.common.URIEntity; +import edu.isi.wings.common.kb.KBUtils; +import edu.isi.wings.util.oodt.CurationServiceAPI; import java.io.File; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.HashMap; import java.util.Properties; - import org.apache.oodt.cas.filemgr.structs.Element; import org.apache.oodt.cas.filemgr.structs.Product; import org.apache.oodt.cas.filemgr.structs.ProductType; @@ -35,685 +44,731 @@ import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.metadata.Metadata; -import edu.isi.kcap.ontapi.KBAPI; -import edu.isi.wings.catalog.data.api.DataCreationAPI; -import edu.isi.wings.catalog.data.classes.DataItem; -import edu.isi.wings.catalog.data.classes.DataTree; -import edu.isi.wings.catalog.data.classes.DataTreeNode; -import edu.isi.wings.catalog.data.classes.MetadataProperty; -import edu.isi.wings.catalog.data.classes.MetadataValue; -import edu.isi.wings.common.URIEntity; -import edu.isi.wings.common.kb.KBUtils; -import edu.isi.wings.util.oodt.CurationServiceAPI; - public class DataCreationFM implements DataCreationAPI { - XmlRpcFileManagerClient fmclient; - CurationServiceAPI curatorApi; - String policy; - - String fmurl; - String curatorurl; - - String archivedir; - - String dcurl; - String liburl; - String onturl; - - private static HashMap prodTypes = new HashMap(); - - public DataCreationFM(Properties props) { - this.fmurl = props.getProperty("oodt.fmurl"); - this.curatorurl = props.getProperty("oodt.curatorurl"); - this.archivedir = props.getProperty("oodt.archivedir"); - this.policy = props.getProperty("oodt.fmpolicy"); - - this.dcurl = props.getProperty("ont.data.url"); - this.liburl = props.getProperty("lib.domain.data.url"); - this.onturl = props.getProperty("ont.domain.data.url"); - - File f = new File(this.archivedir); - if(!f.exists()) f.mkdirs(); - - try { - this.curatorApi = new CurationServiceAPI(this.curatorurl, this.policy); - this.fmclient = new XmlRpcFileManagerClient(new URL(this.fmurl)); - } - catch (MalformedURLException e) { - e.printStackTrace(); - } - catch (ConnectionException e) { - e.printStackTrace(); - } - } - - @Override - public DataTree getDataHierarchy() { - return this.getDataHieararchy(false); - } - - private DataTree getDataHieararchy(boolean typesonly) { - DataItem rootitem = new DataItem("FileManager", DataItem.DATATYPE); - DataTreeNode rootnode = new DataTreeNode(rootitem); - HashMap tnmap = new HashMap(); - HashMap pmap = this.curatorApi.getParentTypeMap(); - try { - for(ProductType ptype : this.fmclient.getProductTypes()) { + + XmlRpcFileManagerClient fmclient; + CurationServiceAPI curatorApi; + String policy; + + String fmurl; + String curatorurl; + + String archivedir; + + String dcurl; + String liburl; + String onturl; + + private static HashMap prodTypes = + new HashMap(); + + public DataCreationFM(Properties props) { + this.fmurl = props.getProperty("oodt.fmurl"); + this.curatorurl = props.getProperty("oodt.curatorurl"); + this.archivedir = props.getProperty("oodt.archivedir"); + this.policy = props.getProperty("oodt.fmpolicy"); + + this.dcurl = props.getProperty("ont.data.url"); + this.liburl = props.getProperty("lib.domain.data.url"); + this.onturl = props.getProperty("ont.domain.data.url"); + + File f = new File(this.archivedir); + if (!f.exists()) f.mkdirs(); + + try { + this.curatorApi = new CurationServiceAPI(this.curatorurl, this.policy); + this.fmclient = new XmlRpcFileManagerClient(new URL(this.fmurl)); + } catch (MalformedURLException e) { + e.printStackTrace(); + } catch (ConnectionException e) { + e.printStackTrace(); + } + } + + @Override + public DataTree getDataHierarchy() { + return this.getDataHieararchy(false); + } + + private DataTree getDataHieararchy(boolean typesonly) { + DataItem rootitem = new DataItem("FileManager", DataItem.DATATYPE); + DataTreeNode rootnode = new DataTreeNode(rootitem); + HashMap tnmap = new HashMap(); + HashMap pmap = this.curatorApi.getParentTypeMap(); + try { + for (ProductType ptype : this.fmclient.getProductTypes()) { String pid = ptype.getProductTypeId(); String parentid = pmap.get(pid); - while(parentid != null && !parentid.endsWith("#DataObject")) { + while (parentid != null && !parentid.endsWith("#DataObject")) { parentid = pmap.get(parentid); } - if(parentid == null && !pid.endsWith("#DataObject")) - continue; - DataItem ptitem = new DataItem(pid, DataItem.DATATYPE); - DataTreeNode ptitemnode = new DataTreeNode(ptitem); - for (Product p : this.fmclient.getProductsByProductType(ptype)) { - DataItem pitem = new DataItem(p.getProductId(), DataItem.DATA); - DataTreeNode pitemnode = new DataTreeNode(pitem); - ptitemnode.addChild(pitemnode); - } - tnmap.put(ptype.getProductTypeId(), ptitemnode); - rootnode.addChild(ptitemnode); - } - for(String childid: pmap.keySet()) { - String parentid = pmap.get(childid); - DataTreeNode ptn = tnmap.get(parentid); - DataTreeNode ctn = tnmap.get(childid); - if(ptn != null && ctn != null) { - rootnode.removeChild(ctn); - ptn.addChild(ctn); - } - } - return new DataTree(rootnode); - } - catch (CatalogException e) { - e.printStackTrace(); - } catch (RepositoryManagerException e) { - e.printStackTrace(); - } - return null; - } - - @Override - public DataTree getDatatypeHierarchy() { - return this.getDataHieararchy(true); - } - - @Override - public DataTree getMetricsHierarchy() { - // TODO Auto-generated method stub - return null; - } - - @Override - public ArrayList getAllDatatypeIds() { - ArrayList ids = new ArrayList(); - try { - for(ProductType ptype : this.fmclient.getProductTypes()) { - for (Product p : this.fmclient.getProductsByProductType(ptype)) { - ids.add(p.getProductId()); - } - } - } - catch (CatalogException e) { - e.printStackTrace(); - } catch (RepositoryManagerException e) { - e.printStackTrace(); - } - return ids; - } - - @Override - public MetadataProperty getMetadataProperty(String propid) { - try { - Element el = this.fmclient.getElementById(propid); - MetadataProperty prop = new MetadataProperty(el.getElementId(), MetadataProperty.DATATYPE); - for(String domid : this.curatorApi.getProductTypeIdsHavingElement(el)) { - prop.addDomain(domid); - } - return prop; - } catch (ValidationLayerException e) { - e.printStackTrace(); - } - return null; - } - - @Override - public ArrayList getMetadataProperties(String dtypeid, boolean direct) { - ArrayList props = new ArrayList(); - try { - HashMap pmap = this.curatorApi.getParentTypeMap(); - String tmptypeid = dtypeid; - while(true) { - ProductType type = this.fmclient.getProductTypeById(tmptypeid); - for(Element el : this.curatorApi.getElementsForProductType(type, true)) { - MetadataProperty prop = - new MetadataProperty(el.getElementId(), MetadataProperty.DATATYPE); - prop.addDomain(type.getProductTypeId()); - prop.setRange(KBUtils.XSD+"string"); - props.add(prop); - } - if(pmap.containsKey(tmptypeid)) - tmptypeid = pmap.get(tmptypeid); - else - break; - if(tmptypeid == null || tmptypeid.equals("")) - break; - } - } catch (RepositoryManagerException e) { - e.printStackTrace(); - } - return props; - } - - @Override - public ArrayList getAllMetadataProperties() { - ArrayList list = new ArrayList(); - HashMap elmap = new HashMap(); - try { - for (ProductType type : this.fmclient.getProductTypes()) { - for (Element el : this.curatorApi.getElementsForProductType(type, true)) { - if (elmap.containsKey(el.getElementId())) - continue; - elmap.put(el.getElementId(), true); - MetadataProperty prop = new MetadataProperty(el.getElementId(), - MetadataProperty.DATATYPE); - prop.addDomain(type.getProductTypeId()); - prop.setRange(KBUtils.XSD + "string"); - list.add(prop); - } - - } - } catch (RepositoryManagerException e) { - e.printStackTrace(); - } - return list; - } - - @Override - public DataItem getDatatypeForData(String dataid) { - try { - Product prod = this.fmclient.getProductById(dataid); - return new DataItem(prod.getProductType().getProductTypeId(), DataItem.DATATYPE); - } catch (CatalogException e) { - e.printStackTrace(); - } - return null; - } - - @Override - public ArrayList getDataForDatatype(String dtypeid, boolean direct) { - ArrayList ditems = new ArrayList(); - try { - for(Product prod : - this.fmclient.getProductsByProductType(this.fmclient.getProductTypeById(dtypeid))) { - ditems.add(new DataItem(prod.getProductType().getProductTypeId(), DataItem.DATATYPE)); - } - } catch (CatalogException e) { - e.printStackTrace(); - } catch (RepositoryManagerException e) { - e.printStackTrace(); - } - return ditems; - } - - @Override - public String getTypeNameFormat(String dtypeid) { - // TODO Not Supported - return null; - } - - @Override - public String getDataLocation(String dataid) { - try { - Product prod = this.fmclient.getProductById(dataid); - for(Reference ref : this.fmclient.getProductReferences(prod)) { - return ref.getDataStoreReference(); - } - } catch (CatalogException e) { - e.printStackTrace(); - } - return null; - } - - @Override - public ArrayList getMetadataValues(String dataid, ArrayList propids) { - ArrayList vals = new ArrayList(); - try { - Product prod = this.fmclient.getProductById(dataid); - Metadata meta = this.fmclient.getMetadata(prod); - for(String propid : propids) { - Element el = this.fmclient.getElementById(propid); - MetadataValue val = new MetadataValue(propid, - meta.getMetadata(el.getElementName()), MetadataValue.DATATYPE); - vals.add(val); - } - } catch (CatalogException e) { - e.printStackTrace(); - } catch (ValidationLayerException e) { - e.printStackTrace(); - } - return vals; - } - - @Override - public boolean addDatatype(String dtypeid, String parentid) { - String dtypename = new URIEntity(dtypeid).getName(); - String desc = "A product type for "+dtypename+" files"; - String ver = "org.apache.oodt.cas.filemgr.versioning.BasicVersioner"; - String repo = "file://"+this.archivedir; - ProductType type = new ProductType(dtypeid, dtypeid, desc, repo, ver); - try { - this.fmclient.addProductType(type); - if(parentid != null) - return this.curatorApi.addParentForProductType(type, parentid); - } catch (RepositoryManagerException e) { - e.printStackTrace(); - } - return false; - } - - @Override - public boolean removeDatatype(String dtypeid) { - try { - ProductType type = this.fmclient.getProductTypeById(dtypeid); - for(Product prod : this.fmclient.getProductsByProductType(type)) { - this.fmclient.removeProduct(prod); - } - return this.curatorApi.removeProductType(type); - } catch (RepositoryManagerException e) { - e.printStackTrace(); - } catch (CatalogException e) { - e.printStackTrace(); - } - return false; - } - - @Override - public boolean renameDatatype(String newtypeid, String oldtypeid) { - try { - HashMap pmap = this.curatorApi.getParentTypeMap(); - this.addDatatype(newtypeid, - pmap.containsKey(oldtypeid) ? pmap.get(oldtypeid) : null); - this.save(); - - ProductType otype = this.fmclient.getProductTypeById(oldtypeid); - ProductType ntype = this.fmclient.getProductTypeById(newtypeid); - for(Product prod : this.fmclient.getProductsByProductType(otype)) { - prod.setProductType(ntype); - this.fmclient.modifyProduct(prod); - } - boolean ok1 = this.curatorApi.addElementsForProductType(ntype, - this.curatorApi.getElementsForProductType(otype, true)); - boolean ok2 = this.curatorApi.removeAllElementsForProductType(otype); - boolean ok3 = this.curatorApi.removeProductType(otype); - return ok1 && ok2 && ok3; - } catch (RepositoryManagerException e) { - e.printStackTrace(); - } catch (CatalogException e) { - e.printStackTrace(); - } - return false; - } - - @Override - public boolean moveDatatypeParent(String dtypeid, String fromtypeid, String totypeid) { - // Not currently supported - return false; - } - - @Override - public boolean moveDataParent(String arg0, String arg1, String arg2) { - // Not currently supported - return false; - } - - @Override - public boolean addData(String dataid, String dtypeid) { - prodTypes.put(dataid, dtypeid); - return true; - } - - @Override - public boolean renameData(String newdataid, String olddataid) { - // TODO Not Implemented - return false; - } - - @Override - public boolean removeData(String dataid) { - try { - Product prod = this.fmclient.getProductById(dataid); - this.fmclient.removeProduct(prod); - return true; - } catch (CatalogException e) { - e.printStackTrace(); - } - return false; - } - - @Override - public boolean setDataLocation(String dataid, String locuri) { - String dtypeid = prodTypes.get(dataid); - if(dtypeid == null) - return false; - if(locuri == null) - return false; - - ProductType type; - try { - type = this.fmclient.getProductTypeById(dtypeid); - } catch (RepositoryManagerException e1) { - e1.printStackTrace(); - return false; - } - String dataname = new URIEntity(dataid).getName(); - - ArrayList refs = new ArrayList(); - File locf = new File(locuri); - if(!locf.exists()) - return false; - - long filesize = locf.length(); - refs.add(new Reference(locf.toURI().toString(), "", filesize)); - - Product prod = new Product(dataname, type, - Product.STRUCTURE_FLAT, Product.STATUS_TRANSFER, refs); - prod.setProductId(dataid); - try { - this.fmclient.ingestProduct(prod, new Metadata(), false); - return true; - } catch (Exception e) { - e.printStackTrace(); - } - return false; - } - - @Override - public boolean setTypeNameFormat(String dtypeid, String format) { - // TODO Not Supported - return false; - } - - @Override - public boolean addObjectPropertyValue(String dataid, String propid, String valid) { - return this.addDatatypePropertyValue(dataid, propid, valid); - } - - @Override - public boolean addDatatypePropertyValue(String dataid, String propid, Object val) { - return this.addDatatypePropertyValue(dataid, propid, val.toString(), null); - } - - @Override - public boolean addDatatypePropertyValue(String dataid, String propid, String val, String xsdtype) { - try { - Product prod = this.fmclient.getProductById(dataid); - if(prod == null) - return false; - Element el = this.fmclient.getElementById(propid); - if(el == null) - return false; - Metadata meta = this.fmclient.getMetadata(prod); - meta.removeMetadata(el.getElementName()); - meta.addMetadata(el.getElementName(), val); - this.fmclient.updateMetadata(prod, meta); - return true; - } - catch (ValidationLayerException e) { - e.printStackTrace(); - } catch (CatalogException e) { - e.printStackTrace(); - } - return false; - } - - @Override - public boolean removePropertyValue(String dataid, String propid, Object val) { - try { - Product prod = this.fmclient.getProductById(dataid); - if(prod == null) - return false; - Metadata newm = new Metadata(); - Metadata meta = this.fmclient.getMetadata(prod); - for(String mpropid : meta.getAllKeys()) { - if(mpropid.equals(propid)) - continue; - newm.addMetadata(mpropid, meta.getMetadata(mpropid)); - } - this.fmclient.updateMetadata(prod, newm); - return true; - } catch (CatalogException e) { - e.printStackTrace(); - } - return false; - } - - @Override - public boolean removeAllPropertyValues(String dataid, ArrayList propids) { - try { - Product prod = this.fmclient.getProductById(dataid); - if(prod == null) - return false; - Metadata newm = new Metadata(); - this.fmclient.updateMetadata(prod, newm); - return true; - } catch (CatalogException e) { - e.printStackTrace(); - } - return false; - } - - @Override - public boolean addMetadataProperty(String propid, String domain, String range) { - if(domain == null) - return false; - try { - ProductType type = this.fmclient.getProductTypeById(domain); - String propname = new URIEntity(propid).getName(); - Element el = new Element(propid, propname, "", "", "Element "+propname, ""); - // FIXME: Range ignored for now - ArrayList elementList = new ArrayList(); - elementList.add(el); - return this.curatorApi.addElementsForProductType(type, elementList); - } - catch (RepositoryManagerException e) { - e.printStackTrace(); - } - return false; - } - - @Override - public boolean addMetadataPropertyDomain(String propid, String domain) { - try { - Element el = this.fmclient.getElementById(propid); - if(el == null) - return false; - ProductType type = this.fmclient.getProductTypeById(domain); - if(type == null) - return false; - ArrayList elist = new ArrayList(); - elist.add(el); - this.curatorApi.addElementsForProductType(type, elist); - return true; - } - catch (RepositoryManagerException e) { - e.printStackTrace(); - } - catch (ValidationLayerException e) { - e.printStackTrace(); - } - return false; - } - - @Override - public boolean removeMetadataProperty(String propid) { - try { - Element el = this.fmclient.getElementById(propid); - if(el == null) - return false; - ArrayList elist = new ArrayList(); - elist.add(el); - for(ProductType type : this.fmclient.getProductTypes()) { - this.curatorApi.removeElementsForProductType(type, elist); - } - return true; - } - catch (ValidationLayerException e) { - e.printStackTrace(); - } catch (RepositoryManagerException e) { - e.printStackTrace(); - } - return false; - } - - @Override - public boolean removeMetadataPropertyDomain(String propid, String domain) { - try { - Element el = this.fmclient.getElementById(propid); - if(el == null) - return false; - ProductType type = this.fmclient.getProductTypeById(domain); - if(type == null) - return false; - ArrayList elist = new ArrayList(); - elist.add(el); - return this.curatorApi.removeElementsForProductType(type, elist); - } - catch (RepositoryManagerException e) { - e.printStackTrace(); - } - catch (ValidationLayerException e) { - e.printStackTrace(); - } - return false; - } - - @Override - public boolean renameMetadataProperty(String oldid, String newid) { - // TODO NOT implemented - return false; - } - - @Override - public boolean save() { - this.fmclient.refreshConfigAndPolicy(); - return true; - } - - @Override - public void copyFrom(DataCreationAPI dc) { - DataTree tree = dc.getDataHierarchy(); - DataTreeNode root = tree.getRoot(); - this.initialSync(dc, root, null); - this.save(); - } - - @Override - public DataCreationAPI getExternalCatalog() { - // TODO NOT implemented - return null; - } - - @Override - public void setExternalCatalog(DataCreationAPI dc) { - // TODO NOT implemented - - } - - /* - * Private functions - */ - private void initialSync(DataCreationAPI dc, DataTreeNode node, DataTreeNode parent) { - // Replace top DataObject with local namespaced DataObject - String dtypeid = node.getItem().getID(); - if(dtypeid.equals(this.dcurl + "#DataObject")) - dtypeid = this.onturl + "#DataObject"; - - ProductType ptype = null; - try { - ptype = this.fmclient.getProductTypeById(dtypeid); - } catch (RepositoryManagerException e) { - ptype = null; - } - - if (ptype == null) { - String pdtypeid = parent != null ? parent.getItem().getID() : null; - if(pdtypeid != null && pdtypeid.equals(this.dcurl + "#DataObject")) - pdtypeid = this.onturl + "#DataObject"; - this.addDatatype(dtypeid, pdtypeid); - this.save(); - - ArrayList props = new ArrayList(); - HashMap propElements = new HashMap(); - for (MetadataProperty prop : dc.getMetadataProperties(node.getItem().getID(), false)) { - props.add(prop.getID()); - Element el; - try { - el = this.fmclient.getElementById(prop.getID()); - } catch (ValidationLayerException e) { - this.addMetadataProperty(prop.getID(), dtypeid, null); - this.save(); - try { - el = this.fmclient.getElementById(prop.getID()); - } catch (ValidationLayerException e1) { - el = null; - } - } - if(el != null) - propElements.put(prop.getID(), el); - } - for (DataTreeNode cnode : node.getChildren()) { - if (cnode.getItem().getType() == DataItem.DATA) { - String dataid = cnode.getItem().getID(); - Product prod; - try { - prod = this.fmclient.getProductById(dataid); - } catch (CatalogException e) { - prod = null; - } - if (prod == null) { - this.addData(dataid, dtypeid); - this.setDataLocation(dataid, dc.getDataLocation(dataid)); - this.addMetadataValues(dataid, dc.getMetadataValues(dataid, props)); - } - } else if (cnode.getItem().getType() == DataItem.DATATYPE) { - this.initialSync(dc, cnode, node); - } - } - } - else { - for (DataTreeNode cnode : node.getChildren()) { - if (cnode.getItem().getType() == DataItem.DATATYPE) { - this.initialSync(dc, cnode, node); - } - } - } - } - private boolean addMetadataValues(String dataid, ArrayList mvalues) { - try { - Product prod; - try { - prod = this.fmclient.getProductById(dataid); - } catch (CatalogException e) { - prod = null; - } - if(prod == null) - return false; - - Metadata meta = new Metadata(); - for(MetadataValue mval : mvalues) { - String propid = mval.getPropertyId(); - meta.addMetadata(propid, mval.getValue().toString()); - } - this.fmclient.addMetadata(prod, meta); - return true; - } catch (CatalogException e) { - e.printStackTrace(); - } - return false; - } + if (parentid == null && !pid.endsWith("#DataObject")) continue; + DataItem ptitem = new DataItem(pid, DataItem.DATATYPE); + DataTreeNode ptitemnode = new DataTreeNode(ptitem); + for (Product p : this.fmclient.getProductsByProductType(ptype)) { + DataItem pitem = new DataItem(p.getProductId(), DataItem.DATA); + DataTreeNode pitemnode = new DataTreeNode(pitem); + ptitemnode.addChild(pitemnode); + } + tnmap.put(ptype.getProductTypeId(), ptitemnode); + rootnode.addChild(ptitemnode); + } + for (String childid : pmap.keySet()) { + String parentid = pmap.get(childid); + DataTreeNode ptn = tnmap.get(parentid); + DataTreeNode ctn = tnmap.get(childid); + if (ptn != null && ctn != null) { + rootnode.removeChild(ctn); + ptn.addChild(ctn); + } + } + return new DataTree(rootnode); + } catch (CatalogException e) { + e.printStackTrace(); + } catch (RepositoryManagerException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public DataTree getDatatypeHierarchy() { + return this.getDataHieararchy(true); + } + + @Override + public DataTree getMetricsHierarchy() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ArrayList getAllDatatypeIds() { + ArrayList ids = new ArrayList(); + try { + for (ProductType ptype : this.fmclient.getProductTypes()) { + for (Product p : this.fmclient.getProductsByProductType(ptype)) { + ids.add(p.getProductId()); + } + } + } catch (CatalogException e) { + e.printStackTrace(); + } catch (RepositoryManagerException e) { + e.printStackTrace(); + } + return ids; + } + + @Override + public MetadataProperty getMetadataProperty(String propid) { + try { + Element el = this.fmclient.getElementById(propid); + MetadataProperty prop = new MetadataProperty( + el.getElementId(), + MetadataProperty.DATATYPE + ); + for (String domid : this.curatorApi.getProductTypeIdsHavingElement(el)) { + prop.addDomain(domid); + } + return prop; + } catch (ValidationLayerException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public ArrayList getMetadataProperties( + String dtypeid, + boolean direct + ) { + ArrayList props = new ArrayList(); + try { + HashMap pmap = this.curatorApi.getParentTypeMap(); + String tmptypeid = dtypeid; + while (true) { + ProductType type = this.fmclient.getProductTypeById(tmptypeid); + for (Element el : this.curatorApi.getElementsForProductType( + type, + true + )) { + MetadataProperty prop = new MetadataProperty( + el.getElementId(), + MetadataProperty.DATATYPE + ); + prop.addDomain(type.getProductTypeId()); + prop.setRange(KBUtils.XSD + "string"); + props.add(prop); + } + if (pmap.containsKey(tmptypeid)) tmptypeid = + pmap.get(tmptypeid); else break; + if (tmptypeid == null || tmptypeid.equals("")) break; + } + } catch (RepositoryManagerException e) { + e.printStackTrace(); + } + return props; + } + + @Override + public ArrayList getAllMetadataProperties() { + ArrayList list = new ArrayList(); + HashMap elmap = new HashMap(); + try { + for (ProductType type : this.fmclient.getProductTypes()) { + for (Element el : this.curatorApi.getElementsForProductType( + type, + true + )) { + if (elmap.containsKey(el.getElementId())) continue; + elmap.put(el.getElementId(), true); + MetadataProperty prop = new MetadataProperty( + el.getElementId(), + MetadataProperty.DATATYPE + ); + prop.addDomain(type.getProductTypeId()); + prop.setRange(KBUtils.XSD + "string"); + list.add(prop); + } + } + } catch (RepositoryManagerException e) { + e.printStackTrace(); + } + return list; + } + + @Override + public DataItem getDatatypeForData(String dataid) { + try { + Product prod = this.fmclient.getProductById(dataid); + return new DataItem( + prod.getProductType().getProductTypeId(), + DataItem.DATATYPE + ); + } catch (CatalogException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public ArrayList getDataForDatatype( + String dtypeid, + boolean direct + ) { + ArrayList ditems = new ArrayList(); + try { + for (Product prod : this.fmclient.getProductsByProductType( + this.fmclient.getProductTypeById(dtypeid) + )) { + ditems.add( + new DataItem( + prod.getProductType().getProductTypeId(), + DataItem.DATATYPE + ) + ); + } + } catch (CatalogException e) { + e.printStackTrace(); + } catch (RepositoryManagerException e) { + e.printStackTrace(); + } + return ditems; + } + + @Override + public String getTypeNameFormat(String dtypeid) { + // TODO Not Supported + return null; + } + + @Override + public String getDataLocation(String dataid) { + try { + Product prod = this.fmclient.getProductById(dataid); + for (Reference ref : this.fmclient.getProductReferences(prod)) { + return ref.getDataStoreReference(); + } + } catch (CatalogException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public ArrayList getMetadataValues( + String dataid, + ArrayList propids + ) { + ArrayList vals = new ArrayList(); + try { + Product prod = this.fmclient.getProductById(dataid); + Metadata meta = this.fmclient.getMetadata(prod); + for (String propid : propids) { + Element el = this.fmclient.getElementById(propid); + MetadataValue val = new MetadataValue( + propid, + meta.getMetadata(el.getElementName()), + MetadataValue.DATATYPE + ); + vals.add(val); + } + } catch (CatalogException e) { + e.printStackTrace(); + } catch (ValidationLayerException e) { + e.printStackTrace(); + } + return vals; + } + + @Override + public boolean addDatatype(String dtypeid, String parentid) { + String dtypename = new URIEntity(dtypeid).getName(); + String desc = "A product type for " + dtypename + " files"; + String ver = "org.apache.oodt.cas.filemgr.versioning.BasicVersioner"; + String repo = "file://" + this.archivedir; + ProductType type = new ProductType(dtypeid, dtypeid, desc, repo, ver); + try { + this.fmclient.addProductType(type); + if (parentid != null) return this.curatorApi.addParentForProductType( + type, + parentid + ); + } catch (RepositoryManagerException e) { + e.printStackTrace(); + } + return false; + } + + @Override + public boolean removeDatatype(String dtypeid) { + try { + ProductType type = this.fmclient.getProductTypeById(dtypeid); + for (Product prod : this.fmclient.getProductsByProductType(type)) { + this.fmclient.removeProduct(prod); + } + return this.curatorApi.removeProductType(type); + } catch (RepositoryManagerException e) { + e.printStackTrace(); + } catch (CatalogException e) { + e.printStackTrace(); + } + return false; + } + + @Override + public boolean renameDatatype(String newtypeid, String oldtypeid) { + try { + HashMap pmap = this.curatorApi.getParentTypeMap(); + this.addDatatype( + newtypeid, + pmap.containsKey(oldtypeid) ? pmap.get(oldtypeid) : null + ); + this.save(); + + ProductType otype = this.fmclient.getProductTypeById(oldtypeid); + ProductType ntype = this.fmclient.getProductTypeById(newtypeid); + for (Product prod : this.fmclient.getProductsByProductType(otype)) { + prod.setProductType(ntype); + this.fmclient.modifyProduct(prod); + } + boolean ok1 = + this.curatorApi.addElementsForProductType( + ntype, + this.curatorApi.getElementsForProductType(otype, true) + ); + boolean ok2 = this.curatorApi.removeAllElementsForProductType(otype); + boolean ok3 = this.curatorApi.removeProductType(otype); + return ok1 && ok2 && ok3; + } catch (RepositoryManagerException e) { + e.printStackTrace(); + } catch (CatalogException e) { + e.printStackTrace(); + } + return false; + } + + @Override + public boolean moveDatatypeParent( + String dtypeid, + String fromtypeid, + String totypeid + ) { + // Not currently supported + return false; + } + + @Override + public boolean moveDataParent(String arg0, String arg1, String arg2) { + // Not currently supported + return false; + } + + @Override + public boolean addData(String dataid, String dtypeid) { + prodTypes.put(dataid, dtypeid); + return true; + } + + @Override + public boolean renameData(String newdataid, String olddataid) { + // TODO Not Implemented + return false; + } + + @Override + public boolean removeData(String dataid) { + try { + Product prod = this.fmclient.getProductById(dataid); + this.fmclient.removeProduct(prod); + return true; + } catch (CatalogException e) { + e.printStackTrace(); + } + return false; + } + + @Override + public boolean setDataLocation(String dataid, String locuri) { + String dtypeid = prodTypes.get(dataid); + if (dtypeid == null) return false; + if (locuri == null) return false; + + ProductType type; + try { + type = this.fmclient.getProductTypeById(dtypeid); + } catch (RepositoryManagerException e1) { + e1.printStackTrace(); + return false; + } + String dataname = new URIEntity(dataid).getName(); + + ArrayList refs = new ArrayList(); + File locf = new File(locuri); + if (!locf.exists()) return false; + + long filesize = locf.length(); + refs.add(new Reference(locf.toURI().toString(), "", filesize)); + + Product prod = new Product( + dataname, + type, + Product.STRUCTURE_FLAT, + Product.STATUS_TRANSFER, + refs + ); + prod.setProductId(dataid); + try { + this.fmclient.ingestProduct(prod, new Metadata(), false); + return true; + } catch (Exception e) { + e.printStackTrace(); + } + return false; + } + + @Override + public boolean setTypeNameFormat(String dtypeid, String format) { + // TODO Not Supported + return false; + } + + @Override + public boolean addObjectPropertyValue( + String dataid, + String propid, + String valid + ) { + return this.addDatatypePropertyValue(dataid, propid, valid); + } + + @Override + public boolean addDatatypePropertyValue( + String dataid, + String propid, + Object val + ) { + return this.addDatatypePropertyValue(dataid, propid, val.toString(), null); + } + + @Override + public boolean addDatatypePropertyValue( + String dataid, + String propid, + String val, + String xsdtype + ) { + try { + Product prod = this.fmclient.getProductById(dataid); + if (prod == null) return false; + Element el = this.fmclient.getElementById(propid); + if (el == null) return false; + Metadata meta = this.fmclient.getMetadata(prod); + meta.removeMetadata(el.getElementName()); + meta.addMetadata(el.getElementName(), val); + this.fmclient.updateMetadata(prod, meta); + return true; + } catch (ValidationLayerException e) { + e.printStackTrace(); + } catch (CatalogException e) { + e.printStackTrace(); + } + return false; + } + + @Override + public boolean removePropertyValue(String dataid, String propid, Object val) { + try { + Product prod = this.fmclient.getProductById(dataid); + if (prod == null) return false; + Metadata newm = new Metadata(); + Metadata meta = this.fmclient.getMetadata(prod); + for (String mpropid : meta.getAllKeys()) { + if (mpropid.equals(propid)) continue; + newm.addMetadata(mpropid, meta.getMetadata(mpropid)); + } + this.fmclient.updateMetadata(prod, newm); + return true; + } catch (CatalogException e) { + e.printStackTrace(); + } + return false; + } + + @Override + public boolean removeAllPropertyValues( + String dataid, + ArrayList propids + ) { + try { + Product prod = this.fmclient.getProductById(dataid); + if (prod == null) return false; + Metadata newm = new Metadata(); + this.fmclient.updateMetadata(prod, newm); + return true; + } catch (CatalogException e) { + e.printStackTrace(); + } + return false; + } + + @Override + public boolean addMetadataProperty( + String propid, + String domain, + String range + ) { + if (domain == null) return false; + try { + ProductType type = this.fmclient.getProductTypeById(domain); + String propname = new URIEntity(propid).getName(); + Element el = new Element( + propid, + propname, + "", + "", + "Element " + propname, + "" + ); + // FIXME: Range ignored for now + ArrayList elementList = new ArrayList(); + elementList.add(el); + return this.curatorApi.addElementsForProductType(type, elementList); + } catch (RepositoryManagerException e) { + e.printStackTrace(); + } + return false; + } + + @Override + public boolean addMetadataPropertyDomain(String propid, String domain) { + try { + Element el = this.fmclient.getElementById(propid); + if (el == null) return false; + ProductType type = this.fmclient.getProductTypeById(domain); + if (type == null) return false; + ArrayList elist = new ArrayList(); + elist.add(el); + this.curatorApi.addElementsForProductType(type, elist); + return true; + } catch (RepositoryManagerException e) { + e.printStackTrace(); + } catch (ValidationLayerException e) { + e.printStackTrace(); + } + return false; + } + + @Override + public boolean removeMetadataProperty(String propid) { + try { + Element el = this.fmclient.getElementById(propid); + if (el == null) return false; + ArrayList elist = new ArrayList(); + elist.add(el); + for (ProductType type : this.fmclient.getProductTypes()) { + this.curatorApi.removeElementsForProductType(type, elist); + } + return true; + } catch (ValidationLayerException e) { + e.printStackTrace(); + } catch (RepositoryManagerException e) { + e.printStackTrace(); + } + return false; + } + + @Override + public boolean removeMetadataPropertyDomain(String propid, String domain) { + try { + Element el = this.fmclient.getElementById(propid); + if (el == null) return false; + ProductType type = this.fmclient.getProductTypeById(domain); + if (type == null) return false; + ArrayList elist = new ArrayList(); + elist.add(el); + return this.curatorApi.removeElementsForProductType(type, elist); + } catch (RepositoryManagerException e) { + e.printStackTrace(); + } catch (ValidationLayerException e) { + e.printStackTrace(); + } + return false; + } + + @Override + public boolean renameMetadataProperty(String oldid, String newid) { + // TODO NOT implemented + return false; + } + + @Override + public boolean save() { + this.fmclient.refreshConfigAndPolicy(); + return true; + } + + @Override + public void copyFrom(DataCreationAPI dc) { + DataTree tree = dc.getDataHierarchy(); + DataTreeNode root = tree.getRoot(); + this.initialSync(dc, root, null); + this.save(); + } + + @Override + public DataCreationAPI getExternalCatalog() { + // TODO NOT implemented + return null; + } + + @Override + public void setExternalCatalog(DataCreationAPI dc) { + // TODO NOT implemented + + } + + /* + * Private functions + */ + private void initialSync( + DataCreationAPI dc, + DataTreeNode node, + DataTreeNode parent + ) { + // Replace top DataObject with local namespaced DataObject + String dtypeid = node.getItem().getID(); + if (dtypeid.equals(this.dcurl + "#DataObject")) dtypeid = + this.onturl + "#DataObject"; + + ProductType ptype = null; + try { + ptype = this.fmclient.getProductTypeById(dtypeid); + } catch (RepositoryManagerException e) { + ptype = null; + } + + if (ptype == null) { + String pdtypeid = parent != null ? parent.getItem().getID() : null; + if ( + pdtypeid != null && pdtypeid.equals(this.dcurl + "#DataObject") + ) pdtypeid = this.onturl + "#DataObject"; + this.addDatatype(dtypeid, pdtypeid); + this.save(); + + ArrayList props = new ArrayList(); + HashMap propElements = new HashMap(); + for (MetadataProperty prop : dc.getMetadataProperties( + node.getItem().getID(), + false + )) { + props.add(prop.getID()); + Element el; + try { + el = this.fmclient.getElementById(prop.getID()); + } catch (ValidationLayerException e) { + this.addMetadataProperty(prop.getID(), dtypeid, null); + this.save(); + try { + el = this.fmclient.getElementById(prop.getID()); + } catch (ValidationLayerException e1) { + el = null; + } + } + if (el != null) propElements.put(prop.getID(), el); + } + for (DataTreeNode cnode : node.getChildren()) { + if (cnode.getItem().getType() == DataItem.DATA) { + String dataid = cnode.getItem().getID(); + Product prod; + try { + prod = this.fmclient.getProductById(dataid); + } catch (CatalogException e) { + prod = null; + } + if (prod == null) { + this.addData(dataid, dtypeid); + this.setDataLocation(dataid, dc.getDataLocation(dataid)); + this.addMetadataValues(dataid, dc.getMetadataValues(dataid, props)); + } + } else if (cnode.getItem().getType() == DataItem.DATATYPE) { + this.initialSync(dc, cnode, node); + } + } + } else { + for (DataTreeNode cnode : node.getChildren()) { + if (cnode.getItem().getType() == DataItem.DATATYPE) { + this.initialSync(dc, cnode, node); + } + } + } + } + + private boolean addMetadataValues( + String dataid, + ArrayList mvalues + ) { + try { + Product prod; + try { + prod = this.fmclient.getProductById(dataid); + } catch (CatalogException e) { + prod = null; + } + if (prod == null) return false; + + Metadata meta = new Metadata(); + for (MetadataValue mval : mvalues) { + String propid = mval.getPropertyId(); + meta.addMetadata(propid, mval.getValue().toString()); + } + this.fmclient.addMetadata(prod, meta); + return true; + } catch (CatalogException e) { + e.printStackTrace(); + } + return false; + } @Override public boolean end() { @@ -754,7 +809,7 @@ public boolean start_write() { @Override public void stop_batch_operation() { // TODO Auto-generated method stub - + } @Override @@ -764,7 +819,11 @@ public HashMap> getAllDatatypeDatasets() { } @Override - public boolean moveDatatypeParentInLibrary(String dtypeid, String fromtypeid, String totypeid) { + public boolean moveDatatypeParentInLibrary( + String dtypeid, + String fromtypeid, + String totypeid + ) { // TODO Auto-generated method stub return false; } @@ -792,5 +851,4 @@ public boolean delete() { // TODO Auto-generated method stub return false; } - } diff --git a/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTExecutionEngine.java b/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTExecutionEngine.java index 69a312a3..5d558aee 100644 --- a/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTExecutionEngine.java +++ b/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTExecutionEngine.java @@ -17,19 +17,7 @@ package edu.isi.wings.execution.engine.api.impl.oodt; -import java.io.File; -import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Properties; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.commons.io.FileUtils; -import org.apache.oodt.cas.filemgr.datatransfer.DataTransfer; -import org.apache.oodt.cas.filemgr.datatransfer.RemoteDataTransferFactory; -import org.apache.oodt.cas.filemgr.structs.Product; -import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; +import edu.isi.kcap.ontapi.KBTriple; import edu.isi.wings.execution.engine.api.PlanExecutionEngine; import edu.isi.wings.execution.engine.api.StepExecutionEngine; import edu.isi.wings.execution.engine.classes.RuntimeInfo; @@ -38,13 +26,26 @@ import edu.isi.wings.execution.tools.api.ExecutionLoggerAPI; import edu.isi.wings.execution.tools.api.ExecutionMonitorAPI; import edu.isi.wings.execution.tools.api.ExecutionResourceAPI; -import edu.isi.kcap.ontapi.KBTriple; import edu.isi.wings.workflow.plan.classes.ExecutionFile; import edu.isi.wings.workflow.template.TemplateFactory; import edu.isi.wings.workflow.template.api.Template; import edu.isi.wings.workflow.template.classes.variables.Variable; +import java.io.File; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Properties; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.io.FileUtils; +import org.apache.oodt.cas.filemgr.datatransfer.DataTransfer; +import org.apache.oodt.cas.filemgr.datatransfer.RemoteDataTransferFactory; +import org.apache.oodt.cas.filemgr.structs.Product; +import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; + +public class OODTExecutionEngine + implements PlanExecutionEngine, StepExecutionEngine { -public class OODTExecutionEngine implements PlanExecutionEngine, StepExecutionEngine{ Properties props; protected int maxParallel = 4; @@ -94,17 +95,24 @@ public void execute(RuntimePlan exe) { String wmsurl = props.getProperty("oodt.wmsurl"); String libns = props.getProperty("lib.domain.data.url") + "#"; - String codedir = props.getProperty("lib.domain.code.storage") + File.separator; - String datadir = props.getProperty("lib.domain.data.storage") + File.separator; - + String codedir = + props.getProperty("lib.domain.code.storage") + File.separator; + String datadir = + props.getProperty("lib.domain.data.storage") + File.separator; + // Get Variable metadata (predicted) - Template tmpl = TemplateFactory.getTemplate(props, exe.getExpandedTemplateID()); - org.apache.oodt.cas.metadata.Metadata meta = + Template tmpl = TemplateFactory.getTemplate( + props, + exe.getExpandedTemplateID() + ); + org.apache.oodt.cas.metadata.Metadata meta = + new org.apache.oodt.cas.metadata.Metadata(); + for (Variable var : tmpl.getVariables()) { + org.apache.oodt.cas.metadata.Metadata vmeta = new org.apache.oodt.cas.metadata.Metadata(); - for(Variable var : tmpl.getVariables()) { - org.apache.oodt.cas.metadata.Metadata vmeta = - new org.apache.oodt.cas.metadata.Metadata(); - for(KBTriple t : tmpl.getConstraintEngine().getConstraints(var.getID())) { + for (KBTriple t : tmpl + .getConstraintEngine() + .getConstraints(var.getID())) { vmeta.addMetadata(t.getPredicate().getID(), t.getObject().toString()); } meta.addMetadata(var.getBinding().getName(), vmeta); @@ -114,18 +122,35 @@ public void execute(RuntimePlan exe) { File f = File.createTempFile("oodt-run-", ""); if (f.delete() && f.mkdirs()) { this.jobdir = f.getAbsolutePath() + File.separator; - this.wlogfile = exe.getName()+".log"; - this.adapter = new OODTWorkflowAdapter(wmurl, wmsurl, - fmurl, libns, - codedir, datadir, - jobdir, wlogfile); + this.wlogfile = exe.getName() + ".log"; + this.adapter = + new OODTWorkflowAdapter( + wmurl, + wmsurl, + fmurl, + libns, + codedir, + datadir, + jobdir, + wlogfile + ); this.adapter.runWorkflow(exe, meta); // Start Monitoring thread - this.monitoringThread = new Thread( - new ExecutionMonitoringThread(this, exe, this.logger, this.monitor, - this.jobdir, datadir, this.wlogfile, - fmurl, libns)); + this.monitoringThread = + new Thread( + new ExecutionMonitoringThread( + this, + exe, + this.logger, + this.monitor, + this.jobdir, + datadir, + this.wlogfile, + fmurl, + libns + ) + ); this.monitoringThread.start(); } } catch (Exception e) { @@ -142,9 +167,9 @@ public void onStepEnd(RuntimePlan exe) { @Override public void abort(RuntimePlan exe) { // Abort plan - if(this.monitoringThread != null && - this.monitoringThread.isAlive()) - this.monitoringThread.interrupt(); + if ( + this.monitoringThread != null && this.monitoringThread.isAlive() + ) this.monitoringThread.interrupt(); this.adapter.stopWorkflow(exe); } @@ -180,6 +205,7 @@ public ExecutionLoggerAPI getExecutionLogger() { } class ExecutionMonitoringThread implements Runnable { + RuntimePlan planexe; String jobdir; String datadir; @@ -190,9 +216,17 @@ class ExecutionMonitoringThread implements Runnable { ExecutionLoggerAPI logger; ExecutionMonitorAPI monitor; - public ExecutionMonitoringThread(PlanExecutionEngine planEngine, - RuntimePlan planexe, ExecutionLoggerAPI logger, ExecutionMonitorAPI monitor, - String jobdir, String datadir, String wlogfile, String fmurl, String libns) { + public ExecutionMonitoringThread( + PlanExecutionEngine planEngine, + RuntimePlan planexe, + ExecutionLoggerAPI logger, + ExecutionMonitorAPI monitor, + String jobdir, + String datadir, + String wlogfile, + String fmurl, + String libns + ) { this.planEngine = planEngine; this.planexe = planexe; this.logger = logger; @@ -208,11 +242,11 @@ public ExecutionMonitoringThread(PlanExecutionEngine planEngine, public void run() { planexe.onStart(this.logger); try { - HashMap jobstatus = - new HashMap(); - + HashMap jobstatus = + new HashMap(); + int osleeptime = 1000; - int maxsleeptime = 4*osleeptime; + int maxsleeptime = 4 * osleeptime; int sleeptime = osleeptime; XmlRpcFileManagerClient fmclient = null; @@ -221,38 +255,40 @@ public void run() { fmclient = new XmlRpcFileManagerClient(new URL(fmurl)); dt = new RemoteDataTransferFactory().createDataTransfer(); dt.setFileManagerUrl(new URL(fmurl)); - } - catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); return; } - + Product prod = null; int count = 0; - while(true) { + while (true) { try { prod = fmclient.getProductById(this.wlogfile); prod.setProductReferences(fmclient.getProductReferences(prod)); break; - } - catch (Exception e) { + } catch (Exception e) { Thread.sleep(2000); count++; - if(count == 20) - return; + if (count == 20) return; } } - - Pattern pattern = Pattern.compile("^(Job\\d+)\\s+\\((.+)\\)\\s*:\\s+(.+)$"); - while(true) { + + Pattern pattern = Pattern.compile( + "^(Job\\d+)\\s+\\((.+)\\)\\s*:\\s+(.+)$" + ); + while (true) { dt.retrieveProduct(prod, new File(jobdir)); - - for(String line : FileUtils.readLines( - new File(this.jobdir + this.wlogfile))) { + + for (String line : FileUtils.readLines( + new File(this.jobdir + this.wlogfile) + )) { Matcher mat = pattern.matcher(line); - if(mat.find()) { + if (mat.find()) { String jobname = mat.group(2); - RuntimeInfo.Status status = RuntimeInfo.Status.valueOf(mat.group(3)); + RuntimeInfo.Status status = RuntimeInfo.Status.valueOf( + mat.group(3) + ); jobstatus.put(jobname, status); } } @@ -262,32 +298,38 @@ public void run() { steps.addAll(planexe.getQueue().getRunningSteps()); boolean shortsleep = false; - for(RuntimeStep stepexe : steps) { + for (RuntimeStep stepexe : steps) { String jobname = stepexe.getStep().getName(); - if(jobstatus.containsKey(jobname)) { - if(stepexe.getRuntimeInfo().getStatus() == - RuntimeInfo.Status.QUEUED) { + if (jobstatus.containsKey(jobname)) { + if ( + stepexe.getRuntimeInfo().getStatus() == + RuntimeInfo.Status.QUEUED + ) { stepexe.setRuntimePlan(planexe); stepexe.onStart(logger); shortsleep = true; } RuntimeInfo.Status status = jobstatus.get(jobname); - if(status == RuntimeInfo.Status.SUCCESS || - status == RuntimeInfo.Status.FAILURE) { + if ( + status == RuntimeInfo.Status.SUCCESS || + status == RuntimeInfo.Status.FAILURE + ) { // Fetch log file File f = new File(this.jobdir + jobname + ".log"); String log = ""; - if(!f.exists()) { + if (!f.exists()) { try { Product logprod = fmclient.getProductById( - planexe.getName() + "-" + f.getName()); - logprod.setProductReferences(fmclient.getProductReferences(logprod)); + planexe.getName() + "-" + f.getName() + ); + logprod.setProductReferences( + fmclient.getProductReferences(logprod) + ); dt.retrieveProduct(logprod, new File(this.jobdir)); log = FileUtils.readFileToString(f); fmclient.removeProduct(logprod); - } - catch (Exception e) {} + } catch (Exception e) {} } stepexe.onEnd(logger, status, log); shortsleep = true; @@ -296,23 +338,26 @@ public void run() { } // Retrieve output files (and metafiles) - for(RuntimeStep stepexe : planexe.getQueue().getFinishedSteps()) { - for(ExecutionFile file : stepexe.getStep().getOutputFiles()) { + for (RuntimeStep stepexe : planexe.getQueue().getFinishedSteps()) { + for (ExecutionFile file : stepexe.getStep().getOutputFiles()) { File f = new File(datadir + file.getName()); - if(!f.exists()) { + if (!f.exists()) { try { String outprodid = this.libns + file.getBinding(); Product outprod = fmclient.getProductById(outprodid); - outprod.setProductReferences(fmclient.getProductReferences(outprod)); + outprod.setProductReferences( + fmclient.getProductReferences(outprod) + ); dt.retrieveProduct(outprod, new File(datadir)); Product metprod = fmclient.getProductById(outprodid + ".met"); - if(metprod != null) { - metprod.setProductReferences(fmclient.getProductReferences(metprod)); + if (metprod != null) { + metprod.setProductReferences( + fmclient.getProductReferences(metprod) + ); dt.retrieveProduct(metprod, new File(datadir)); } - } - catch (Exception e) { + } catch (Exception e) { //e.printStackTrace(); } } @@ -320,31 +365,33 @@ public void run() { } steps = planexe.getQueue().getNextStepsToExecute(); - if(steps.size() == 0) { + if (steps.size() == 0) { // Nothing to execute. Check if finished - if(planexe.getQueue().getRunningSteps().size() == 0) { + if (planexe.getQueue().getRunningSteps().size() == 0) { RuntimeInfo.Status status = RuntimeInfo.Status.FAILURE; - if(planexe.getQueue().getFinishedSteps().size() == - planexe.getQueue().getAllSteps().size()) { - if(planexe.getPlan().isIncomplete()) { + if ( + planexe.getQueue().getFinishedSteps().size() == + planexe.getQueue().getAllSteps().size() + ) { + if (planexe.getPlan().isIncomplete()) { // If the plan is incomplete, then replan and continue System.out.println("Replanning, and re-executing"); planexe = this.monitor.rePlan(planexe); - if(planexe.getRuntimeInfo().getStatus() == - RuntimeInfo.Status.FAILURE) { + if ( + planexe.getRuntimeInfo().getStatus() == + RuntimeInfo.Status.FAILURE + ) { status = RuntimeInfo.Status.FAILURE; planexe.onEnd(this.logger, status, "Finished"); fmclient.removeProduct(prod); break; - } - else { + } else { // Quit monitoring and run the new plan fmclient.removeProduct(prod); this.planEngine.execute(planexe); break; } - } - else { + } else { status = RuntimeInfo.Status.SUCCESS; planexe.onEnd(this.logger, status, "Finished"); fmclient.removeProduct(prod); @@ -353,14 +400,19 @@ public void run() { } } } - - sleeptime = shortsleep ? osleeptime : - (sleeptime >= maxsleeptime ? maxsleeptime : sleeptime*2); + + sleeptime = + shortsleep + ? osleeptime + : (sleeptime >= maxsleeptime ? maxsleeptime : sleeptime * 2); Thread.sleep(sleeptime); } - } - catch (Exception e) { - this.planexe.onEnd(this.logger, RuntimeInfo.Status.FAILURE, e.getMessage()); + } catch (Exception e) { + this.planexe.onEnd( + this.logger, + RuntimeInfo.Status.FAILURE, + e.getMessage() + ); } } } @@ -368,8 +420,7 @@ public void run() { @Override public void setExecutionMonitor(ExecutionMonitorAPI monitor) { this.monitor = monitor; - if (this.stepEngine != this) - this.stepEngine.setExecutionMonitor(monitor); + if (this.stepEngine != this) this.stepEngine.setExecutionMonitor(monitor); } @Override @@ -380,13 +431,11 @@ public ExecutionMonitorAPI getExecutionMonitor() { @Override public void setExecutionResource(ExecutionResourceAPI resource) { this.resource = resource; - if(this.stepEngine != this) - this.stepEngine.setExecutionResource(resource); + if (this.stepEngine != this) this.stepEngine.setExecutionResource(resource); } @Override public ExecutionResourceAPI getExecutionResource() { return this.resource; } - } diff --git a/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTWorkflowAdapter.java b/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTWorkflowAdapter.java index 275ac3b8..080b5236 100644 --- a/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTWorkflowAdapter.java +++ b/engines/oodt/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/oodt/OODTWorkflowAdapter.java @@ -17,6 +17,10 @@ package edu.isi.wings.execution.engine.api.impl.oodt; +import edu.isi.wings.execution.engine.classes.RuntimePlan; +import edu.isi.wings.execution.engine.classes.RuntimeStep; +import edu.isi.wings.workflow.plan.api.ExecutionStep; +import edu.isi.wings.workflow.plan.classes.ExecutionFile; import java.io.PrintStream; import java.net.MalformedURLException; import java.net.URL; @@ -24,7 +28,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.TreeMap; - import org.apache.oodt.cas.metadata.Metadata; import org.apache.oodt.cas.wmservices.client.WmServicesClient; import org.apache.oodt.cas.workflow.structs.Graph; @@ -33,123 +36,130 @@ import org.apache.oodt.cas.workflow.structs.WorkflowTask; import org.apache.oodt.cas.workflow.structs.WorkflowTaskConfiguration; import org.apache.oodt.cas.workflow.system.XmlRpcWorkflowManagerClient; -import edu.isi.wings.execution.engine.classes.RuntimePlan; -import edu.isi.wings.execution.engine.classes.RuntimeStep; -import edu.isi.wings.workflow.plan.api.ExecutionStep; -import edu.isi.wings.workflow.plan.classes.ExecutionFile; public class OODTWorkflowAdapter { - String codedir; - String datadir; - String jobdir; - String wmurl; - String fmurl; - String wmsurl; - String fmprefix; - String wlogfile; - - XmlRpcWorkflowManagerClient wmclient; - - public OODTWorkflowAdapter(String wmurl, String wmsurl, - String fmurl, String fmprefix, - String codedir, String datadir, - String jobdir, String wlogfile) { - - this.wmurl = wmurl; - this.wmsurl = wmsurl; - this.fmurl = fmurl; - this.fmprefix = fmprefix; - - try { - this.wmclient = new XmlRpcWorkflowManagerClient(new URL(wmurl)); - } catch (MalformedURLException e) { - e.printStackTrace(); - } - - this.codedir = codedir; - this.datadir = datadir; - this.jobdir = jobdir; - this.wlogfile = wlogfile; - } - - public Workflow runWorkflow(RuntimePlan planexe, Metadata meta) - throws Exception { - - // Get List of all Jobs - HashMap tasksById = new HashMap(); - HashMap opProducers = new HashMap(); - HashMap taskJobs = new HashMap(); - for(RuntimeStep stepexe : planexe.getQueue().getAllSteps()) { - ExecutionStep step = stepexe.getStep(); - - ArrayList inputs = new ArrayList(); - ArrayList outputs = new ArrayList(); - String argstring = ""; - for(String argname : step.getInvocationArguments().keySet()) { - argstring += argname + " "; - for(Object arg : step.getInvocationArguments().get(argname)) { - if(arg instanceof String) - argstring += arg; - else if(arg instanceof ExecutionFile) - argstring += this.jobdir + ((ExecutionFile)arg).getBinding(); - argstring += " "; - } - } - for(ExecutionFile input : step.getInputFiles()) - inputs.add(input.getBinding()); - for(ExecutionFile output : step.getOutputFiles()) - outputs.add(output.getBinding()); - WorkflowTask task = this.getTask(planexe, stepexe, inputs, outputs, argstring); - - tasksById.put(stepexe.getName(), task); - taskJobs.put(stepexe.getName(), step.getCodeBinding().getID()); - for(ExecutionFile output : step.getOutputFiles()) - opProducers.put(output.getName(), stepexe.getName()); - } - - // Get Parent Child Relationship between jobs - HashMap > parents = - new HashMap >(); - for (RuntimeStep stepexe : planexe.getQueue().getAllSteps()) { - ExecutionStep step = stepexe.getStep(); - ArrayList chparents = parents.get(stepexe.getName()); - if (chparents == null) - chparents = new ArrayList(); - for (ExecutionFile input : step.getInputFiles()) { - if (opProducers.containsKey(input.getName())) - chparents.add(opProducers.get(input.getName())); - } - parents.put(stepexe.getName(), chparents); - } - // Arrange Jobs into Job Levels - TreeMap> jobLevels = new TreeMap>(); - for (String jobid : tasksById.keySet()) { - Integer lvl = new Integer(getJobLevel(jobid, 0, parents)); - ArrayList lvljobs; - if (jobLevels.containsKey(lvl)) { - lvljobs = jobLevels.get(lvl); - } else { - lvljobs = new ArrayList(); - jobLevels.put(lvl, lvljobs); - } - lvljobs.add(jobid); - } - - // Create workflow graph + String codedir; + String datadir; + String jobdir; + String wmurl; + String fmurl; + String wmsurl; + String fmprefix; + String wlogfile; + + XmlRpcWorkflowManagerClient wmclient; + + public OODTWorkflowAdapter( + String wmurl, + String wmsurl, + String fmurl, + String fmprefix, + String codedir, + String datadir, + String jobdir, + String wlogfile + ) { + this.wmurl = wmurl; + this.wmsurl = wmsurl; + this.fmurl = fmurl; + this.fmprefix = fmprefix; + + try { + this.wmclient = new XmlRpcWorkflowManagerClient(new URL(wmurl)); + } catch (MalformedURLException e) { + e.printStackTrace(); + } + + this.codedir = codedir; + this.datadir = datadir; + this.jobdir = jobdir; + this.wlogfile = wlogfile; + } + + public Workflow runWorkflow(RuntimePlan planexe, Metadata meta) + throws Exception { + // Get List of all Jobs + HashMap tasksById = + new HashMap(); + HashMap opProducers = new HashMap(); + HashMap taskJobs = new HashMap(); + for (RuntimeStep stepexe : planexe.getQueue().getAllSteps()) { + ExecutionStep step = stepexe.getStep(); + + ArrayList inputs = new ArrayList(); + ArrayList outputs = new ArrayList(); + String argstring = ""; + for (String argname : step.getInvocationArguments().keySet()) { + argstring += argname + " "; + for (Object arg : step.getInvocationArguments().get(argname)) { + if (arg instanceof String) argstring += arg; else if ( + arg instanceof ExecutionFile + ) argstring += this.jobdir + ((ExecutionFile) arg).getBinding(); + argstring += " "; + } + } + for (ExecutionFile input : step.getInputFiles()) inputs.add( + input.getBinding() + ); + for (ExecutionFile output : step.getOutputFiles()) outputs.add( + output.getBinding() + ); + WorkflowTask task = + this.getTask(planexe, stepexe, inputs, outputs, argstring); + + tasksById.put(stepexe.getName(), task); + taskJobs.put(stepexe.getName(), step.getCodeBinding().getID()); + for (ExecutionFile output : step.getOutputFiles()) opProducers.put( + output.getName(), + stepexe.getName() + ); + } + + // Get Parent Child Relationship between jobs + HashMap> parents = + new HashMap>(); + for (RuntimeStep stepexe : planexe.getQueue().getAllSteps()) { + ExecutionStep step = stepexe.getStep(); + ArrayList chparents = parents.get(stepexe.getName()); + if (chparents == null) chparents = new ArrayList(); + for (ExecutionFile input : step.getInputFiles()) { + if (opProducers.containsKey(input.getName())) chparents.add( + opProducers.get(input.getName()) + ); + } + parents.put(stepexe.getName(), chparents); + } + + // Arrange Jobs into Job Levels + TreeMap> jobLevels = + new TreeMap>(); + for (String jobid : tasksById.keySet()) { + Integer lvl = new Integer(getJobLevel(jobid, 0, parents)); + ArrayList lvljobs; + if (jobLevels.containsKey(lvl)) { + lvljobs = jobLevels.get(lvl); + } else { + lvljobs = new ArrayList(); + jobLevels.put(lvl, lvljobs); + } + lvljobs.add(jobid); + } + + // Create workflow graph Graph graph = new Graph(); graph.setExecutionType("sequential"); graph.setModelId(planexe.getName()); graph.setModelName(planexe.getName()); - + ParentChildWorkflow pcw = new ParentChildWorkflow(graph); pcw.setId(planexe.getName()); - + // Add Jobs to Workflow int jobnum = 1; for (Integer lvl : jobLevels.keySet()) { Graph subGraph = null; - + /*if(jobLevels.get(lvl).size() > 1) { subGraph = new Graph(); subGraph.setParent(graph); @@ -158,108 +168,123 @@ else if(arg instanceof ExecutionFile) subGraph.setModelName(planexe.getName()+"_level_"+lvl); } else {*/ - subGraph = graph; + subGraph = graph; //} - + for (String jobid : jobLevels.get(lvl)) { WorkflowTask task = tasksById.get(jobid); String compid = taskJobs.get(jobid); // Set job number specific configs String taskid = "Job" + jobnum; - task.getTaskConfig().addConfigProperty("LOGFILE", - task.getTaskName() + ".log"); + task + .getTaskConfig() + .addConfigProperty("LOGFILE", task.getTaskName() + ".log"); task.getTaskConfig().addConfigProperty("JOBID", taskid); task.getTaskConfig().addConfigProperty("COMPONENT_ID", compid); - + Graph taskGraph = new Graph(); taskGraph.setExecutionType("task"); taskGraph.setModelIdRef(task.getTaskId()); taskGraph.setTask(task); - + taskGraph.setParent(subGraph); subGraph.getChildren().add(taskGraph); - + pcw.getTasks().add(task); jobnum++; } - - if(graph != subGraph) - graph.getChildren().add(subGraph); + + if (graph != subGraph) graph.getChildren().add(subGraph); } PrintStream wlog = new PrintStream(this.jobdir + this.wlogfile); wlog.println("Total Jobs: " + (jobnum - 1)); wlog.close(); - + try { WmServicesClient wmclient = new WmServicesClient(this.wmsurl); wmclient.addPackagedWorkflow(pcw.getId(), pcw); this.wmclient.sendEvent(pcw.getId(), meta); return pcw; - } - catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); } return null; - } - - public boolean stopWorkflow(RuntimePlan exe) { - try { - return this.wmclient.stopWorkflowInstance(exe.getName()); - } catch (Exception e) { - e.printStackTrace(); - return false; - } - } - - private int getJobLevel(String jobid, int level, - HashMap > parents) { - int maxlvl = 0; - ArrayList parentIds = parents.get(jobid); - if (parentIds == null || parentIds.size() == 0) { - maxlvl = level; - } else { - for (String pjobid : parentIds) { - int lvl = getJobLevel(pjobid, level + 1, parents); - if (lvl > maxlvl) { - maxlvl = lvl; - } - } - } - return maxlvl; - } - - private WorkflowTask getTask(RuntimePlan planexe, RuntimeStep exe, - ArrayList ips, ArrayList ops, String arg) - throws Exception { - //checkAndCreateTask(this.wmclient, exe.getName(), exe.getName()); - WorkflowTask task = new WorkflowTask(); - //task.setConditions(Collections.emptyList()); - task.setRequiredMetFields(Collections.emptyList()); - task.setTaskId(planexe.getName() + "-" + exe.getName()); - task.setTaskInstanceClassName("org.apache.oodt.cas.workflow.misc.WingsTask"); - task.setTaskName(exe.getName()); - task.setTaskConfig(this.getTaskConfiguration(exe, ips, ops, arg)); - return task; - } - - private WorkflowTaskConfiguration getTaskConfiguration(RuntimeStep exe, - ArrayList ips, ArrayList ops, String arg) { - WorkflowTaskConfiguration config = new WorkflowTaskConfiguration(); - for(int i=1; i<=ips.size(); i++) { - config.addConfigProperty("INPUT"+i, ips.get(i-1)); - } - for(int i=1; i<=ops.size(); i++) { - config.addConfigProperty("OUTPUT"+i, ops.get(i-1)); - } - config.addConfigProperty("ARGUMENT", arg); - config.addConfigProperty("SCRIPT_PATH", exe.getStep().getCodeBinding().getLocation()); - config.addConfigProperty("JOB_DIR", this.jobdir); - config.addConfigProperty("DATA_DIR", this.datadir); - config.addConfigProperty("FM_URL", this.fmurl); - config.addConfigProperty("FM_PREFIX", this.fmprefix); - config.addConfigProperty("W_LOGFILE", this.wlogfile); - config.addConfigProperty("TASKNAME", exe.getName()); - return config; - } + } + + public boolean stopWorkflow(RuntimePlan exe) { + try { + return this.wmclient.stopWorkflowInstance(exe.getName()); + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + private int getJobLevel( + String jobid, + int level, + HashMap> parents + ) { + int maxlvl = 0; + ArrayList parentIds = parents.get(jobid); + if (parentIds == null || parentIds.size() == 0) { + maxlvl = level; + } else { + for (String pjobid : parentIds) { + int lvl = getJobLevel(pjobid, level + 1, parents); + if (lvl > maxlvl) { + maxlvl = lvl; + } + } + } + return maxlvl; + } + + private WorkflowTask getTask( + RuntimePlan planexe, + RuntimeStep exe, + ArrayList ips, + ArrayList ops, + String arg + ) throws Exception { + //checkAndCreateTask(this.wmclient, exe.getName(), exe.getName()); + WorkflowTask task = new WorkflowTask(); + //task.setConditions(Collections.emptyList()); + task.setRequiredMetFields(Collections.emptyList()); + task.setTaskId(planexe.getName() + "-" + exe.getName()); + task.setTaskInstanceClassName( + "org.apache.oodt.cas.workflow.misc.WingsTask" + ); + task.setTaskName(exe.getName()); + task.setTaskConfig(this.getTaskConfiguration(exe, ips, ops, arg)); + return task; + } + + private WorkflowTaskConfiguration getTaskConfiguration( + RuntimeStep exe, + ArrayList ips, + ArrayList ops, + String arg + ) { + WorkflowTaskConfiguration config = new WorkflowTaskConfiguration(); + for (int i = 1; i <= ips.size(); i++) { + config.addConfigProperty("INPUT" + i, ips.get(i - 1)); + } + for (int i = 1; i <= ops.size(); i++) { + config.addConfigProperty("OUTPUT" + i, ops.get(i - 1)); + } + config.addConfigProperty("ARGUMENT", arg); + config.addConfigProperty( + "SCRIPT_PATH", + exe.getStep().getCodeBinding().getLocation() + ); + config.addConfigProperty("JOB_DIR", this.jobdir); + config.addConfigProperty("DATA_DIR", this.datadir); + config.addConfigProperty("FM_URL", this.fmurl); + config.addConfigProperty("FM_PREFIX", this.fmprefix); + config.addConfigProperty("W_LOGFILE", this.wlogfile); + config.addConfigProperty("TASKNAME", exe.getName()); + return config; + } } diff --git a/engines/oodt/adapter/src/main/java/edu/isi/wings/util/oodt/CurationServiceAPI.java b/engines/oodt/adapter/src/main/java/edu/isi/wings/util/oodt/CurationServiceAPI.java index 1e80a990..b4e92b0a 100644 --- a/engines/oodt/adapter/src/main/java/edu/isi/wings/util/oodt/CurationServiceAPI.java +++ b/engines/oodt/adapter/src/main/java/edu/isi/wings/util/oodt/CurationServiceAPI.java @@ -17,6 +17,7 @@ package edu.isi.wings.util.oodt; +import com.google.gson.Gson; import java.io.DataOutputStream; import java.net.HttpURLConnection; import java.net.URL; @@ -25,127 +26,166 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; - import org.apache.commons.io.IOUtils; import org.apache.oodt.cas.filemgr.structs.Element; import org.apache.oodt.cas.filemgr.structs.ProductType; -import com.google.gson.Gson; - public class CurationServiceAPI { - String policy; - String curatorurl; - Gson gson; - String service = "/services/metadata/productType/"; - - public CurationServiceAPI(String curatorurl, String policy) { - this.curatorurl = curatorurl; - this.policy = policy; - this.gson = new Gson(); - } - - private String query(String method, String op, Object... args) { - String url = this.curatorurl + this.service + op; - try { - String params = "policy=" + URLEncoder.encode(this.policy, "UTF-8"); - for (int i = 0; i < args.length; i += 2) { - params += "&" + args[i] + "=" - + URLEncoder.encode(args[i+1].toString(), "UTF-8"); - } - - URL urlobj = new URL(url); - if("GET".equals(method)) - urlobj = new URL(url + "?" + params); - HttpURLConnection con = (HttpURLConnection) urlobj.openConnection(); - con.setRequestMethod(method); - if(!"GET".equals(method)) { - con.setDoOutput(true); - DataOutputStream out = new DataOutputStream(con.getOutputStream()); - out.writeBytes(params); - out.flush(); - out.close(); - } - - String result = IOUtils.toString(con.getInputStream()); - con.disconnect(); - return result; - - } catch (Exception e) { - e.printStackTrace(); - } - return null; - } - - public boolean removeProductType(ProductType type) { - String result = this.query("DELETE", "remove", - "id", type.getProductTypeId()); - return Boolean.parseBoolean(result); - } - - public HashMap getParentTypeMap() { - String result = this.query("GET", "parentmap"); - HashMap map = gson.fromJson(result, HashMap.class); - HashMap ptypeMap = new HashMap(); - for(Object key : map.keySet()) { - ptypeMap.put((String)key, (String)map.get(key)); - } - return ptypeMap; - } - - public boolean addParentForProductType(ProductType type, String parentId) { - String result = this.query("POST", "parent/add", - "id", type.getProductTypeId(), - "parentId", parentId); - return Boolean.parseBoolean(result); - } - - public boolean removeParentForProductType(ProductType type) { - String result = this.query("DELETE", "parent/remove", "id", type.getProductTypeId()); - return Boolean.parseBoolean(result); - } - - public boolean addElementsForProductType(ProductType type, List elementList) { - String elementIds = ""; - for(Element element: elementList) - elementIds += (elementIds != "" ? "," : "") + element.getElementId(); - String result = this.query("POST", "elements/add", - "id", type.getProductTypeId(), - "elementIds", elementIds); - return Boolean.parseBoolean(result); - } - - public List getElementsForProductType(ProductType type, boolean direct) { - String result = this.query("GET", "elements", - "id", type.getProductTypeId(), - "direct", direct); - List elementList = new ArrayList(); - Object[] elementIds = gson.fromJson(result, ArrayList.class).toArray(); - for(Object elementId : elementIds) { - String id = (String)elementId; - elementList.add(new Element(id, id, "", "", "Automatically Added", "")); - } - return elementList; - } - - public boolean removeAllElementsForProductType(ProductType type) { - String result = this.query("DELETE", "elements/remove/all", - "id", type.getProductTypeId()); - return Boolean.parseBoolean(result); - } - - public boolean removeElementsForProductType(ProductType type, List elementList) { - String elementIds = ""; - for(Element element: elementList) - elementIds += (elementIds != "" ? "," : "") + element.getElementId(); - String result = this.query("DELETE", "elements/remove", - "id", type.getProductTypeId(), - "elementIds", elementIds); - return Boolean.parseBoolean(result); - } - - public List getProductTypeIdsHavingElement(Element el) { - String result = this.query("GET", "typeswithelement/"+el.getElementId(), "id"); - Object[] typeIds = gson.fromJson(result, ArrayList.class).toArray(); - return Arrays.asList((String[])typeIds); - } + + String policy; + String curatorurl; + Gson gson; + String service = "/services/metadata/productType/"; + + public CurationServiceAPI(String curatorurl, String policy) { + this.curatorurl = curatorurl; + this.policy = policy; + this.gson = new Gson(); + } + + private String query(String method, String op, Object... args) { + String url = this.curatorurl + this.service + op; + try { + String params = "policy=" + URLEncoder.encode(this.policy, "UTF-8"); + for (int i = 0; i < args.length; i += 2) { + params += + "&" + + args[i] + + "=" + + URLEncoder.encode(args[i + 1].toString(), "UTF-8"); + } + + URL urlobj = new URL(url); + if ("GET".equals(method)) urlobj = new URL(url + "?" + params); + HttpURLConnection con = (HttpURLConnection) urlobj.openConnection(); + con.setRequestMethod(method); + if (!"GET".equals(method)) { + con.setDoOutput(true); + DataOutputStream out = new DataOutputStream(con.getOutputStream()); + out.writeBytes(params); + out.flush(); + out.close(); + } + + String result = IOUtils.toString(con.getInputStream()); + con.disconnect(); + return result; + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + public boolean removeProductType(ProductType type) { + String result = + this.query("DELETE", "remove", "id", type.getProductTypeId()); + return Boolean.parseBoolean(result); + } + + public HashMap getParentTypeMap() { + String result = this.query("GET", "parentmap"); + HashMap map = gson.fromJson(result, HashMap.class); + HashMap ptypeMap = new HashMap(); + for (Object key : map.keySet()) { + ptypeMap.put((String) key, (String) map.get(key)); + } + return ptypeMap; + } + + public boolean addParentForProductType(ProductType type, String parentId) { + String result = + this.query( + "POST", + "parent/add", + "id", + type.getProductTypeId(), + "parentId", + parentId + ); + return Boolean.parseBoolean(result); + } + + public boolean removeParentForProductType(ProductType type) { + String result = + this.query("DELETE", "parent/remove", "id", type.getProductTypeId()); + return Boolean.parseBoolean(result); + } + + public boolean addElementsForProductType( + ProductType type, + List elementList + ) { + String elementIds = ""; + for (Element element : elementList) elementIds += + (elementIds != "" ? "," : "") + element.getElementId(); + String result = + this.query( + "POST", + "elements/add", + "id", + type.getProductTypeId(), + "elementIds", + elementIds + ); + return Boolean.parseBoolean(result); + } + + public List getElementsForProductType( + ProductType type, + boolean direct + ) { + String result = + this.query( + "GET", + "elements", + "id", + type.getProductTypeId(), + "direct", + direct + ); + List elementList = new ArrayList(); + Object[] elementIds = gson.fromJson(result, ArrayList.class).toArray(); + for (Object elementId : elementIds) { + String id = (String) elementId; + elementList.add(new Element(id, id, "", "", "Automatically Added", "")); + } + return elementList; + } + + public boolean removeAllElementsForProductType(ProductType type) { + String result = + this.query( + "DELETE", + "elements/remove/all", + "id", + type.getProductTypeId() + ); + return Boolean.parseBoolean(result); + } + + public boolean removeElementsForProductType( + ProductType type, + List elementList + ) { + String elementIds = ""; + for (Element element : elementList) elementIds += + (elementIds != "" ? "," : "") + element.getElementId(); + String result = + this.query( + "DELETE", + "elements/remove", + "id", + type.getProductTypeId(), + "elementIds", + elementIds + ); + return Boolean.parseBoolean(result); + } + + public List getProductTypeIdsHavingElement(Element el) { + String result = + this.query("GET", "typeswithelement/" + el.getElementId(), "id"); + Object[] typeIds = gson.fromJson(result, ArrayList.class).toArray(); + return Arrays.asList((String[]) typeIds); + } } diff --git a/engines/oodt/task/src/main/java/org/apache/oodt/cas/workflow/misc/WingsTask.java b/engines/oodt/task/src/main/java/org/apache/oodt/cas/workflow/misc/WingsTask.java index b351ccae..68a3cfb7 100644 --- a/engines/oodt/task/src/main/java/org/apache/oodt/cas/workflow/misc/WingsTask.java +++ b/engines/oodt/task/src/main/java/org/apache/oodt/cas/workflow/misc/WingsTask.java @@ -48,12 +48,7 @@ import java.util.Properties; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; - import org.apache.commons.io.FileUtils; - -//OODT imports -import org.apache.oodt.cas.workflow.structs.WorkflowTaskInstance; -import org.apache.oodt.cas.workflow.structs.WorkflowTaskConfiguration; import org.apache.oodt.cas.filemgr.datatransfer.DataTransfer; import org.apache.oodt.cas.filemgr.datatransfer.RemoteDataTransferFactory; import org.apache.oodt.cas.filemgr.structs.Product; @@ -63,11 +58,14 @@ import org.apache.oodt.cas.filemgr.structs.exceptions.RepositoryManagerException; import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient; import org.apache.oodt.cas.metadata.Metadata; +import org.apache.oodt.cas.workflow.structs.WorkflowTaskConfiguration; +//OODT imports +import org.apache.oodt.cas.workflow.structs.WorkflowTaskInstance; /** * @author Varun Ratnakar * @version $Revsion$ - * + * *

* A Wings Task (http://www.wings-workflows.org) *

@@ -75,14 +73,13 @@ public class WingsTask implements WorkflowTaskInstance { /** - * + * */ - public WingsTask() { - } + public WingsTask() {} /* * (non-Javadoc) - * + * * @see * org.apache.oodt.cas.workflow.structs.WorkflowTaskInstance#run(java.util * .Map, org.apache.oodt.cas.workflow.structs.WorkflowTaskConfiguration) @@ -111,17 +108,16 @@ public void run(Metadata metadata, WorkflowTaskConfiguration config) { String logfile = props.getProperty("LOGFILE"); String wlogfile = props.getProperty("W_LOGFILE"); String tplid = wlogfile.replace(".log", ""); - + PrintStream wlogout = null; PrintStream logout = null; XmlRpcFileManagerClient fmclient = null; try { - fmclient = new XmlRpcFileManagerClient(new URL( - fmurl)); + fmclient = new XmlRpcFileManagerClient(new URL(fmurl)); DataTransfer dt = new RemoteDataTransferFactory().createDataTransfer(); dt.setFileManagerUrl(new URL(fmurl)); - + // Check if outputs already exist in the file manager boolean outputs_already_present = true; for (String op : outputs) { @@ -129,20 +125,17 @@ public void run(Metadata metadata, WorkflowTaskConfiguration config) { Product prod = null; try { prod = fmclient.getProductById(prodid); - } - catch (Exception e) {} - if(prod == null) { + } catch (Exception e) {} + if (prod == null) { outputs_already_present = false; } } // If outputs already present, no need to execute - if(outputs_already_present) - return; - + if (outputs_already_present) return; File tmpdir = File.createTempFile("oodt-run-", ""); - if (tmpdir.delete() && tmpdir.mkdirs()) - jobdir = tmpdir.getAbsolutePath() + File.separator; + if (tmpdir.delete() && tmpdir.mkdirs()) jobdir = + tmpdir.getAbsolutePath() + File.separator; argstring = argstring.replace(origjobdir, jobdir); @@ -151,8 +144,14 @@ public void run(Metadata metadata, WorkflowTaskConfiguration config) { wlogout.println(jobid + " (" + tname + "): RUNNING"); wlogout.close(); - this.uploadProduct(wlogfile, wlogfile, "GenericFile", - new File(jobdir + wlogfile), new Metadata(), fmclient); + this.uploadProduct( + wlogfile, + wlogfile, + "GenericFile", + new File(jobdir + wlogfile), + new Metadata(), + fmclient + ); wlogout = new PrintStream(new FileOutputStream(jobdir + wlogfile, true)); logout.println("[INFO]: Component Initializing"); @@ -177,16 +176,16 @@ public void run(Metadata metadata, WorkflowTaskConfiguration config) { cprod.setProductReferences(fmclient.getProductReferences(cprod)); dt.retrieveProduct(cprod, compdir); String scriptPath = null; - for(File czip : compdir.listFiles()) { - if(czip.getName().endsWith(".zip")) { + for (File czip : compdir.listFiles()) { + if (czip.getName().endsWith(".zip")) { this.unZipIt(czip.getAbsolutePath(), compdir.getAbsolutePath()); - File tmpf = new File(compdir.getAbsolutePath() + File.separator + "run"); - if(!tmpf.exists()) - tmpf = new File(compdir.getAbsolutePath() + File.separator + "run.bat"); + File tmpf = new File( + compdir.getAbsolutePath() + File.separator + "run" + ); + if (!tmpf.exists()) tmpf = + new File(compdir.getAbsolutePath() + File.separator + "run.bat"); scriptPath = tmpf.getAbsolutePath(); - } - else - scriptPath = czip.getAbsolutePath(); + } else scriptPath = czip.getAbsolutePath(); } File scriptf = new File(scriptPath); scriptf.setExecutable(true); @@ -213,26 +212,24 @@ public void run(Metadata metadata, WorkflowTaskConfiguration config) { } process.waitFor(); int exitStatus = process.exitValue(); - if (exitStatus != 0) - throw new Exception( - "[ERROR] Component failed with a non-zero exit code"); + if (exitStatus != 0) throw new Exception( + "[ERROR] Component failed with a non-zero exit code" + ); // Ingest output files to file manager for (String op : outputs) { File f = new File(jobdir + op); - + File metf = new File(jobdir + op + ".met"); HashMap cmeta = new HashMap(); - if(metf.exists()) { - for(Object ln : FileUtils.readLines(metf)) { + if (metf.exists()) { + for (Object ln : FileUtils.readLines(metf)) { String metline = (String) ln; String[] kv = metline.split("\\s*=\\s*"); - if(kv.length == 2) - cmeta.put(kv[0], kv[1]); + if (kv.length == 2) cmeta.put(kv[0], kv[1]); } } - if (!f.exists()) - throw new Exception("[ERROR] Missing Output " + op); + if (!f.exists()) throw new Exception("[ERROR] Missing Output " + op); if (f.exists()) { logout.println("[INFO] Putting Output into File Manager: " + op); @@ -241,12 +238,12 @@ public void run(Metadata metadata, WorkflowTaskConfiguration config) { Metadata meta = metadata.getSubMetadata(op); String prodtypeid = meta.getMetadata(typeid); meta.removeMetadata(typeid); - + // Override metadata with custom metadata (if any) - for(String key : meta.getAllKeys()) { + for (String key : meta.getAllKeys()) { String[] nsname = key.split("#"); - if(nsname.length == 2) { - if(cmeta.containsKey(nsname[1])) { + if (nsname.length == 2) { + if (cmeta.containsKey(nsname[1])) { meta.removeMetadata(key); meta.addMetadata(key, cmeta.get(nsname[1])); } @@ -254,15 +251,21 @@ public void run(Metadata metadata, WorkflowTaskConfiguration config) { } // Upload output to file manager - String prodid = fmprefix + op; + String prodid = fmprefix + op; this.uploadProduct(prodid, op, prodtypeid, f, meta, fmclient); } - + if (metf.exists()) { String metname = op + ".met"; String prodid = fmprefix + metname; - this.uploadProduct(prodid, metname, "GenericFile", - metf, new Metadata(), fmclient); + this.uploadProduct( + prodid, + metname, + "GenericFile", + metf, + new Metadata(), + fmclient + ); } } logout.println("SUCCESS: Component finished successfully !"); @@ -279,12 +282,24 @@ public void run(Metadata metadata, WorkflowTaskConfiguration config) { } } try { - if(fmclient != null) { - this.uploadProduct(wlogfile, wlogfile, "GenericFile", - new File(jobdir + wlogfile), new Metadata(), fmclient); + if (fmclient != null) { + this.uploadProduct( + wlogfile, + wlogfile, + "GenericFile", + new File(jobdir + wlogfile), + new Metadata(), + fmclient + ); String logid = tplid + "-" + logfile; - this.uploadProduct(logid, logid, "GenericFile", - new File(jobdir + logfile), new Metadata(), fmclient); + this.uploadProduct( + logid, + logid, + "GenericFile", + new File(jobdir + logfile), + new Metadata(), + fmclient + ); } } catch (CatalogException e) { e.printStackTrace(); @@ -293,33 +308,43 @@ public void run(Metadata metadata, WorkflowTaskConfiguration config) { } } - private void uploadProduct(String prodid, String prodname, String prodtypeid, - File f, Metadata meta, XmlRpcFileManagerClient fmclient) - throws CatalogException, RepositoryManagerException { + private void uploadProduct( + String prodid, + String prodname, + String prodtypeid, + File f, + Metadata meta, + XmlRpcFileManagerClient fmclient + ) throws CatalogException, RepositoryManagerException { ArrayList refs = new ArrayList(); long filesize = f.length(); refs.add(new Reference(f.toURI().toString(), "", filesize)); Product prod = null; - try { prod = fmclient.getProductById(prodid); } - catch (Exception e) {} - if(prod != null) - fmclient.removeProduct(prod); + try { + prod = fmclient.getProductById(prodid); + } catch (Exception e) {} + if (prod != null) fmclient.removeProduct(prod); ProductType type = null; try { type = fmclient.getProductTypeById(prodtypeid); - } - catch (Exception e) { + } catch (Exception e) { String desc = ""; String ver = "org.apache.oodt.cas.filemgr.versioning.BasicVersioner"; String repo = "file:///tmp"; type = new ProductType(prodtypeid, prodtypeid, desc, repo, ver); fmclient.addProductType(type); } - - prod = new Product(prodname, type, - Product.STRUCTURE_FLAT, Product.STATUS_TRANSFER, refs); + + prod = + new Product( + prodname, + type, + Product.STRUCTURE_FLAT, + Product.STATUS_TRANSFER, + refs + ); prod.setProductId(prodid); // Ingest new product @@ -329,35 +354,35 @@ private void uploadProduct(String prodid, String prodname, String prodtypeid, e.printStackTrace(); } } - + private void unZipIt(String zipFile, String outputFolder) { byte[] buffer = new byte[1024]; - try{ + try { File folder = new File(outputFolder); - if(!folder.exists()){ + if (!folder.exists()) { folder.mkdir(); } ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFile)); ZipEntry ze = zis.getNextEntry(); - while(ze!=null) { + while (ze != null) { String fileName = ze.getName(); File newFile = new File(outputFolder + File.separator + fileName); new File(newFile.getParent()).mkdirs(); - FileOutputStream fos = new FileOutputStream(newFile); + FileOutputStream fos = new FileOutputStream(newFile); int len; while ((len = zis.read(buffer)) > 0) { fos.write(buffer, 0, len); } - fos.close(); + fos.close(); ze = zis.getNextEntry(); } zis.closeEntry(); zis.close(); - } catch(IOException ex){ - ex.printStackTrace(); + } catch (IOException ex) { + ex.printStackTrace(); } - } + } private ArrayList fetchFromProps(Properties props, String argtype) { ArrayList args = new ArrayList(); diff --git a/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/client/WmServicesClient.java b/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/client/WmServicesClient.java index 0d314807..4f46b3e2 100644 --- a/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/client/WmServicesClient.java +++ b/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/client/WmServicesClient.java @@ -21,7 +21,6 @@ import java.net.HttpURLConnection; import java.net.URL; import java.net.URLEncoder; - import org.apache.commons.io.IOUtils; import org.apache.oodt.cas.wmservices.repository.PackagedWorkflowManager; import org.apache.oodt.cas.workflow.structs.Workflow; @@ -29,16 +28,17 @@ /** * Client class to connect to wmservices - * + * * @author vratnakar */ public class WmServicesClient { + String serverurl; String service = "/service/"; /** * Constructor - * + * * @param workflowDir * the directory where workflow files exist */ @@ -48,7 +48,7 @@ public WmServicesClient(String serverurl) { /** * Add a Packaged workflow - * + * * @param workflowId * id of the workflow * @param workflow @@ -57,43 +57,56 @@ public WmServicesClient(String serverurl) { * @throws RepositoryException */ public boolean addPackagedWorkflow(String workflowId, Workflow workflow) - throws RepositoryException { - + throws RepositoryException { try { PackagedWorkflowManager editor = new PackagedWorkflowManager(); String xml = editor.serializeWorkflow(workflow); // Now Make a POST call to the servlet with this xml - String result = this.query("POST", "addPackagedRepositoryWorkflow", "workflowID", - workflowId, "workflowXML", xml); + String result = + this.query( + "POST", + "addPackagedRepositoryWorkflow", + "workflowID", + workflowId, + "workflowXML", + xml + ); return Boolean.parseBoolean(result); } catch (Exception e) { - throw new RepositoryException("Could not add packaged workflow: " - + e.getMessage()); + throw new RepositoryException( + "Could not add packaged workflow: " + e.getMessage() + ); } } /** * Delete a Packaged workflow - * + * * @param workflowId * id of the workflow to be deleted * @return true if operation successful * @throws RepositoryException */ public boolean deletePackagedWorkflow(String workflowId) - throws RepositoryException { + throws RepositoryException { try { // Now Make a POST call to the servlet with this xml - String result = this.query("POST", "deletePackagedRepositoryWorkflow", "workflowID", - workflowId); + String result = + this.query( + "POST", + "deletePackagedRepositoryWorkflow", + "workflowID", + workflowId + ); return Boolean.parseBoolean(result); } catch (Exception e) { - throw new RepositoryException("Could not delete packaged workflow: " - + e.getMessage()); + throw new RepositoryException( + "Could not delete packaged workflow: " + e.getMessage() + ); } } - + // Private functions private String query(String method, String op, Object... args) { @@ -101,10 +114,9 @@ private String query(String method, String op, Object... args) { try { String params = ""; for (int i = 0; i < args.length; i += 2) { - if (i > 0) - params += "&"; - params += args[i] + "=" - + URLEncoder.encode(args[i + 1].toString(), "UTF-8"); + if (i > 0) params += "&"; + params += + args[i] + "=" + URLEncoder.encode(args[i + 1].toString(), "UTF-8"); } if ("GET".equals(method)) { URL urlobj = new URL(url + "?" + params); @@ -123,7 +135,6 @@ private String query(String method, String op, Object... args) { con.disconnect(); return result; } - } catch (Exception e) { e.printStackTrace(); } diff --git a/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/repository/PackagedWorkflowManager.java b/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/repository/PackagedWorkflowManager.java index 2d78b896..10052380 100644 --- a/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/repository/PackagedWorkflowManager.java +++ b/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/repository/PackagedWorkflowManager.java @@ -22,12 +22,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; - import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; - -import org.w3c.dom.Document; -import org.w3c.dom.Element; import org.apache.commons.io.FileUtils; import org.apache.oodt.cas.workflow.repository.PackagedWorkflowRepository; import org.apache.oodt.cas.workflow.structs.Graph; @@ -38,36 +34,39 @@ import org.apache.oodt.cas.workflow.structs.WorkflowTaskConfiguration; import org.apache.oodt.cas.workflow.structs.exceptions.RepositoryException; import org.apache.oodt.commons.xml.XMLUtils; +import org.w3c.dom.Document; +import org.w3c.dom.Element; /** * Helper class to handle PackagedWorkflowRepository workflows - * + * * @author vratnakar */ public class PackagedWorkflowManager { + private PackagedWorkflowRepository repo; /** * Constructor - * + * * @param workflowDir * directory where packaged workflows exist * @throws InstantiationException */ @SuppressWarnings("unchecked") - public PackagedWorkflowManager() - throws InstantiationException { + public PackagedWorkflowManager() throws InstantiationException { this.repo = new PackagedWorkflowRepository(Collections.EMPTY_LIST); } /** * Add a workflow to the repository - * + * * @param workflow * a {@Link Workflow} to add into the repository * @throws RepositoryException */ - public void addWorkflow(Workflow workflow, String workflowDir) throws RepositoryException { + public void addWorkflow(Workflow workflow, String workflowDir) + throws RepositoryException { this.loadTasksToRepo(workflow); String workflowId = this.repo.addWorkflow(workflow); String filePath = workflowDir + File.separator + workflowId + ".xml"; @@ -76,12 +75,13 @@ public void addWorkflow(Workflow workflow, String workflowDir) throws Repository /** * Serialize a workflow - * + * * @param workflow * @return XML representation of the workflow * @throws RepositoryException */ - public String serializeWorkflow(Workflow workflow) throws RepositoryException { + public String serializeWorkflow(Workflow workflow) + throws RepositoryException { try { this.loadTasksToRepo(workflow); String workflowId = this.repo.addWorkflow(workflow); @@ -91,14 +91,15 @@ public String serializeWorkflow(Workflow workflow) throws RepositoryException { f.delete(); return workflowXML; } catch (Exception e) { - throw new RepositoryException("Failed to serialize workflow: " - + e.getMessage()); + throw new RepositoryException( + "Failed to serialize workflow: " + e.getMessage() + ); } } /** * Parse a workflow - * + * * @param workflowID * workflow id * @param workflowXML @@ -107,17 +108,19 @@ public String serializeWorkflow(Workflow workflow) throws RepositoryException { * @throws RepositoryException */ public Workflow parsePackagedWorkflow(String workflowID, String workflowXML) - throws RepositoryException { + throws RepositoryException { try { File tmpfile = File.createTempFile("tempworkflow-", "-packaged"); FileUtils.writeStringToFile(tmpfile, workflowXML); PackagedWorkflowRepository tmprepo = new PackagedWorkflowRepository( - Collections.singletonList(tmpfile)); + Collections.singletonList(tmpfile) + ); tmpfile.delete(); return tmprepo.getWorkflowById(workflowID); } catch (Exception e) { - throw new RepositoryException("Failed to parse workflow xml: " - + e.getMessage()); + throw new RepositoryException( + "Failed to parse workflow xml: " + e.getMessage() + ); } } @@ -125,30 +128,31 @@ public Workflow parsePackagedWorkflow(String workflowID, String workflowXML) private void loadTasksToRepo(Workflow workflow) throws RepositoryException { for (WorkflowTask task : workflow.getTasks()) { - if (this.repo.getTaskById(task.getTaskId()) == null) - this.repo.addTask(task); + if (this.repo.getTaskById(task.getTaskId()) == null) this.repo.addTask( + task + ); } } @SuppressWarnings("unchecked") private void saveWorkflow(String workflowId, String filePath) - throws RepositoryException { + throws RepositoryException { List pcwlist = new ArrayList(); // Check if the workflow exists - ParentChildWorkflow pcw = (ParentChildWorkflow) repo - .getWorkflowById(workflowId); + ParentChildWorkflow pcw = (ParentChildWorkflow) repo.getWorkflowById( + workflowId + ); if (pcw == null) { // Else check if this workflow Id is found in the event map // - It would be here if the top task is a parallel task pcwlist = repo.getWorkflowsForEvent(workflowId); } - if (pcw != null) - pcwlist.add(pcw); + if (pcw != null) pcwlist.add(pcw); - if (pcwlist.isEmpty()) - throw new RepositoryException("Cannot find " + workflowId - + " in the repository"); + if (pcwlist.isEmpty()) throw new RepositoryException( + "Cannot find " + workflowId + " in the repository" + ); try { DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); @@ -163,17 +167,15 @@ private void saveWorkflow(String workflowId, String filePath) // Create the workflow itself if (pcwlist.size() == 1) { - Element elem = this.addGraphToDocument(document, pcwlist.get(0) - .getGraph()); - if (elem != null) - rootElem.appendChild(elem); + Element elem = + this.addGraphToDocument(document, pcwlist.get(0).getGraph()); + if (elem != null) rootElem.appendChild(elem); } else { Element parallelElem = document.createElement("parallel"); parallelElem.setAttribute("id", workflowId); for (ParentChildWorkflow cpcw : pcwlist) { Element elem = this.addGraphToDocument(document, cpcw.getGraph()); - if (elem != null) - parallelElem.appendChild(elem); + if (elem != null) parallelElem.appendChild(elem); } rootElem.appendChild(parallelElem); } @@ -181,15 +183,17 @@ private void saveWorkflow(String workflowId, String filePath) for (Object obj : repo.getTasksByWorkflowId(workflowId)) { WorkflowTask task = (WorkflowTask) obj; Element elem = this.createTaskElement(document, task); - if (elem != null) - rootElem.appendChild(elem); + if (elem != null) rootElem.appendChild(elem); } - XMLUtils.writeXmlToStream(document, new FileOutputStream(new File( - filePath))); + XMLUtils.writeXmlToStream( + document, + new FileOutputStream(new File(filePath)) + ); } catch (Exception e) { e.printStackTrace(); - throw new RepositoryException("Could not save workflow. " - + e.getMessage()); + throw new RepositoryException( + "Could not save workflow. " + e.getMessage() + ); } } @@ -198,10 +202,12 @@ private Element addGraphToDocument(Document document, Graph g) { Element elem = null; if ("parallel".equals(exetype) || "sequential".equals(exetype)) { elem = document.createElement(exetype); - if (g.getModelId() != null && !g.getModelId().equals("")) - elem.setAttribute("id", g.getModelId()); - if (g.getModelName() != null && !g.getModelName().equals("")) - elem.setAttribute("name", g.getModelName()); + if ( + g.getModelId() != null && !g.getModelId().equals("") + ) elem.setAttribute("id", g.getModelId()); + if ( + g.getModelName() != null && !g.getModelName().equals("") + ) elem.setAttribute("name", g.getModelName()); } else if ("task".equals(exetype) || "condition".equals(exetype)) { elem = document.createElement(exetype); if (g.getModelIdRef() != null && !g.getModelIdRef().equals("")) { @@ -249,8 +255,10 @@ private Element createTaskElement(Document document, WorkflowTask task) { taskElem.appendChild(conditionsElem); } - if (task.getRequiredMetFields() != null - && task.getRequiredMetFields().size() > 0) { + if ( + task.getRequiredMetFields() != null && + task.getRequiredMetFields().size() > 0 + ) { Element reqMetFieldsElem = document.createElement("requiredMetFields"); for (Object obj : task.getRequiredMetFields()) { String metField = (String) obj; diff --git a/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/resources/Resource.java b/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/resources/Resource.java index 8e3363ed..2c60c5bb 100644 --- a/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/resources/Resource.java +++ b/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/resources/Resource.java @@ -20,16 +20,16 @@ import java.io.File; import java.util.logging.Level; import java.util.logging.Logger; - import javax.servlet.ServletContext; import javax.ws.rs.core.Context; - import org.apache.oodt.cas.wmservices.servlets.WmServicesServlet; import org.apache.oodt.cas.workflow.system.XmlRpcWorkflowManagerClient; public abstract class Resource { - private static final Logger LOGGER = Logger.getLogger(Resource.class - .getName()); + + private static final Logger LOGGER = Logger.getLogger( + Resource.class.getName() + ); // Servlet context @Context @@ -42,17 +42,18 @@ public abstract class Resource { * if an object cannot be retrieved from the context attribute */ public File getContextPkgReposDir() throws Exception { - Object repositoryDirObject = context - .getAttribute(WmServicesServlet.ATTR_NAME_PKG_REPO_DIR); - if (repositoryDirObject != null - && repositoryDirObject instanceof File) { + Object repositoryDirObject = context.getAttribute( + WmServicesServlet.ATTR_NAME_PKG_REPO_DIR + ); + if (repositoryDirObject != null && repositoryDirObject instanceof File) { return (File) repositoryDirObject; } - String message = "Unable to retrieve packaged repository directory from the servlet context."; + String message = + "Unable to retrieve packaged repository directory from the servlet context."; LOGGER.log(Level.WARNING, message); throw new Exception(message); } - + /** * Gets the servlet's workflow manager client instance from the servlet * context. @@ -63,20 +64,23 @@ public File getContextPkgReposDir() throws Exception { */ public XmlRpcWorkflowManagerClient getContextClient() throws Exception { // Get the workflow manager client from the servlet context. - Object clientObject = context - .getAttribute(WmServicesServlet.ATTR_NAME_CLIENT); - if (clientObject != null - && clientObject instanceof XmlRpcWorkflowManagerClient) { + Object clientObject = context.getAttribute( + WmServicesServlet.ATTR_NAME_CLIENT + ); + if ( + clientObject != null && + clientObject instanceof XmlRpcWorkflowManagerClient + ) { return (XmlRpcWorkflowManagerClient) clientObject; } - String message = "Unable to retrieve workflow manager client from the " - + "servlet context."; + String message = + "Unable to retrieve workflow manager client from the " + + "servlet context."; LOGGER.log(Level.WARNING, message); throw new Exception(message); } - /** * Sets the servlet context. * @param context diff --git a/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/resources/WorkflowResource.java b/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/resources/WorkflowResource.java index 4a3c4c08..117c70e9 100644 --- a/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/resources/WorkflowResource.java +++ b/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/resources/WorkflowResource.java @@ -37,12 +37,10 @@ import java.io.File; import java.util.logging.Level; import java.util.logging.Logger; - import javax.ws.rs.DELETE; import javax.ws.rs.FormParam; import javax.ws.rs.POST; import javax.ws.rs.Path; - import org.apache.oodt.cas.wmservices.repository.PackagedWorkflowManager; import org.apache.oodt.cas.workflow.structs.Workflow; @@ -52,14 +50,15 @@ * @author vratnakar */ public class WorkflowResource extends Resource { - private static final Logger LOGGER = Logger.getLogger(WorkflowResource.class - .getName()); + + private static final Logger LOGGER = Logger.getLogger( + WorkflowResource.class.getName() + ); /** * Default constructor. */ - public WorkflowResource() { - } + public WorkflowResource() {} /** * Add Packaged Repository Workflow @@ -73,13 +72,16 @@ public WorkflowResource() { @POST @Path("/addPackagedRepositoryWorkflow") public boolean addPackagedRepositoryWorkflow( - @FormParam("workflowID") String workflowID, - @FormParam("workflowXML") String workflowXML) throws Exception { + @FormParam("workflowID") String workflowID, + @FormParam("workflowXML") String workflowXML + ) throws Exception { try { PackagedWorkflowManager pwmanager = new PackagedWorkflowManager(); - Workflow workflow = pwmanager.parsePackagedWorkflow(workflowID, workflowXML); - if(workflow == null) - return false; + Workflow workflow = pwmanager.parsePackagedWorkflow( + workflowID, + workflowXML + ); + if (workflow == null) return false; String workflowDir = this.getContextPkgReposDir().getAbsolutePath(); pwmanager.addWorkflow(workflow, workflowDir); return getContextClient().refreshRepository(); @@ -101,13 +103,12 @@ public boolean addPackagedRepositoryWorkflow( @DELETE @Path("/deletePackagedRepositoryWorkflow") public boolean deletePackagedRepositoryWorkflow( - @FormParam("workflowID") String workflowID) throws Exception { + @FormParam("workflowID") String workflowID + ) throws Exception { try { File wflowFile = getPackagedRepositoryWorkflowFile(workflowID); - if (wflowFile.delete()) - return getContextClient().refreshRepository(); - else - return false; + if (wflowFile.delete()) return getContextClient() + .refreshRepository(); else return false; } catch (Exception e) { String message = "Unable to delete workflow. "; message += e.getMessage(); @@ -118,8 +119,12 @@ public boolean deletePackagedRepositoryWorkflow( // Private functions private File getPackagedRepositoryWorkflowFile(String workflowID) - throws Exception { - return new File(this.getContextPkgReposDir().getAbsolutePath() - + File.separator + workflowID + ".xml"); + throws Exception { + return new File( + this.getContextPkgReposDir().getAbsolutePath() + + File.separator + + workflowID + + ".xml" + ); } } diff --git a/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/servlets/WmServicesServlet.java b/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/servlets/WmServicesServlet.java index c5effc67..3ba6d0ec 100644 --- a/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/servlets/WmServicesServlet.java +++ b/engines/oodt/wmservices/src/main/java/org/apache/oodt/cas/wmservices/servlets/WmServicesServlet.java @@ -22,35 +22,35 @@ import java.net.URL; import java.util.logging.Level; import java.util.logging.Logger; - import javax.servlet.ServletConfig; import javax.servlet.ServletContext; import javax.servlet.ServletException; - import org.apache.cxf.jaxrs.servlet.CXFNonSpringJaxrsServlet; import org.apache.oodt.cas.metadata.util.PathUtils; import org.apache.oodt.cas.workflow.system.XmlRpcWorkflowManagerClient; /** * Initialize workflow manager services servlet - * + * * Set the following Parameters in context.xml for the webapp: *
    *
  • workflow.url *
  • packagedRepo.dir.path *
- * + * * @author vratnakar */ public class WmServicesServlet extends CXFNonSpringJaxrsServlet { - private static final Logger LOGGER = Logger - .getLogger(WmServicesServlet.class.getName()); - + + private static final Logger LOGGER = Logger.getLogger( + WmServicesServlet.class.getName() + ); + private static final long serialVersionUID = -7830210280506307805L; // Default URL for the workflow manager private static final String DEFAULT_WM_URL = "http://localhost:9001"; - + // Servlet context parameter names. private static final String PARAM_NAME_URL = "workflow.url"; private static final String PARAM_NAME_PKGREPO_DIR = "packagedRepo.dir.path"; // For Packaged Repo @@ -60,14 +60,13 @@ public class WmServicesServlet extends CXFNonSpringJaxrsServlet { * workflow manager, a {@link XmlRpcWorkflowManagerClient} object. */ public static final String ATTR_NAME_CLIENT = "client"; - + /** * The name of the servlet context attribute that holds the workflow manager's * packaged repository directory: a {@link File} object. */ public static final String ATTR_NAME_PKG_REPO_DIR = "pkgRepoFilesDir"; - - + @Override public void init(ServletConfig configuration) throws ServletException { super.init(configuration); @@ -82,9 +81,13 @@ public void init(ServletConfig configuration) throws ServletException { url = new URL(PathUtils.replaceEnvVariables(urlParameter)); } else { // Try the default URL for the workflow manager. - LOGGER.log(Level.WARNING, - "Unable to find the servlet context parameter" + " (\"" - + PARAM_NAME_URL + "\") for the workflow manager's URL."); + LOGGER.log( + Level.WARNING, + "Unable to find the servlet context parameter" + + " (\"" + + PARAM_NAME_URL + + "\") for the workflow manager's URL." + ); url = new URL(DEFAULT_WM_URL); } // Attempt to connect the client to the workflow manager and if successful @@ -92,21 +95,28 @@ public void init(ServletConfig configuration) throws ServletException { XmlRpcWorkflowManagerClient client = new XmlRpcWorkflowManagerClient(url); context.setAttribute(ATTR_NAME_CLIENT, client); } catch (MalformedURLException e) { - LOGGER.log(Level.SEVERE, - "Encountered a malformed URL for the workflow manager.", e); + LOGGER.log( + Level.SEVERE, + "Encountered a malformed URL for the workflow manager.", + e + ); throw new ServletException(e); } // Initialize the repository directory String pkgRepoDirPath = context.getInitParameter(PARAM_NAME_PKGREPO_DIR); if (pkgRepoDirPath != null) { - File workflowDir = new File(PathUtils.replaceEnvVariables(pkgRepoDirPath)); + File workflowDir = new File( + PathUtils.replaceEnvVariables(pkgRepoDirPath) + ); if (workflowDir.exists() && workflowDir.isDirectory()) { context.setAttribute(ATTR_NAME_PKG_REPO_DIR, workflowDir); } else { - LOGGER.log(Level.SEVERE, - "Unable to locate the Packaged repository directory (" - + workflowDir.getAbsolutePath()); + LOGGER.log( + Level.SEVERE, + "Unable to locate the Packaged repository directory (" + + workflowDir.getAbsolutePath() + ); } } } diff --git a/engines/pegasus/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/pegasus/PegasusExecutionEngine.java b/engines/pegasus/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/pegasus/PegasusExecutionEngine.java index c347fe3b..bd182b58 100644 --- a/engines/pegasus/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/pegasus/PegasusExecutionEngine.java +++ b/engines/pegasus/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/pegasus/PegasusExecutionEngine.java @@ -27,17 +27,6 @@ import edu.isi.wings.execution.tools.api.ExecutionMonitorAPI; import edu.isi.wings.execution.tools.api.ExecutionResourceAPI; import edu.isi.wings.workflow.plan.api.ExecutionStep; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.log4j.Logger; - -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.DocumentBuilder; -import org.w3c.dom.Document; -import org.w3c.dom.NodeList; -import org.w3c.dom.Node; -import org.w3c.dom.Element; - import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -45,536 +34,616 @@ import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.log4j.Logger; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; -public class PegasusExecutionEngine implements PlanExecutionEngine, StepExecutionEngine { - String baseDir; - Properties props; - Thread monitoringThread; - //protected static ExecutorService executor = Executors.newWorkStealingPool(16); +public class PegasusExecutionEngine + implements PlanExecutionEngine, StepExecutionEngine { - PlanExecutionEngine planEngine; - StepExecutionEngine stepEngine; - PegasusWorkflowAdapter adapter; + String baseDir; + Properties props; + Thread monitoringThread; + //protected static ExecutorService executor = Executors.newWorkStealingPool(16); - private Logger log = null; + PlanExecutionEngine planEngine; + StepExecutionEngine stepEngine; + PegasusWorkflowAdapter adapter; - protected ExecutionLoggerAPI logger; - protected ExecutionMonitorAPI monitor; - protected ExecutionResourceAPI resource; + private Logger log = null; - public PegasusExecutionEngine(Properties props) { - this.props = props; - this.stepEngine = this; - this.planEngine = this; - this.log = Logger.getLogger(this.getClass()); - } + protected ExecutionLoggerAPI logger; + protected ExecutionMonitorAPI monitor; + protected ExecutionResourceAPI resource; - @Override - public void execute(RuntimePlan exe) { - String pegasusHome = props.getProperty("pegasus.home") + File.separator; - String siteCatalog = props.getProperty("pegasus.site-catalog"); - String site = props.getProperty("pegasus.site", "local"); - String storageDir = props.getProperty("pegasus.storage-dir", FileUtils.getTempDirectoryPath()) + File.separator; + public PegasusExecutionEngine(Properties props) { + this.props = props; + this.stepEngine = this; + this.planEngine = this; + this.log = Logger.getLogger(this.getClass()); + } - try { - exe.onStart(logger); - - // Create base directory - File baseDir = new File(storageDir + exe.getName()); - FileUtils.forceMkdir(baseDir); + @Override + public void execute(RuntimePlan exe) { + String pegasusHome = props.getProperty("pegasus.home") + File.separator; + String siteCatalog = props.getProperty("pegasus.site-catalog"); + String site = props.getProperty("pegasus.site", "local"); + String storageDir = + props.getProperty( + "pegasus.storage-dir", + FileUtils.getTempDirectoryPath() + ) + + File.separator; + + try { + exe.onStart(logger); - this.baseDir = baseDir.getAbsolutePath() + File.separator; - this.adapter = new PegasusWorkflowAdapter(this.props); + // Create base directory + File baseDir = new File(storageDir + exe.getName()); + FileUtils.forceMkdir(baseDir); - String submitDir = this.baseDir + "submit" + File.separator; + this.baseDir = baseDir.getAbsolutePath() + File.separator; + this.adapter = new PegasusWorkflowAdapter(this.props); - // Construct Pegasus Workflow, and submit it for execution - this.adapter.runWorkflow(exe, siteCatalog, site, this.baseDir); + String submitDir = this.baseDir + "submit" + File.separator; - // Start Monitoring thread - /* + // Construct Pegasus Workflow, and submit it for execution + this.adapter.runWorkflow(exe, siteCatalog, site, this.baseDir); + + // Start Monitoring thread + /* executor.submit(new ExecutionMonitoringThread(this, exe, logger, monitor, submitDir, pegasusHome)); */ - Thread monitoringThread = new Thread(new ExecutionMonitoringThread(this, exe, logger, monitor, - submitDir, pegasusHome)); - monitoringThread.start(); - - } catch (Exception e) { - exe.onEnd(this.logger, RuntimeInfo.Status.FAILURE, e.getMessage()); - log.error(e.getMessage(), e); - } - + Thread monitoringThread = new Thread( + new ExecutionMonitoringThread( + this, + exe, + logger, + monitor, + submitDir, + pegasusHome + ) + ); + monitoringThread.start(); + } catch (Exception e) { + exe.onEnd(this.logger, RuntimeInfo.Status.FAILURE, e.getMessage()); + log.error(e.getMessage(), e); } - - @Override - public void onStepEnd(RuntimePlan exe) { - // Do nothing + } + + @Override + public void onStepEnd(RuntimePlan exe) { + // Do nothing + } + + /** + * * Stops a Pegasus workflow by executing pegasus-remove + * + * @param exe + */ + @Override + public void abort(RuntimePlan exe) { + String storageDir = + props.getProperty( + "pegasus.storage-dir", + FileUtils.getTempDirectoryPath() + ) + + File.separator; + + // Abort plan + if (this.monitoringThread != null && this.monitoringThread.isAlive()) { + this.monitoringThread.interrupt(); } - /** - * * Stops a Pegasus workflow by executing pegasus-remove - * - * @param exe - */ - @Override - public void abort(RuntimePlan exe) { - String storageDir = props.getProperty("pegasus.storage-dir", FileUtils.getTempDirectoryPath()) + File.separator; - - // Abort plan - if (this.monitoringThread != null && this.monitoringThread.isAlive()) { - this.monitoringThread.interrupt(); - } - - this.adapter.stopWorkflow(exe, storageDir + exe.getName() + File.separator); + this.adapter.stopWorkflow(exe, storageDir + exe.getName() + File.separator); + } + + @Override + public void execute(RuntimeStep exe, RuntimePlan plan) { + // Execute Step: Do nothing + } + + @Override + public void abort(RuntimeStep exe) { + // Abort Step: Do nothing + } + + @Override + public void setPlanExecutionEngine(PlanExecutionEngine engine) { + this.planEngine = engine; + } + + @Override + public PlanExecutionEngine getPlanExecutionEngine() { + return this.planEngine; + } + + @Override + public int getMaxParallelSteps() { + return -1; + } + + @Override + public void setMaxParallelSteps(int num) { + return; + } + + @Override + public void setStepExecutionEngine(StepExecutionEngine engine) { + this.stepEngine = engine; + } + + @Override + public StepExecutionEngine getStepExecutionEngine() { + return this.stepEngine; + } + + @Override + public void setExecutionLogger(ExecutionLoggerAPI logger) { + this.logger = logger; + } + + @Override + public ExecutionLoggerAPI getExecutionLogger() { + return this.logger; + } + + @Override + public void setExecutionMonitor(ExecutionMonitorAPI monitor) { + this.monitor = monitor; + if (this.stepEngine != this) { + this.stepEngine.setExecutionMonitor(monitor); } - - @Override - public void execute(RuntimeStep exe, RuntimePlan plan) { - // Execute Step: Do nothing + } + + @Override + public ExecutionMonitorAPI getExecutionMonitor() { + return this.monitor; + } + + @Override + public void setExecutionResource(ExecutionResourceAPI resource) { + this.resource = resource; + if (this.stepEngine != this) { + this.stepEngine.setExecutionResource(resource); } - - @Override - public void abort(RuntimeStep exe) { - // Abort Step: Do nothing + } + + @Override + public ExecutionResourceAPI getExecutionResource() { + return this.resource; + } + + /** + * Returns an iterator which can be used to traverse a file one line at a time. + * + * @param fileName + * @param skip Skip first skip lines + * @return + * @throws IOException + */ + public Iterator lineIterator(String fileName, int skip) + throws IOException { + final BufferedReader r = new BufferedReader( + new FileReader(new File(fileName)) + ); + + for (int i = 0; i < skip; ++i) { + String s = r.readLine(); + if (s == null) { + break; + } } - @Override - public void setPlanExecutionEngine(PlanExecutionEngine engine) { - this.planEngine = engine; - } + Iterator it = new Iterator() { + private String line = null; + private boolean read = true; - @Override - public PlanExecutionEngine getPlanExecutionEngine() { - return this.planEngine; - } - - @Override - public int getMaxParallelSteps() { - return -1; - } + @Override + public boolean hasNext() { + try { + if (read) { + line = r.readLine(); + read = false; + } + + if (line == null) { + close(); + } + + return line != null; + } catch (IOException e) { + close(); + return false; + } + } + + @Override + public String next() { + if (hasNext()) { + read = true; + return line; + } else { + return null; + } + } - @Override - public void setMaxParallelSteps(int num) { - return; - } + private void close() { + if (r != null) { + try { + r.close(); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + } + } - @Override - public void setStepExecutionEngine(StepExecutionEngine engine) { - this.stepEngine = engine; - } + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; - @Override - public StepExecutionEngine getStepExecutionEngine() { - return this.stepEngine; - } + return it; + } - @Override - public void setExecutionLogger(ExecutionLoggerAPI logger) { - this.logger = logger; - } + class ExecutionMonitoringThread implements Runnable { - @Override - public ExecutionLoggerAPI getExecutionLogger() { - return this.logger; + int sleepTime = 10000; + int jobstateLogMark = 0; + String submitDir = null; + RuntimePlan plan = null; + String pegasusHome = null; + private Logger log = null; + Set failedJobs = null; + ExecutionLoggerAPI logger = null; + ExecutionMonitorAPI monitor = null; + PlanExecutionEngine planEngine = null; + Map jobMap = null; + Map jobFileLoc = null; + RuntimeInfo.Status workflowStatus = null; + + public ExecutionMonitoringThread( + PlanExecutionEngine planEngine, + RuntimePlan plan, + ExecutionLoggerAPI logger, + ExecutionMonitorAPI monitor, + String submitDir, + String pegasusHome + ) { + this.plan = plan; + this.logger = logger; + this.monitor = monitor; + this.submitDir = submitDir; + this.planEngine = planEngine; + this.pegasusHome = pegasusHome; + this.failedJobs = new HashSet(); + this.log = Logger.getLogger(this.getClass()); + this.jobMap = new HashMap(); + this.jobFileLoc = new HashMap(); } - @Override - public void setExecutionMonitor(ExecutionMonitorAPI monitor) { - this.monitor = monitor; - if (this.stepEngine != this) { - this.stepEngine.setExecutionMonitor(monitor); - } + private int getSleepTime() { + return sleepTime; } - @Override - public ExecutionMonitorAPI getExecutionMonitor() { - return this.monitor; - } + /** + * Write out the entire jobstate log file to the plan log. + */ + private void writeJobstateLog() { + try { + Iterator it = lineIterator(submitDir + "jobstate.log", 0); - @Override - public void setExecutionResource(ExecutionResourceAPI resource) { - this.resource = resource; - if (this.stepEngine != this) { - this.stepEngine.setExecutionResource(resource); + plan.getRuntimeInfo().addLog("Jobstate Log File ***************"); + while (it.hasNext()) { + plan.getRuntimeInfo().addLog(it.next()); } - } - - @Override - public ExecutionResourceAPI getExecutionResource() { - return this.resource; + plan.getRuntimeInfo().addLog("*********************************"); + } catch (IOException e) { + plan + .getRuntimeInfo() + .addLog("Exception while reading jobstate.log " + e.getMessage()); + } } /** - * Returns an iterator which can be used to traverse a file one line at a time. - * - * @param fileName - * @param skip Skip first skip lines - * @return - * @throws IOException + * Get most recent job retry count. i.e. If .out.002 then return 002. */ - public Iterator lineIterator(String fileName, int skip) throws IOException { - final BufferedReader r = new BufferedReader(new FileReader(new File(fileName))); - - for (int i = 0; i < skip; ++i) { - String s = r.readLine(); - if (s == null) { - break; - } + private String getRetryCount(String jobLoc, String jobName) { + int retryCount = 0; + + while (true) { + if ( + new File( + jobLoc + "/" + jobName + ".out" + String.format(".%03d", retryCount) + ) + .exists() + ) { + retryCount++; + } else { + if (retryCount > 0) { + retryCount--; + } + break; } + } - Iterator it = new Iterator() { - private String line = null; - private boolean read = true; - - @Override - public boolean hasNext() { - try { - if (read) { - line = r.readLine(); - read = false; - } - - if (line == null) { - close(); - } - - return line != null; - - } catch (IOException e) { - close(); - return false; - } - } - - @Override - public String next() { - if (hasNext()) { - read = true; - return line; - } else { - return null; - } - } - - private void close() { - if (r != null) { - try { - r.close(); - } catch (IOException e) { - log.error(e.getMessage(), e); - } - } - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - - return it; + return String.format(".%03d", retryCount); } - - class ExecutionMonitoringThread implements Runnable { - int sleepTime = 10000; - int jobstateLogMark = 0; - String submitDir = null; - RuntimePlan plan = null; - String pegasusHome = null; - private Logger log = null; - Set failedJobs = null; - ExecutionLoggerAPI logger = null; - ExecutionMonitorAPI monitor = null; - PlanExecutionEngine planEngine = null; - Map jobMap = null; - Map jobFileLoc = null; - RuntimeInfo.Status workflowStatus = null; - - public ExecutionMonitoringThread(PlanExecutionEngine planEngine, RuntimePlan plan, ExecutionLoggerAPI logger, - ExecutionMonitorAPI monitor, String submitDir, String pegasusHome) { - this.plan = plan; - this.logger = logger; - this.monitor = monitor; - this.submitDir = submitDir; - this.planEngine = planEngine; - this.pegasusHome = pegasusHome; - this.failedJobs = new HashSet(); - this.log = Logger.getLogger(this.getClass()); - this.jobMap = new HashMap(); - this.jobFileLoc = new HashMap(); - } - - private int getSleepTime() { - return sleepTime; + private String showStdOutErr(String jobLoc, String jobName) + throws IOException, Exception { + String ext = getRetryCount(jobLoc, jobName); + + File kickstartFile = new File(jobLoc + "/" + jobName + ".out" + ext); + log.debug( + "Reading " + + kickstartFile.getCanonicalPath() + + " Exists? " + + kickstartFile.exists() + ); + + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document doc = dBuilder.parse(kickstartFile); + + //optional, but recommended + //read this - http://stackoverflow.com/questions/13786607/normalization-in-dom-parsing-with-java-how-does-it-work + doc.getDocumentElement().normalize(); + + NodeList nodeList = doc.getElementsByTagName("statcall"); + StringBuffer out = new StringBuffer(); + StringBuffer err = new StringBuffer(); + StringBuffer sb = new StringBuffer(); + + for (int temp = 0; temp < nodeList.getLength(); temp++) { + Node n = nodeList.item(temp); + + if (n.getNodeType() == Node.ELEMENT_NODE) { + Element e = (Element) n; + + if ( + e.getAttribute("id").equals("stdout") && + e.getElementsByTagName("data").getLength() > 0 + ) { + out.append(e.getElementsByTagName("data").item(0).getTextContent()); + } else if ( + e.getAttribute("id").equals("stderr") && + e.getElementsByTagName("data").getLength() > 0 + ) { + err.append(e.getElementsByTagName("data").item(0).getTextContent()); + } } + } + + // Standard Output + if (out.length() > 0) { + sb.append(String.format("%n---Standard Output---%n")); + sb.append(out.toString().trim()); + sb.append(String.format("%n---Standard Output---%n")); + } + + // Standard Error + if (err.length() > 0) { + sb.append(String.format("%n---Standard Error---%n")); + sb.append(err.toString().trim()); + sb.append(String.format("%n---Standard Error---%n")); + } + + return sb.toString(); + } - /** - * Write out the entire jobstate log file to the plan log. - */ - private void writeJobstateLog() { - try { - Iterator it = lineIterator(submitDir + "jobstate.log", 0); - - plan.getRuntimeInfo().addLog("Jobstate Log File ***************"); - while (it.hasNext()) { - plan.getRuntimeInfo().addLog(it.next()); + /** + * Traverse the jobstate log and update the status of the workflow, and all the jobs. + * For jobs that Wings is aware off, update the job's step objects status attribute. + * For jobs that Wings is NOT aware off, like transfer, or cleanup jobs, update the overall workflow status, + * if any of them have failed. + */ + private void updateJobStatus() { + try { + String line; + String[] lineParts; + Iterator it = lineIterator( + submitDir + "jobstate.log", + jobstateLogMark + ); + + log.debug("Skipping " + jobstateLogMark + " line(s) of jobstate.log"); + + while (it.hasNext()) { + line = it.next(); + lineParts = line.split("\\s+"); + + String jobName = lineParts[1]; + String jobState = lineParts[2]; + String jobLoc = this.jobFileLoc.get(jobName); + RuntimeStep step = jobMap.get(jobName); + + jobstateLogMark += 1; + log.debug("Processing Jobstate Log line " + jobstateLogMark); + + if (jobName.equals("INTERNAL")) { + // 1457574965 INTERNAL *** DAGMAN_STARTED/FINISHED 1 *** + // 1457574967 INTERNAL *** MONITORD_STARTED/FINISHED 0 *** + log.debug("jobName is INTERNAL, skipping..."); + continue; + } else { + if (step == null) { + log.debug("Job: " + jobName + " not known to Wings"); + if ( + jobState.equals("JOB_FAILURE") || + jobState.equals("POST_SCRIPT_FAILURE") + ) { + failedJobs.add(jobName); + log.debug("Job: " + jobName + " failed (" + jobState + ")"); + } + } else { + boolean isFailed = false; + RuntimeInfo.Status newStatus = null; + RuntimeInfo.Status oldStatus = step.getRuntimeInfo().getStatus(); + + if (jobState.equals("SUBMIT")) { + newStatus = RuntimeInfo.Status.QUEUED; + if (oldStatus != newStatus) { + step.setRuntimePlan(plan); + step.onStart(logger); } - plan.getRuntimeInfo().addLog("*********************************"); - } catch (IOException e) { - plan.getRuntimeInfo().addLog("Exception while reading jobstate.log " + e.getMessage()); - } - } - - /** - * Get most recent job retry count. i.e. If .out.002 then return 002. - */ - private String getRetryCount(String jobLoc, String jobName) { - int retryCount = 0; - - while (true) { - if (new File(jobLoc + "/" + jobName + ".out" + String.format(".%03d", retryCount)).exists()) { - retryCount++; - } else { - if (retryCount > 0) { - retryCount--; - } - break; + } else if (jobState.equals("EXECUTE")) { + newStatus = RuntimeInfo.Status.RUNNING; + } else if (jobState.equals("POST_SCRIPT_SUCCESS")) { + newStatus = RuntimeInfo.Status.SUCCESS; + if (oldStatus != newStatus) { + step.getRuntimeInfo().addLog(showStdOutErr(jobLoc, jobName)); + step.onEnd(logger, newStatus, "Success"); } + } else if (jobState.equals("POST_SCRIPT_FAILURE")) { + isFailed = true; + newStatus = RuntimeInfo.Status.FAILURE; + if (oldStatus != newStatus) { + step.getRuntimeInfo().addLog(showStdOutErr(jobLoc, jobName)); + step.onEnd(logger, newStatus, "Failure"); + } + } else { + newStatus = oldStatus; + } + + if (isFailed) { + failedJobs.add(jobName); + } else { + failedJobs.remove(jobName); + } + + // Job status only contains jobs that Wings is aware of. + // Jobs created by Pegasus to manage data transfer, and cleanup are ignored. + step.getRuntimeInfo().setStatus(newStatus); + log.debug( + "Job Name: " + + jobName + + " Old Status: " + + oldStatus + + " New Status: " + + newStatus + ); } - - return String.format(".%03d", retryCount); + } } - private String showStdOutErr(String jobLoc, String jobName) throws IOException, Exception { - String ext = getRetryCount(jobLoc, jobName); - - File kickstartFile = new File(jobLoc + "/" + jobName + ".out" + ext); - log.debug("Reading " + kickstartFile.getCanonicalPath() + " Exists? " + kickstartFile.exists()); - - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); - Document doc = dBuilder.parse(kickstartFile); + workflowStatus = + failedJobs.size() > 0 + ? RuntimeInfo.Status.FAILURE + : RuntimeInfo.Status.SUCCESS; + log.debug("Overall Workflow Status: " + workflowStatus); + } catch (IOException e) { + plan + .getRuntimeInfo() + .addLog("Exception while reading jobstate.log " + e.getMessage()); + } catch (Exception e) { + e.printStackTrace(); + plan.getRuntimeInfo().addLog("Exception " + e.getMessage()); + } + } - //optional, but recommended - //read this - http://stackoverflow.com/questions/13786607/normalization-in-dom-parsing-with-java-how-does-it-work - doc.getDocumentElement().normalize(); + private String getJobName(RuntimeStep step) { + ExecutionStep eStep = step.getStep(); + String componentName = eStep + .getCodeBinding() + .getCodeDirectory() + .replaceAll(".*/", ""); + String stepname = componentName + "_" + step.getName(); + stepname = stepname.replaceAll("[^a-zA-Z0-9_\\-]", "_"); + System.out.println(stepname); + return stepname; + } - NodeList nodeList = doc.getElementsByTagName("statcall"); - StringBuffer out = new StringBuffer(); - StringBuffer err = new StringBuffer(); - StringBuffer sb = new StringBuffer(); + /** + * After Pegasus has planned a workflow, it would have added new jobs for file transfer, cleanup, etc. + * Wings isn't aware of these jobs yet. This methd will create dummy RuntimeSteps to represent these new jobs. + */ + private void registerPegasusJobsWithPlan() throws IOException { + RuntimeStep step = null; + ExecutionQueue queue = plan.getQueue(); - for (int temp = 0; temp < nodeList.getLength(); temp++) { - Node n = nodeList.item(temp); + Iterator it = FileUtils.iterateFiles( + new java.io.File(submitDir), + new String[] { "sub" }, + true + ); - if (n.getNodeType() == Node.ELEMENT_NODE) { - Element e = (Element) n; + log.debug("Iterating over " + submitDir + " to get all .sub files"); - if (e.getAttribute("id").equals("stdout") && e.getElementsByTagName("data").getLength() > 0) { - out.append(e.getElementsByTagName("data").item(0).getTextContent()); - } else if (e.getAttribute("id").equals("stderr") && e.getElementsByTagName("data").getLength() > 0) { - err.append(e.getElementsByTagName("data").item(0).getTextContent()); - } - } - } + while (it.hasNext()) { + final File file = it.next(); + final String sub = FilenameUtils.removeExtension(file.getName()); - // Standard Output - if (out.length() > 0) { - sb.append(String.format("%n---Standard Output---%n")); - sb.append(out.toString().trim()); - sb.append(String.format("%n---Standard Output---%n")); - } - - // Standard Error - if (err.length() > 0) { - sb.append(String.format("%n---Standard Error---%n")); - sb.append(err.toString().trim()); - sb.append(String.format("%n---Standard Error---%n")); - } - - return sb.toString(); + if (sub.endsWith(".condor")) { + log.debug("Ignoring " + sub + ".sub file"); + continue; } - /** - * Traverse the jobstate log and update the status of the workflow, and all the jobs. - * For jobs that Wings is aware off, update the job's step objects status attribute. - * For jobs that Wings is NOT aware off, like transfer, or cleanup jobs, update the overall workflow status, - * if any of them have failed. - */ - private void updateJobStatus() { - try { - String line; - String[] lineParts; - Iterator it = lineIterator(submitDir + "jobstate.log", jobstateLogMark); - - log.debug("Skipping " + jobstateLogMark + " line(s) of jobstate.log"); - - while (it.hasNext()) { - line = it.next(); - lineParts = line.split("\\s+"); - - String jobName = lineParts[1]; - String jobState = lineParts[2]; - String jobLoc = this.jobFileLoc.get(jobName); - RuntimeStep step = jobMap.get(jobName); - - jobstateLogMark += 1; - log.debug("Processing Jobstate Log line " + jobstateLogMark); - - if (jobName.equals("INTERNAL")) { - // 1457574965 INTERNAL *** DAGMAN_STARTED/FINISHED 1 *** - // 1457574967 INTERNAL *** MONITORD_STARTED/FINISHED 0 *** - log.debug("jobName is INTERNAL, skipping..."); - continue; - } else { - if (step == null) { - log.debug("Job: " + jobName + " not known to Wings"); - if (jobState.equals("JOB_FAILURE") || jobState.equals("POST_SCRIPT_FAILURE")) { - failedJobs.add(jobName); - log.debug("Job: " + jobName + " failed (" + jobState + ")"); - } - } else { - boolean isFailed = false; - RuntimeInfo.Status newStatus = null; - RuntimeInfo.Status oldStatus = step.getRuntimeInfo().getStatus(); - - if (jobState.equals("SUBMIT")) { - newStatus = RuntimeInfo.Status.QUEUED; - if(oldStatus != newStatus) { - step.setRuntimePlan(plan); - step.onStart(logger); - } - } else if (jobState.equals("EXECUTE")) { - newStatus = RuntimeInfo.Status.RUNNING; - - } else if (jobState.equals("POST_SCRIPT_SUCCESS")) { - newStatus = RuntimeInfo.Status.SUCCESS; - if (oldStatus != newStatus) { - step.getRuntimeInfo().addLog(showStdOutErr(jobLoc, jobName)); - step.onEnd(logger, newStatus, "Success"); - } - } else if (jobState.equals("POST_SCRIPT_FAILURE")) { - isFailed = true; - newStatus = RuntimeInfo.Status.FAILURE; - if (oldStatus != newStatus) { - step.getRuntimeInfo().addLog(showStdOutErr(jobLoc, jobName)); - step.onEnd(logger, newStatus, "Failure"); - } - } else { - newStatus = oldStatus; - } - - if (isFailed) { - failedJobs.add(jobName); - } else { - failedJobs.remove(jobName); - } - - // Job status only contains jobs that Wings is aware of. - // Jobs created by Pegasus to manage data transfer, and cleanup are ignored. - step.getRuntimeInfo().setStatus(newStatus); - log.debug("Job Name: " + jobName + " Old Status: " + oldStatus + " New Status: " + newStatus); - } - } - } - - workflowStatus = failedJobs.size() > 0 ? RuntimeInfo.Status.FAILURE : RuntimeInfo.Status.SUCCESS; - log.debug("Overall Workflow Status: " + workflowStatus); - } catch (IOException e) { - plan.getRuntimeInfo().addLog("Exception while reading jobstate.log " + e.getMessage()); - } catch (Exception e) { - e.printStackTrace(); - plan.getRuntimeInfo().addLog("Exception " + e.getMessage()); - } - } + if (!jobMap.containsKey(sub)) { + log.debug("Adding new runtime step for " + sub); + step = new RuntimeStep(plan.getNamespace() + sub); - private String getJobName(RuntimeStep step) { - ExecutionStep eStep = step.getStep(); - String componentName = eStep.getCodeBinding().getCodeDirectory().replaceAll(".*/", ""); - String stepname = componentName + "_" + step.getName(); - stepname = stepname.replaceAll("[^a-zA-Z0-9_\\-]", "_"); - System.out.println(stepname); - return stepname; + step.setRuntimeInfo(new RuntimeInfo()); + step.getRuntimeInfo().setStatus(null); + queue.addStep(step); + jobMap.put(sub, step); } - /** - * After Pegasus has planned a workflow, it would have added new jobs for file transfer, cleanup, etc. - * Wings isn't aware of these jobs yet. This methd will create dummy RuntimeSteps to represent these new jobs. - */ - private void registerPegasusJobsWithPlan() throws IOException { - RuntimeStep step = null; - ExecutionQueue queue = plan.getQueue(); - - Iterator it = FileUtils.iterateFiles(new java.io.File(submitDir), new String[]{"sub"}, true); - - log.debug("Iterating over " + submitDir + " to get all .sub files"); - - while (it.hasNext()) { - final File file = it.next(); - final String sub = FilenameUtils.removeExtension(file.getName()); - - if (sub.endsWith(".condor")) { - log.debug("Ignoring " + sub + ".sub file"); - continue; - } - - if (!jobMap.containsKey(sub)) { - log.debug("Adding new runtime step for " + sub); - step = new RuntimeStep(plan.getNamespace() + sub); - - step.setRuntimeInfo(new RuntimeInfo()); - step.getRuntimeInfo().setStatus(null); - queue.addStep(step); - jobMap.put(sub, step); - } + jobFileLoc.put(sub, file.getParentFile().getCanonicalPath()); + log.debug( + "Job " + sub + " Sub File " + file.getParentFile().getCanonicalPath() + ); + } + } - jobFileLoc.put(sub, file.getParentFile().getCanonicalPath()); - log.debug("Job " + sub + " Sub File " + file.getParentFile().getCanonicalPath()); - } + @Override + public void run() { + // Workflow is successful by default + workflowStatus = RuntimeInfo.Status.SUCCESS; + + try { + File done = new File(submitDir + "monitord.done"); + + // Initialize status of all jobs to UNREADY, i.e. NULL + for (RuntimeStep step : plan.getQueue().getAllSteps()) { + step.getRuntimeInfo().setStatus(null); + String jobName = getJobName(step); + jobMap.put(jobName, step); } - @Override - public void run() { - // Workflow is successful by default - workflowStatus = RuntimeInfo.Status.SUCCESS; - - try { - File done = new File(submitDir + "monitord.done"); - - // Initialize status of all jobs to UNREADY, i.e. NULL - for (RuntimeStep step : plan.getQueue().getAllSteps()) { - step.getRuntimeInfo().setStatus(null); - String jobName = getJobName(step); - jobMap.put(jobName, step); - } - - registerPegasusJobsWithPlan(); - - while (true) { - updateJobStatus(); + registerPegasusJobsWithPlan(); - // Workflow finished? i.e. Presence of monitord.done file - if (!done.exists()) { - Thread.sleep(getSleepTime()); - } else { - writeJobstateLog(); + while (true) { + updateJobStatus(); - if (workflowStatus == RuntimeInfo.Status.SUCCESS) { - plan.onEnd(logger, workflowStatus, "Successful"); - } else { - plan.onEnd(logger, workflowStatus, "Failure"); - } + // Workflow finished? i.e. Presence of monitord.done file + if (!done.exists()) { + Thread.sleep(getSleepTime()); + } else { + writeJobstateLog(); - break; - } - } - } catch (Exception e) { - log.error(e.getMessage(), e); - writeJobstateLog(); - plan.onEnd(logger, RuntimeInfo.Status.FAILURE, e.getMessage()); + if (workflowStatus == RuntimeInfo.Status.SUCCESS) { + plan.onEnd(logger, workflowStatus, "Successful"); + } else { + plan.onEnd(logger, workflowStatus, "Failure"); } + + break; + } } + } catch (Exception e) { + log.error(e.getMessage(), e); + writeJobstateLog(); + plan.onEnd(logger, RuntimeInfo.Status.FAILURE, e.getMessage()); + } } + } } diff --git a/engines/pegasus/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/pegasus/PegasusWorkflowAdapter.java b/engines/pegasus/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/pegasus/PegasusWorkflowAdapter.java index af48ab70..44896730 100644 --- a/engines/pegasus/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/pegasus/PegasusWorkflowAdapter.java +++ b/engines/pegasus/adapter/src/main/java/edu/isi/wings/execution/engine/api/impl/pegasus/PegasusWorkflowAdapter.java @@ -27,457 +27,534 @@ import edu.isi.wings.execution.engine.classes.RuntimeStep; import edu.isi.wings.workflow.plan.api.ExecutionStep; import edu.isi.wings.workflow.plan.classes.ExecutionFile; -import org.apache.log4j.Logger; - import java.io.*; import java.nio.file.Files; -import java.nio.file.Paths; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.*; +import org.apache.log4j.Logger; public class PegasusWorkflowAdapter { - ADAG adag = null; - - String codeDir = null; - String dataDir = null; - String pegasusHome = null; - String cleanup = null; - boolean force = false; - - Properties props = null; - - Set inputs = null; - Set executables = null; - - private Map req = null; - - private Logger log = null; - /** - * @param props Properties with lib.domain.code.storage, lib.domain.data.storage, pegasus.home, - * and pegasus.force properties. - */ - public PegasusWorkflowAdapter(Properties props) throws Exception { - this.props = props; - this.codeDir = props.getProperty("lib.domain.code.storage") + java.io.File.separator; - this.dataDir = props.getProperty("lib.domain.data.storage") + java.io.File.separator; - this.pegasusHome = props.getProperty("pegasus.home") + java.io.File.separator; - this.cleanup = props.getProperty("pegasus.cleanup", "inplace"); - this.force = Boolean.parseBoolean(props.getProperty("pegasus.force", "false")); - - this.inputs = new HashSet(); - this.executables = new HashSet(); - this.log = Logger.getLogger(this.getClass()); - - // Is Pegasus directory valid? - if (new java.io.File(pegasusHome).isDirectory()) { - if (!new java.io.File(pegasusHome + "/bin/pegasus-plan").canExecute()) { - log.error("Invalid Pegasus Home: " + pegasusHome + "/bin/pegasus-plan not found"); - throw new Exception("Invalid Pegasus Home: " + pegasusHome + "/bin/pegasus-plan not found"); - } - } else { - log.error("Invalid Pegasus Home: " + pegasusHome + " is not a directory"); - throw new Exception("Invalid Pegasus Home: " + pegasusHome + " is not a directory"); - } + ADAG adag = null; + + String codeDir = null; + String dataDir = null; + String pegasusHome = null; + String cleanup = null; + boolean force = false; + + Properties props = null; + + Set inputs = null; + Set executables = null; + + private Map req = null; + + private Logger log = null; + + /** + * @param props Properties with lib.domain.code.storage, lib.domain.data.storage, pegasus.home, + * and pegasus.force properties. + */ + public PegasusWorkflowAdapter(Properties props) throws Exception { + this.props = props; + this.codeDir = + props.getProperty("lib.domain.code.storage") + java.io.File.separator; + this.dataDir = + props.getProperty("lib.domain.data.storage") + java.io.File.separator; + this.pegasusHome = + props.getProperty("pegasus.home") + java.io.File.separator; + this.cleanup = props.getProperty("pegasus.cleanup", "inplace"); + this.force = + Boolean.parseBoolean(props.getProperty("pegasus.force", "false")); + + this.inputs = new HashSet(); + this.executables = new HashSet(); + this.log = Logger.getLogger(this.getClass()); + + // Is Pegasus directory valid? + if (new java.io.File(pegasusHome).isDirectory()) { + if (!new java.io.File(pegasusHome + "/bin/pegasus-plan").canExecute()) { + log.error( + "Invalid Pegasus Home: " + pegasusHome + "/bin/pegasus-plan not found" + ); + throw new Exception( + "Invalid Pegasus Home: " + pegasusHome + "/bin/pegasus-plan not found" + ); + } + } else { + log.error("Invalid Pegasus Home: " + pegasusHome + " is not a directory"); + throw new Exception( + "Invalid Pegasus Home: " + pegasusHome + " is not a directory" + ); + } + } + + public ADAG runWorkflow( + RuntimePlan plan, + String siteCatalog, + String site, + String baseDir + ) throws Exception { + log.debug("Plan Workflow: " + plan.getName()); + adag = new ADAG(plan.getName()); + + // Construct map for components' requirement. + constructComponentRequirementMap(); + + // Add jobs to the workflow + for (RuntimeStep step : plan.getQueue().getAllSteps()) { + // Invoking Pegasus with --input-dir, so we don't need to create a Replica Catalog + registerWithReplicaCatalog(step); + + // Register Wings components and there dependent files. + registerWithTransformationCatalog(step); + + // Construct Pegasus Job for each Wings Node. + buildAndRegisterJob(plan, step); + // Job Dependencies + // Pegasus planner automatically identifies dependencies based on input/output files used by a job } - public ADAG runWorkflow(RuntimePlan plan, String siteCatalog, String site, String baseDir) throws Exception { - log.debug("Plan Workflow: " + plan.getName()); - adag = new ADAG(plan.getName()); - - // Construct map for components' requirement. - constructComponentRequirementMap(); - - // Add jobs to the workflow - for (RuntimeStep step : plan.getQueue().getAllSteps()) { - // Invoking Pegasus with --input-dir, so we don't need to create a Replica Catalog - registerWithReplicaCatalog(step); - - // Register Wings components and there dependent files. - registerWithTransformationCatalog(step); - - // Construct Pegasus Job for each Wings Node. - buildAndRegisterJob(plan, step); - - // Job Dependencies - // Pegasus planner automatically identifies dependencies based on input/output files used by a job - } - - // Write DAX file to submit dir - adag.writeToFile(baseDir + plan.getName() + ".dax"); - - Process process = null; - try { - // Write Properties file to submit dir - String props = writePropertiesFile(baseDir, siteCatalog); - List command = new ArrayList(); - - command.add(pegasusHome + "bin/pegasus-plan"); - command.add(props); + // Write DAX file to submit dir + adag.writeToFile(baseDir + plan.getName() + ".dax"); - command.add("--dax"); - command.add(baseDir + plan.getName() + ".dax"); + Process process = null; + try { + // Write Properties file to submit dir + String props = writePropertiesFile(baseDir, siteCatalog); + List command = new ArrayList(); - command.add("--dir"); - command.add(baseDir); + command.add(pegasusHome + "bin/pegasus-plan"); + command.add(props); - command.add("--relative-submit-dir"); - command.add("submit"); + command.add("--dax"); + command.add(baseDir + plan.getName() + ".dax"); - command.add("--output-dir"); - command.add(dataDir); + command.add("--dir"); + command.add(baseDir); - command.add("--sites"); - command.add(site); + command.add("--relative-submit-dir"); + command.add("submit"); - command.add("--verbose"); + command.add("--output-dir"); + command.add(dataDir); - command.add("--cleanup"); - command.add(this.cleanup.toLowerCase()); + command.add("--sites"); + command.add(site); - if (this.force) { - command.add("--force"); - } + command.add("--verbose"); - command.add("--submit"); + command.add("--cleanup"); + command.add(this.cleanup.toLowerCase()); - // Execute pegasus-plan - process = new ProcessBuilder(command).redirectErrorStream(true).start(); + if (this.force) { + command.add("--force"); + } - writeOutStd(process, plan); + command.add("--submit"); - process.waitFor(); + // Execute pegasus-plan + process = new ProcessBuilder(command).redirectErrorStream(true).start(); - } catch (Exception e) { - log.error(e.getMessage(), e); - } + writeOutStd(process, plan); - if (process.exitValue() == 0) { - return adag; - } else { - log.debug("Plan Failure: " + plan.getName() + " Exit Code: " + process.exitValue()); - throw new Exception("pegasus-plan failed with exit code " + process.exitValue()); - } + process.waitFor(); + } catch (Exception e) { + log.error(e.getMessage(), e); } - /** - * Write standard out from a process to the plan log. - * - * @param p Process which has been executed - * @param plan Plan whose logger should be used to write out the standard out. - * @throws Exception - */ - private void writeOutStd(Process p, RuntimePlan plan) throws Exception { - String line; - BufferedReader r = null; - try { - r = new BufferedReader(new InputStreamReader(p.getInputStream())); - - while ((line = r.readLine()) != null) { - plan.getRuntimeInfo().addLog(line); - } - } catch (IOException e) { - throw new Exception(e.getMessage()); - } finally { - if (r != null) { - r.close(); - } - } + if (process.exitValue() == 0) { + return adag; + } else { + log.debug( + "Plan Failure: " + plan.getName() + " Exit Code: " + process.exitValue() + ); + throw new Exception( + "pegasus-plan failed with exit code " + process.exitValue() + ); } - - /** - * @param submitDir Directory where the properties file should be written - * @param siteCatalog Location of site-catalog file - */ - private String writePropertiesFile(String submitDir, String siteCatalog) { - StringBuilder props = new StringBuilder(); - props.append("-Dpegasus.catalog.site.file=" + siteCatalog + " "); - props.append("-Dpegasus.catalog.master.url=" + "sqlite:///" + submitDir + "master.db "); - props.append("-Dpegasus.metrics.app=wings"); - - return props.toString(); + } + + /** + * Write standard out from a process to the plan log. + * + * @param p Process which has been executed + * @param plan Plan whose logger should be used to write out the standard out. + * @throws Exception + */ + private void writeOutStd(Process p, RuntimePlan plan) throws Exception { + String line; + BufferedReader r = null; + try { + r = new BufferedReader(new InputStreamReader(p.getInputStream())); + + while ((line = r.readLine()) != null) { + plan.getRuntimeInfo().addLog(line); + } + } catch (IOException e) { + throw new Exception(e.getMessage()); + } finally { + if (r != null) { + r.close(); + } + } + } + + /** + * @param submitDir Directory where the properties file should be written + * @param siteCatalog Location of site-catalog file + */ + private String writePropertiesFile(String submitDir, String siteCatalog) { + StringBuilder props = new StringBuilder(); + props.append("-Dpegasus.catalog.site.file=" + siteCatalog + " "); + props.append( + "-Dpegasus.catalog.master.url=" + "sqlite:///" + submitDir + "master.db " + ); + props.append("-Dpegasus.metrics.app=wings"); + + return props.toString(); + } + + /** + * + */ + private void constructComponentRequirementMap() { + this.req = new HashMap(); + + ComponentCreationAPI api = ComponentFactory.getCreationAPI( + this.props, + true + ); + + for (ComponentTreeNode n : api.getComponentHierarchy(true).flatten()) { + if (n.getCls().getComponent() == null) { + continue; + } + log.debug("Component ID: " + n.getCls().getComponent().getID()); + this.req.put( + n.getCls().getComponent().getID(), + n.getCls().getComponent().getComponentRequirement() + ); + } + } + + /** + * Wings jobs can depend on multiple input files. + * The method identifies files and registers them with replica catalog + * + * @param rStep Step whose input files are being registered. + * @throws Exception + */ + private void registerWithReplicaCatalog(RuntimeStep rStep) throws Exception { + ExecutionStep eStep = rStep.getStep(); + + for (ExecutionFile input : eStep.getInputFiles()) { + String lfn = input.getBinding(); + Path pfn = Paths.get(input.getLocation()); + + if (inputs.contains(lfn)) { + continue; + } + + log.debug( + "Replica: " + + lfn + + " " + + pfn.normalize().toUri() + + " site=local Exists?: " + + pfn.toFile().exists() + ); + + if (pfn.toFile().exists()) { + File f = new File(lfn); + f.addPhysicalFile(pfn.normalize().toUri().toString(), "local"); + adag.addFile(f); + } + + inputs.add(lfn); } - /** - * - */ - private void constructComponentRequirementMap() { - this.req = new HashMap(); - - ComponentCreationAPI api = ComponentFactory.getCreationAPI(this.props, true); + for (ExecutionFile output : eStep.getOutputFiles()) { + String lfn = output.getBinding(); + Path pfn = Paths.get(output.getLocation()); + + if (inputs.contains(lfn)) { + continue; + } + + log.debug( + "Replica: " + + lfn + + " " + + pfn.normalize().toUri() + + " site=local Exists?: " + + pfn.toFile().exists() + ); + + if (pfn.toFile().exists()) { + File f = new File(lfn); + f.addPhysicalFile(pfn.normalize().toUri().toString(), "local"); + adag.addFile(f); + } + + inputs.add(lfn); + } + } + + /** + * Wings components can contain of multiple files. + * Example: wings-labkey/{run,io.sh,sub-dir-1/file-1,sub-dir-2/{file-1,file-2}} + * The method identifies all dependencies, registers files with x bit as executables, and rest as data files + * Transformation: wings-labkey + * Executables: wings-labkey -> run, io.sh + * Data files: sub-dir-1/file-1, sub-dir-2/file-1, sub-dir-2/file-2 + * + * @param rStep Step whose code bindings are being registered. + * @throws Exception + */ + private void registerWithTransformationCatalog(RuntimeStep rStep) + throws Exception { + ExecutionStep eStep = rStep.getStep(); + String componentName = Paths + .get(eStep.getCodeBinding().getCodeDirectory()) + .getFileName() + .toString(); + + if (executables.contains(componentName)) { + return; + } - for (ComponentTreeNode n : api.getComponentHierarchy(true).flatten()) { - if (n.getCls().getComponent() == null) { - continue; - } - log.debug("Component ID: " + n.getCls().getComponent().getID()); - this.req.put(n.getCls().getComponent().getID(), n.getCls().getComponent().getComponentRequirement()); + Transformation transformation = new Transformation(componentName); + String dir = + eStep.getCodeBinding().getCodeDirectory() + java.io.File.separator; + Path profiles = Paths.get(dir + "__pegasus-job.properties"); + + Executable executable = new Executable(componentName); + executable.addPhysicalFile(new PFN("file://" + dir + "run")); + executable.setInstalled(false); + executable.setArchitecture(Executable.ARCH.X86_64); + executable.setOS(Executable.OS.LINUX); + + if (profiles.toFile().exists()) { + InputStream input = null; + String key = null; + String value = null; + String namespace = null; + + try { + Properties jobProfiles = new Properties(); + + input = new FileInputStream(profiles.toFile()); + jobProfiles.load(input); + + Enumeration e = jobProfiles.propertyNames(); + while (e.hasMoreElements()) { + key = (String) e.nextElement(); + value = jobProfiles.getProperty(key); + + int indexOf = key.indexOf('.'); + if (indexOf == -1) { + throw new Exception( + "Invalid Pegasus profile \"" + + key + + "\" in __pegasus-job.properties of component " + + componentName + ); + } + + namespace = key.substring(0, indexOf); + key = key.substring(indexOf + 1); + executable.addProfile( + Profile.NAMESPACE.valueOf(namespace.toLowerCase()), + key, + value + ); + } + } catch (IllegalArgumentException e) { + throw new Exception( + "Invalid Pegasus profile namespace \"" + + namespace + + "\" in __pegasus-job.properties of component " + + componentName, + e + ); + } catch (IOException e) { + e.printStackTrace(); + } finally { + if (input != null) { + try { + input.close(); + } catch (IOException e) { + e.printStackTrace(); + } } + } } + transformation.uses(executable); - /** - * Wings jobs can depend on multiple input files. - * The method identifies files and registers them with replica catalog - * - * @param rStep Step whose input files are being registered. - * @throws Exception - */ - private void registerWithReplicaCatalog(RuntimeStep rStep) throws Exception { - ExecutionStep eStep = rStep.getStep(); - - for (ExecutionFile input : eStep.getInputFiles()) { - String lfn = input.getBinding(); - Path pfn = Paths.get(input.getLocation()); + adag.addExecutable(executable); - if (inputs.contains(lfn)) { - continue; - } + log.debug("Transformation: " + componentName); - log.debug("Replica: " + lfn + " " + pfn.normalize().toUri() + " site=local Exists?: " + pfn.toFile().exists()); - - if (pfn.toFile().exists()) { - File f = new File(lfn); - f.addPhysicalFile(pfn.normalize().toUri().toString(), "local"); - adag.addFile(f); - } + Stack stack = new Stack(); + stack.push(dir); + log.debug("Main Dir: " + dir); - inputs.add(lfn); + /* + * Iterate over the component directory to register all files with the transformation catalog. + * Executable Files are registered as such. + */ + while (!stack.isEmpty()) { + String currentDir = stack.pop(); + log.debug("Popped Dir: " + currentDir + " " + stack.size()); + + for (java.io.File file : new java.io.File(currentDir).listFiles()) { + if ( + file.getName().equals(".") || + file.getName().equals("..") || + file.getName().equals("run") || + (Files.exists(profiles) && Files.isSameFile(profiles, file.toPath())) + ) { + log.debug("Skipped: " + file.getName()); + continue; } - for (ExecutionFile output : eStep.getOutputFiles()) { - String lfn = output.getBinding(); - Path pfn = Paths.get(output.getLocation()); - - if (inputs.contains(lfn)) { - continue; + if (file.isDirectory()) { + stack.push(file.getAbsolutePath() + java.io.File.separator); + log.debug("Push: " + stack.peek()); + } else { + String name = file.getAbsolutePath().replaceAll(dir, ""); + if (file.canExecute()) { + Executable e = new Executable(name); + e.addPhysicalFile(new PFN("file://" + file.getAbsolutePath())); + e.setInstalled(false); + e.setArchitecture(Executable.ARCH.X86_64); + e.setOS(Executable.OS.LINUX); + + transformation.uses(e); + + if (!executables.contains(name)) { + adag.addExecutable(e); } - - log.debug("Replica: " + lfn + " " + pfn.normalize().toUri() + " site=local Exists?: " + pfn.toFile().exists()); - - if (pfn.toFile().exists()) { - File f = new File(lfn); - f.addPhysicalFile(pfn.normalize().toUri().toString(), "local"); - adag.addFile(f); + } else { + log.debug("File: " + name + " " + file.getAbsolutePath()); + File f = new File(name); + f.addPhysicalFile(new PFN("file://" + file.getAbsolutePath())); + transformation.uses(f); + if (!executables.contains(name)) { + adag.addFile(f); } + } - inputs.add(lfn); + executables.add(name); } + } } - /** - * Wings components can contain of multiple files. - * Example: wings-labkey/{run,io.sh,sub-dir-1/file-1,sub-dir-2/{file-1,file-2}} - * The method identifies all dependencies, registers files with x bit as executables, and rest as data files - * Transformation: wings-labkey - * Executables: wings-labkey -> run, io.sh - * Data files: sub-dir-1/file-1, sub-dir-2/file-1, sub-dir-2/file-2 - * - * @param rStep Step whose code bindings are being registered. - * @throws Exception - */ - private void registerWithTransformationCatalog(RuntimeStep rStep) throws Exception { - ExecutionStep eStep = rStep.getStep(); - String componentName = Paths.get(eStep.getCodeBinding().getCodeDirectory()).getFileName().toString(); - - if (executables.contains(componentName)) { - return; - } - - Transformation transformation = new Transformation(componentName); - String dir = eStep.getCodeBinding().getCodeDirectory() + java.io.File.separator; - Path profiles = Paths.get(dir + "__pegasus-job.properties"); - - Executable executable = new Executable(componentName); - executable.addPhysicalFile(new PFN("file://" + dir + "run")); - executable.setInstalled(false); - executable.setArchitecture(Executable.ARCH.X86_64); - executable.setOS(Executable.OS.LINUX); - - if (profiles.toFile().exists()) { - InputStream input = null; - String key = null; - String value = null; - String namespace = null; - - try { - Properties jobProfiles = new Properties(); - - input = new FileInputStream(profiles.toFile()); - jobProfiles.load(input); - - Enumeration e = jobProfiles.propertyNames(); - while (e.hasMoreElements()) { - key = (String) e.nextElement(); - value = jobProfiles.getProperty(key); - - int indexOf = key.indexOf('.'); - if (indexOf == -1) { - throw new Exception( - "Invalid Pegasus profile \"" + key + "\" in __pegasus-job.properties of component " + componentName - ); - } - - namespace = key.substring(0, indexOf); - key = key.substring(indexOf + 1); - executable.addProfile( - Profile.NAMESPACE.valueOf(namespace.toLowerCase()), key, value - ); - } - } catch (IllegalArgumentException e) { - throw new Exception( - "Invalid Pegasus profile namespace \"" + namespace + "\" in __pegasus-job.properties of component " + componentName, e - ); - } catch (IOException e) { - e.printStackTrace(); - } finally { - if (input != null) { - try { - input.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } + adag.addTransformation(transformation); + executables.add(componentName); + } + + /** + * Construct Pegasus Job i.e. Add input/output files, arguments, etc. + * + * @param plan Workflow Plan + * @param rStep Steps for which Pegasus job is being built. + * @return edu.isi.pegasus.planner.dax.Job + * @throws Exception + */ + private Job buildAndRegisterJob(RuntimePlan plan, RuntimeStep rStep) + throws Exception { + ExecutionStep eStep = rStep.getStep(); + String componentID = eStep.getCodeBinding().getID(); + String componentName = Paths + .get(eStep.getCodeBinding().getCodeDirectory()) + .getFileName() + .toString(); + + Job job = new Job(rStep.getName(), componentName); + log.debug("Workflow " + plan.getName()); + log.debug("\tJob Name: " + job.getName() + " Job ID: " + job.getId()); + log.debug("\tJob Code Dir:" + eStep.getCodeBinding().getCodeDirectory()); + + // Arguments + for (String name : eStep.getInvocationArguments().keySet()) { + job.addArgument(name); + + for (Object value : eStep.getInvocationArguments().get(name)) { + if (value instanceof String) { + job.addArgument((String) value); + } else if (value instanceof ExecutionFile) { + job.addArgument(new File(((ExecutionFile) value).getBinding())); } - transformation.uses(executable); - - adag.addExecutable(executable); - - log.debug("Transformation: " + componentName); - - Stack stack = new Stack(); - stack.push(dir); - log.debug("Main Dir: " + dir); - - /* - * Iterate over the component directory to register all files with the transformation catalog. - * Executable Files are registered as such. - */ - while (!stack.isEmpty()) { - String currentDir = stack.pop(); - log.debug("Popped Dir: " + currentDir + " " + stack.size()); - - for (java.io.File file : new java.io.File(currentDir).listFiles()) { - if ( - file.getName().equals(".") || file.getName().equals("..") || - file.getName().equals("run") || (Files.exists(profiles) && Files.isSameFile(profiles, file.toPath())) - ) { - log.debug("Skipped: " + file.getName()); - continue; - } - - if (file.isDirectory()) { - stack.push(file.getAbsolutePath() + java.io.File.separator); - log.debug("Push: " + stack.peek()); - } else { - String name = file.getAbsolutePath().replaceAll(dir, ""); - if (file.canExecute()) { - Executable e = new Executable(name); - e.addPhysicalFile(new PFN("file://" + file.getAbsolutePath())); - e.setInstalled(false); - e.setArchitecture(Executable.ARCH.X86_64); - e.setOS(Executable.OS.LINUX); - - transformation.uses(e); - - if (!executables.contains(name)) { - adag.addExecutable(e); - } - } else { - log.debug("File: " + name + " " + file.getAbsolutePath()); - File f = new File(name); - f.addPhysicalFile(new PFN("file://" + file.getAbsolutePath())); - transformation.uses(f); - if (!executables.contains(name)) { - adag.addFile(f); - } - } - - executables.add(name); - } - } - } - - adag.addTransformation(transformation); - executables.add(componentName); + } } - /** - * Construct Pegasus Job i.e. Add input/output files, arguments, etc. - * - * @param plan Workflow Plan - * @param rStep Steps for which Pegasus job is being built. - * @return edu.isi.pegasus.planner.dax.Job - * @throws Exception - */ - private Job buildAndRegisterJob(RuntimePlan plan, RuntimeStep rStep) throws Exception { - ExecutionStep eStep = rStep.getStep(); - String componentID = eStep.getCodeBinding().getID(); - String componentName = Paths.get(eStep.getCodeBinding().getCodeDirectory()).getFileName().toString(); - - Job job = new Job(rStep.getName(), componentName); - log.debug("Workflow " + plan.getName()); - log.debug("\tJob Name: " + job.getName() + " Job ID: " + job.getId()); - log.debug("\tJob Code Dir:" + eStep.getCodeBinding().getCodeDirectory()); - - // Arguments - for (String name : eStep.getInvocationArguments().keySet()) { - job.addArgument(name); - - for (Object value : eStep.getInvocationArguments().get(name)) { - if (value instanceof String) { - job.addArgument((String) value); - } else if (value instanceof ExecutionFile) { - job.addArgument(new File(((ExecutionFile) value).getBinding())); - } - } - } - - // Input Files - for (ExecutionFile input : eStep.getInputFiles()) { - job.uses(new File(input.getBinding()), File.LINK.INPUT); - } - - // Output Files - for (ExecutionFile output : eStep.getOutputFiles()) { - job.uses(new File(output.getBinding()), File.LINK.OUTPUT); - } - - ComponentRequirement tmp = this.req.get(componentID); - if (tmp.getMemoryGB() > 0) { - job.addProfile(Profile.NAMESPACE.pegasus, "memory", (Math.round(Math.ceil(tmp.getMemoryGB() * 1024))) + ""); - } + // Input Files + for (ExecutionFile input : eStep.getInputFiles()) { + job.uses(new File(input.getBinding()), File.LINK.INPUT); + } - if (tmp.getStorageGB() > 0) { - job.addProfile(Profile.NAMESPACE.pegasus, "diskspace", (Math.round(Math.ceil(tmp.getStorageGB() * 1024))) + ""); - } + // Output Files + for (ExecutionFile output : eStep.getOutputFiles()) { + job.uses(new File(output.getBinding()), File.LINK.OUTPUT); + } - adag.addJob(job); - return job; + ComponentRequirement tmp = this.req.get(componentID); + if (tmp.getMemoryGB() > 0) { + job.addProfile( + Profile.NAMESPACE.pegasus, + "memory", + (Math.round(Math.ceil(tmp.getMemoryGB() * 1024))) + "" + ); } - /** - * Stops a Pegasus workflow by executing pegasus-remove - * - * @param plan - * @param baseDir - * @return - */ - public boolean stopWorkflow(RuntimePlan plan, String baseDir) { - try { - log.info("Stop Workflow: " + pegasusHome + "bin/pegasus-remove " + baseDir + "submit"); - Process process = new ProcessBuilder(pegasusHome + "bin/pegasus-remove", baseDir + "submit").redirectErrorStream(true).start(); - process.waitFor(); - writeOutStd(process, plan); - if (process.exitValue() == 0) { - return true; - } else { - return false; - } + if (tmp.getStorageGB() > 0) { + job.addProfile( + Profile.NAMESPACE.pegasus, + "diskspace", + (Math.round(Math.ceil(tmp.getStorageGB() * 1024))) + "" + ); + } - } catch (Exception e) { - log.error(e.getMessage(), e); - return false; - } + adag.addJob(job); + return job; + } + + /** + * Stops a Pegasus workflow by executing pegasus-remove + * + * @param plan + * @param baseDir + * @return + */ + public boolean stopWorkflow(RuntimePlan plan, String baseDir) { + try { + log.info( + "Stop Workflow: " + + pegasusHome + + "bin/pegasus-remove " + + baseDir + + "submit" + ); + Process process = new ProcessBuilder( + pegasusHome + "bin/pegasus-remove", + baseDir + "submit" + ) + .redirectErrorStream(true) + .start(); + process.waitFor(); + writeOutStd(process, plan); + if (process.exitValue() == 0) { + return true; + } else { + return false; + } + } catch (Exception e) { + log.error(e.getMessage(), e); + return false; } + } } diff --git a/planner/src/main/java/edu/isi/wings/catalog/component/ComponentFactory.java b/planner/src/main/java/edu/isi/wings/catalog/component/ComponentFactory.java index 5bfae0b1..523dd60b 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/component/ComponentFactory.java +++ b/planner/src/main/java/edu/isi/wings/catalog/component/ComponentFactory.java @@ -17,86 +17,105 @@ package edu.isi.wings.catalog.component; -import java.io.File; -import java.util.Properties; - import edu.isi.wings.catalog.component.api.ComponentCreationAPI; import edu.isi.wings.catalog.component.api.ComponentReasoningAPI; import edu.isi.wings.catalog.component.api.impl.kb.ComponentCreationKB; import edu.isi.wings.catalog.component.api.impl.kb.ComponentReasoningKB; import edu.isi.wings.common.kb.PropertiesHelper; +import java.io.File; +import java.util.Properties; public class ComponentFactory { - /** - * @param libname - * the library file - * @param props - * The properties for loading the Catalog (Optional). If props - * are provided, it should contain the following keys: - * lib.concrete.url, lib.concrete.rules.path lib.abstract.url, - * lib.abstract.rules.path ont.domain.component.ns, - * ont.domain.data.url, ont.component.url, ont.data.url, - * ont.workflow.url tdb.repository.dir (optional) - * @return a ComponentReasoningAPI Instance - */ - public static ComponentReasoningAPI getReasoningAPI(Properties props) { - if (props == null) - props = createLegacyConfiguration(); - return new ComponentReasoningKB(props); - } - public static ComponentCreationAPI getCreationAPI(Properties props) { - if (props == null) - props = createLegacyConfiguration(); - return new ComponentCreationKB(props); - } + /** + * @param libname + * the library file + * @param props + * The properties for loading the Catalog (Optional). If props + * are provided, it should contain the following keys: + * lib.concrete.url, lib.concrete.rules.path lib.abstract.url, + * lib.abstract.rules.path ont.domain.component.ns, + * ont.domain.data.url, ont.component.url, ont.data.url, + * ont.workflow.url tdb.repository.dir (optional) + * @return a ComponentReasoningAPI Instance + */ + public static ComponentReasoningAPI getReasoningAPI(Properties props) { + if (props == null) props = createLegacyConfiguration(); + return new ComponentReasoningKB(props); + } + + public static ComponentCreationAPI getCreationAPI(Properties props) { + if (props == null) props = createLegacyConfiguration(); + return new ComponentCreationKB(props); + } + + /** + * @param libname + * The library file to use (rest will be derived from wings + * configuration file and using default values) + */ + public static Properties createLegacyConfiguration() { + String libname = "library"; + Properties props = new Properties(); + // Create default urls for the Component Catalog + String pcurl = PropertiesHelper.getPCURL(); + String pcdir = PropertiesHelper.getPCDir(); + String pcdomurl = PropertiesHelper.getPCDomainURL(); + String pcdomdir = PropertiesHelper.getPCDomainDir(); + props.put("lib.concrete.url", pcdomurl + "/" + libname + ".owl"); + props.put("lib.abstract.url", pcdomurl + "/abstract.owl"); + props.put("ont.domain.component.ns", pcdomurl + "/library.owl#"); + props.put("ont.component.url", pcurl + "/ontology.owl"); + props.put( + "lib.concrete.map", + "file:" + pcdomdir + File.separator + libname + ".owl" + ); + props.put( + "lib.abstract.map", + "file:" + pcdomdir + File.separator + "abstract.owl" + ); + props.put( + "ont.component.map", + "file:" + pcdir + File.separator + "ontology.owl" + ); - /** - * @param libname - * The library file to use (rest will be derived from wings - * configuration file and using default values) - */ - public static Properties createLegacyConfiguration() { - String libname = "library"; - Properties props = new Properties(); - // Create default urls for the Component Catalog - String pcurl = PropertiesHelper.getPCURL(); - String pcdir = PropertiesHelper.getPCDir(); - String pcdomurl = PropertiesHelper.getPCDomainURL(); - String pcdomdir = PropertiesHelper.getPCDomainDir(); - props.put("lib.concrete.url", pcdomurl + "/" + libname + ".owl"); - props.put("lib.abstract.url", pcdomurl + "/abstract.owl"); - props.put("ont.domain.component.ns", pcdomurl + "/library.owl#"); - props.put("ont.component.url", pcurl + "/ontology.owl"); - props.put("lib.concrete.map", "file:" + pcdomdir + File.separator + libname + ".owl"); - props.put("lib.abstract.map", "file:" + pcdomdir + File.separator + "abstract.owl"); - props.put("ont.component.map", "file:" + pcdir + File.separator + "ontology.owl"); + // Add paths to the rule files + props.put( + "lib.concrete.rules.path", + pcdomdir + File.separator + libname + ".rules" + ); + props.put( + "lib.abstract.rules.path", + pcdomdir + File.separator + "abstract.rules" + ); - // Add paths to the rule files - props.put("lib.concrete.rules.path", pcdomdir + File.separator + libname + ".rules"); - props.put("lib.abstract.rules.path", pcdomdir + File.separator + "abstract.rules"); + // Add the default code directory + props.put("lib.domain.code.storage", PropertiesHelper.getCodeDirectory()); - // Add the default code directory - props.put("lib.domain.code.storage", PropertiesHelper.getCodeDirectory()); - - // The component catalog uses data properties in component rules - // and so it needs to be aware of the data ontologies as well - String dcurl = PropertiesHelper.getDCURL(); - String dcdomurl = PropertiesHelper.getDCDomainURL(); - String dcdir = PropertiesHelper.getDCDir(); - String dcdomdir = PropertiesHelper.getDCDomainDir(); - props.put("ont.domain.data.url", dcdomurl + "/ontology.owl"); - props.put("ont.data.url", dcurl + "/ontology.owl"); - props.put("ont.domain.data.map", "file:" + dcdomdir + File.separator + "ontology.owl"); - props.put("ont.data.map", "file:" + dcdir + File.separator + "ontology.owl"); + // The component catalog uses data properties in component rules + // and so it needs to be aware of the data ontologies as well + String dcurl = PropertiesHelper.getDCURL(); + String dcdomurl = PropertiesHelper.getDCDomainURL(); + String dcdir = PropertiesHelper.getDCDir(); + String dcdomdir = PropertiesHelper.getDCDomainDir(); + props.put("ont.domain.data.url", dcdomurl + "/ontology.owl"); + props.put("ont.data.url", dcurl + "/ontology.owl"); + props.put( + "ont.domain.data.map", + "file:" + dcdomdir + File.separator + "ontology.owl" + ); + props.put( + "ont.data.map", + "file:" + dcdir + File.separator + "ontology.owl" + ); - // The workflow url is used by the catalog for knowing what namespace it - // uses so it can used to filter these out and to set send some - // properties - // back to the workflow generation/planning system - String wflowurl = PropertiesHelper.getWorkflowOntologyURL(); - props.put("ont.workflow.url", wflowurl); + // The workflow url is used by the catalog for knowing what namespace it + // uses so it can used to filter these out and to set send some + // properties + // back to the workflow generation/planning system + String wflowurl = PropertiesHelper.getWorkflowOntologyURL(); + props.put("ont.workflow.url", wflowurl); - return props; - } + return props; + } } diff --git a/planner/src/main/java/edu/isi/wings/catalog/component/api/ComponentCreationAPI.java b/planner/src/main/java/edu/isi/wings/catalog/component/api/ComponentCreationAPI.java index e5cb9269..e4884dbc 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/component/api/ComponentCreationAPI.java +++ b/planner/src/main/java/edu/isi/wings/catalog/component/api/ComponentCreationAPI.java @@ -17,57 +17,65 @@ package edu.isi.wings.catalog.component.api; -import java.util.ArrayList; - import edu.isi.kcap.ontapi.transactions.TransactionsAPI; import edu.isi.wings.catalog.component.classes.Component; import edu.isi.wings.catalog.component.classes.ComponentTree; +import java.util.ArrayList; public interface ComponentCreationAPI extends TransactionsAPI { - // Query - ComponentTree getComponentHierarchy(boolean details); - - Component getComponent(String cid, boolean details); - - String getComponentLocation(String cid); - - String getDefaultComponentLocation(String cid); - - String getComponentHolderId(String cid); - - ArrayList getParentComponentTypes(String cid); - - // Update - boolean addComponent(Component comp, String pholderid); - - boolean addComponentHolder(String holderid, String pholderid, boolean is_concrete); - - boolean updateComponent(Component newcomp); - - boolean renameComponent(String oldid, String newid); - - boolean removeComponent(String cid, boolean remove_holder, boolean unlink, boolean remove_children); - - boolean removeComponentHolder(String holderid, boolean is_concrete); - - boolean moveChildComponentsTo(String oldid, String newid); - - boolean setComponentLocation(String cid, String location); - - boolean setComponentVersion(String cid, int version); - - boolean incrementComponentVersion(String id); - - // Saving - boolean save(); - - boolean delete(); - - // Copy from another API (Advisable to give the same implementation of the API here) - void copyFrom(ComponentCreationAPI dc); - - // Get/Set external data catalog to copy from - ComponentCreationAPI getExternalCatalog(); - + // Query + ComponentTree getComponentHierarchy(boolean details); + + Component getComponent(String cid, boolean details); + + String getComponentLocation(String cid); + + String getDefaultComponentLocation(String cid); + + String getComponentHolderId(String cid); + + ArrayList getParentComponentTypes(String cid); + + // Update + boolean addComponent(Component comp, String pholderid); + + boolean addComponentHolder( + String holderid, + String pholderid, + boolean is_concrete + ); + + boolean updateComponent(Component newcomp); + + boolean renameComponent(String oldid, String newid); + + boolean removeComponent( + String cid, + boolean remove_holder, + boolean unlink, + boolean remove_children + ); + + boolean removeComponentHolder(String holderid, boolean is_concrete); + + boolean moveChildComponentsTo(String oldid, String newid); + + boolean setComponentLocation(String cid, String location); + + boolean setComponentVersion(String cid, int version); + + boolean incrementComponentVersion(String id); + + // Saving + boolean save(); + + boolean delete(); + + // Copy from another API (Advisable to give the same implementation of the API here) + void copyFrom(ComponentCreationAPI dc); + + // Get/Set external data catalog to copy from + ComponentCreationAPI getExternalCatalog(); + void setExternalCatalog(ComponentCreationAPI dc); } diff --git a/planner/src/main/java/edu/isi/wings/catalog/component/api/ComponentReasoningAPI.java b/planner/src/main/java/edu/isi/wings/catalog/component/api/ComponentReasoningAPI.java index 602585f8..8d066cc1 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/component/api/ComponentReasoningAPI.java +++ b/planner/src/main/java/edu/isi/wings/catalog/component/api/ComponentReasoningAPI.java @@ -17,24 +17,26 @@ package edu.isi.wings.catalog.component.api; -import java.util.ArrayList; - import edu.isi.kcap.ontapi.transactions.TransactionsAPI; import edu.isi.wings.catalog.component.classes.ComponentInvocation; import edu.isi.wings.catalog.component.classes.ComponentPacket; +import java.util.ArrayList; /** * The interface for communicating with the Component Catalog during Workflow * Planning and generation */ public interface ComponentReasoningAPI extends TransactionsAPI { - // Generation API - ArrayList specializeAndFindDataDetails(ComponentPacket details); - - ComponentPacket findDataDetails(ComponentPacket details); + // Generation API + ArrayList specializeAndFindDataDetails( + ComponentPacket details + ); - ArrayList findOutputDataPredictedDescriptions(ComponentPacket details); + ComponentPacket findDataDetails(ComponentPacket details); - ComponentInvocation getComponentInvocation(ComponentPacket details); + ArrayList findOutputDataPredictedDescriptions( + ComponentPacket details + ); + ComponentInvocation getComponentInvocation(ComponentPacket details); } diff --git a/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentCreationKB.java b/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentCreationKB.java index c5958de0..e0d75d34 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentCreationKB.java +++ b/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentCreationKB.java @@ -17,15 +17,6 @@ package edu.isi.wings.catalog.component.api.impl.kb; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Properties; - -import org.apache.commons.io.FileUtils; - import edu.isi.kcap.ontapi.KBAPI; import edu.isi.kcap.ontapi.KBObject; import edu.isi.kcap.ontapi.KBTriple; @@ -36,33 +27,43 @@ import edu.isi.wings.catalog.component.classes.ComponentTree; import edu.isi.wings.catalog.component.classes.ComponentTreeNode; import edu.isi.wings.common.kb.KBUtils; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Properties; +import org.apache.commons.io.FileUtils; + +public class ComponentCreationKB + extends ComponentKB + implements ComponentCreationAPI { -public class ComponentCreationKB extends ComponentKB implements ComponentCreationAPI { ComponentCreationAPI externalCatalog; - - public ComponentCreationKB(Properties props) { - super(props, true, false, true); - - // Separate abstract and concrete if an older version of the catalog - int catVersion = this.getCatalogVersion(); - if(catVersion < 1) { - this.separateAbstractAndConcrete(); - this.setCatalogVersion(1); - } - + + public ComponentCreationKB(Properties props) { + super(props, true, false, true); + // Separate abstract and concrete if an older version of the catalog + int catVersion = this.getCatalogVersion(); + if (catVersion < 1) { + this.separateAbstractAndConcrete(); + this.setCatalogVersion(1); + } + String extern = props.getProperty("extern_component_catalog"); - if(extern != null) { + if (extern != null) { try { Class classz = Class.forName(extern); - ComponentCreationAPI externalCC = - (ComponentCreationAPI) classz.getDeclaredConstructor(Properties.class).newInstance(props); + ComponentCreationAPI externalCC = (ComponentCreationAPI) classz + .getDeclaredConstructor(Properties.class) + .newInstance(props); this.setExternalCatalog(externalCC); } catch (Exception e) { e.printStackTrace(); } } - } - + } + // Cleanup code private void separateAbstractAndConcrete() { try { @@ -72,104 +73,114 @@ private void separateAbstractAndConcrete() { // Delete existing this.delete(); - + // Move components to the respective writers (abstract, or concrete library) ArrayList queue = new ArrayList(); queue.add(tree.getRoot()); HashMap parents = new HashMap(); - while(!queue.isEmpty()) { + while (!queue.isEmpty()) { ComponentTreeNode tn = queue.remove(0); ComponentHolder cholder = tn.getCls(); String parentId = parents.get(cholder.getID()); - if(cholder.getComponent() == null) { + if (cholder.getComponent() == null) { // Add category //System.out.println("Add component holder for "+cholder.getID() + " with parent " + parentId); - this.addComponentHolder(cholder.getID(), parentId, false, this.abs_writerkb); - } - else { + this.addComponentHolder( + cholder.getID(), + parentId, + false, + this.abs_writerkb + ); + } else { Component c = cholder.getComponent(); - KBAPI writerkb = c.isConcrete() ? this.lib_writerkb : this.abs_writerkb; + KBAPI writerkb = c.isConcrete() + ? this.lib_writerkb + : this.abs_writerkb; // Add component //System.out.println("Add component " + c.getID() + " : " + c.getType() + " with parent " + parentId);; - this.addComponent(c, parentId, writerkb); + this.addComponent(c, parentId, writerkb); } - for(ComponentTreeNode childn : tn.getChildren()) { + for (ComponentTreeNode childn : tn.getChildren()) { parents.put(childn.getCls().getID(), cholder.getID()); queue.add(childn); } } - } - catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); } } - @Override - public ComponentTree getComponentHierarchy(boolean details) { - return this.createComponentHierarchy(this.topclass, details); - } - - private ComponentTree createComponentHierarchy(String classid, boolean details) { - ComponentHolder rootitem = new ComponentHolder(classid); - ComponentTreeNode rootnode = new ComponentTreeNode(rootitem); - ArrayList queue = new ArrayList(); - queue.add(rootnode); - - ArrayList tbd = new ArrayList(); - - while (!queue.isEmpty()) { - ComponentTreeNode node = queue.remove(0); - ComponentHolder cls = node.getCls(); - if (cls.getID() == null) - continue; - - this.start_read(); - KBObject clsobj = kb.getConcept(cls.getID()); - if(clsobj == null) continue; - - ArrayList compobjs = kb.getInstancesOfClass(clsobj, true); + @Override + public ComponentTree getComponentHierarchy(boolean details) { + return this.createComponentHierarchy(this.topclass, details); + } + + private ComponentTree createComponentHierarchy( + String classid, + boolean details + ) { + ComponentHolder rootitem = new ComponentHolder(classid); + ComponentTreeNode rootnode = new ComponentTreeNode(rootitem); + ArrayList queue = new ArrayList(); + queue.add(rootnode); + + ArrayList tbd = new ArrayList(); + + while (!queue.isEmpty()) { + ComponentTreeNode node = queue.remove(0); + ComponentHolder cls = node.getCls(); + if (cls.getID() == null) continue; + + this.start_read(); + KBObject clsobj = kb.getConcept(cls.getID()); + if (clsobj == null) continue; + + ArrayList compobjs = kb.getInstancesOfClass(clsobj, true); ArrayList subclasses = this.kb.getSubClasses(clsobj, true); for (KBObject subcls : subclasses) { - if (!subcls.getNamespace().equals(this.pcdomns) - && !subcls.getNamespace().equals(this.pcns)) - continue; + if ( + !subcls.getNamespace().equals(this.pcdomns) && + !subcls.getNamespace().equals(this.pcns) + ) continue; ComponentHolder clsitem = new ComponentHolder(subcls.getID()); ComponentTreeNode childnode = new ComponentTreeNode(clsitem); node.addChild(childnode); queue.add(childnode); - } + } this.end(); - - for (KBObject compobj : compobjs) { - if(cls.getID().equals(this.topclass)) { - // The top class cannot be used as a holder. If we find any - // components having the top class as the holder, we delete them - tbd.add(compobj.getID()); - } - else { - // Add the component as the holder's component - cls.setComponent(this.getComponent(compobj.getID(), details)); - } - } - } - - // cleanup hack (for components that have the top class as the holder) - for(String compid: tbd) { - this.removeComponent(compid, true, true, false); - } - - ComponentTree tree = new ComponentTree(rootnode); - return tree; - } + + for (KBObject compobj : compobjs) { + if (cls.getID().equals(this.topclass)) { + // The top class cannot be used as a holder. If we find any + // components having the top class as the holder, we delete them + tbd.add(compobj.getID()); + } else { + // Add the component as the holder's component + cls.setComponent(this.getComponent(compobj.getID(), details)); + } + } + } + + // cleanup hack (for components that have the top class as the holder) + for (String compid : tbd) { + this.removeComponent(compid, true, true, false); + } + + ComponentTree tree = new ComponentTree(rootnode); + return tree; + } @Override public boolean setComponentLocation(String cid, String location) { KBAPI writerkb = this.getWriterKB(cid); return this.setComponentLocation(cid, location, writerkb); - } - - protected boolean setComponentLocation(String cid, String location, KBAPI writerkb) { + + protected boolean setComponentLocation( + String cid, + String location, + KBAPI writerkb + ) { boolean new_transaction = false; if (!this.is_in_transaction()) { this.start_write(); @@ -180,10 +191,10 @@ protected boolean setComponentLocation(String cid, String location, KBAPI writer KBObject cobj = writerkb.getResource(cid); KBObject locobj = writerkb.createLiteral(location); writerkb.setPropertyValue(cobj, locprop, locobj); - if (this.externalCatalog != null) - this.externalCatalog.setComponentLocation(cid, location); - if (new_transaction) - this.save(); + if ( + this.externalCatalog != null + ) this.externalCatalog.setComponentLocation(cid, location); + if (new_transaction) this.save(); return true; } catch (Exception e) { e.printStackTrace(); @@ -200,8 +211,12 @@ public boolean setComponentVersion(String cid, int version) { KBAPI writerkb = this.getWriterKB(cid); return this.setComponentVersion(cid, version, writerkb); } - - protected boolean setComponentVersion(String cid, int version, KBAPI writerkb) { + + protected boolean setComponentVersion( + String cid, + int version, + KBAPI writerkb + ) { boolean new_transaction = false; if (!this.is_in_transaction()) { this.start_write(); @@ -212,8 +227,7 @@ protected boolean setComponentVersion(String cid, int version, KBAPI writerkb) { KBObject cobj = writerkb.getResource(cid); KBObject versionobj = writerkb.createLiteral(version); writerkb.setPropertyValue(cobj, versionProp, versionobj); - if (new_transaction) - this.save(); + if (new_transaction) this.save(); return true; } catch (Exception e) { e.printStackTrace(); @@ -225,13 +239,16 @@ protected boolean setComponentVersion(String cid, int version, KBAPI writerkb) { return false; } - public boolean setModelCatalogIdentifier(String cid, String modelIdentifier) { KBAPI writerkb = this.getWriterKB(cid); return this.setModelCatalogIdentifier(cid, modelIdentifier, writerkb); } - - protected boolean setModelCatalogIdentifier(String cid, String modelIdentifier, KBAPI writerkb) { + + protected boolean setModelCatalogIdentifier( + String cid, + String modelIdentifier, + KBAPI writerkb + ) { boolean new_transaction = false; if (!this.is_in_transaction()) { this.start_write(); @@ -242,10 +259,10 @@ protected boolean setModelCatalogIdentifier(String cid, String modelIdentifier, KBObject cobj = writerkb.getResource(cid); KBObject locobj = writerkb.createLiteral(modelIdentifier); writerkb.setPropertyValue(cobj, modelIdProp, locobj); - if (this.externalCatalog != null) - this.externalCatalog.setComponentLocation(cid, modelIdentifier); - if (new_transaction) - this.save(); + if ( + this.externalCatalog != null + ) this.externalCatalog.setComponentLocation(cid, modelIdentifier); + if (new_transaction) this.save(); return true; } catch (Exception e) { e.printStackTrace(); @@ -257,179 +274,199 @@ protected boolean setModelCatalogIdentifier(String cid, String modelIdentifier, return false; } + @Override + public boolean updateComponent(Component comp) { + if (comp == null) return false; - @Override - public boolean updateComponent(Component comp) { - if(comp == null) return false; - - // Remove existing component assertions and re-add the new component details - try { - boolean ok1 = this.removeComponent(comp.getID(), false, false, false); - boolean ok2 = this.addComponent(comp, null); - if(this.externalCatalog != null) - this.externalCatalog.updateComponent(comp); - - // TODO: If abstract, update all components defined in all libraries ! - return ok1 && ok2; - } - catch (Exception e) { - e.printStackTrace(); - return false; - } - } - - @Override + // Remove existing component assertions and re-add the new component details + try { + boolean ok1 = this.removeComponent(comp.getID(), false, false, false); + boolean ok2 = this.addComponent(comp, null); + if (this.externalCatalog != null) this.externalCatalog.updateComponent( + comp + ); + + // TODO: If abstract, update all components defined in all libraries ! + return ok1 && ok2; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + @Override public boolean incrementComponentVersion(String cid) { KBAPI writerkb = this.getWriterKB(cid); return this.incrementComponentVersion(cid, writerkb); } - - protected boolean incrementComponentVersion(String cid, KBAPI writerkb) { + + protected boolean incrementComponentVersion(String cid, KBAPI writerkb) { try { this.start_write(); - KBObject compobj = kb.getIndividual(cid); + KBObject compobj = kb.getIndividual(cid); KBObject versionProp = kb.getProperty(this.pcns + "hasVersion"); KBObject versionVal = kb.getPropertyValue(compobj, versionProp); - int currentVersion = (Integer) (versionVal.getValue() != null ? versionVal.getValue() : 0); - - int newVersion = currentVersion+1; + int currentVersion = (Integer) ( + versionVal.getValue() != null ? versionVal.getValue() : 0 + ); + + int newVersion = currentVersion + 1; KBObject cobj = writerkb.getResource(cid); KBObject newVersionVal = writerkb.createLiteral(newVersion); writerkb.setPropertyValue(cobj, versionProp, newVersionVal); return this.save(); - } - catch(Exception e) { + } catch (Exception e) { e.printStackTrace(); this.end(); } - return false; - } - - - @Override - public boolean save() { - boolean ok1 = false, ok2 = false; - if(abs_writerkb != null) - ok1 = this.save(abs_writerkb); - if(lib_writerkb != null) - ok2 = this.save(lib_writerkb); - return ok1 && ok2; - } - - @Override - public boolean addComponent(Component comp, String pholderid) { - KBAPI writerkb = this.getWriterKB((comp.getType() == Component.CONCRETE)); - return this.addComponent(comp, pholderid, writerkb); - } - - protected boolean addComponent(Component comp, String pholderid, KBAPI writerkb) { - // Check for uniqueness of the role names passed in - HashSet unique = new HashSet(); - for (ComponentRole role : comp.getInputs()) - unique.add(role.getRoleName()); - for (ComponentRole role : comp.getOutputs()) - unique.add(role.getRoleName()); - - // If there are some duplicate role ids, return false - if(unique.size() < (comp.getInputs().size() + comp.getOutputs().size())) - return false; - - String cid = comp.getID(); - String cholderid = this.getComponentHolderId(cid); - - try { - this.start_write(); - - // If parent holder passed in, create a holder as subclass of parent holder - // Else assume that current holder already exists and fetch that - KBObject cls; - if(pholderid != null) - cls = writerkb.createClass(cholderid, pholderid); - else - cls = kb.getConcept(cholderid); - - KBObject cobj = writerkb.createObjectOfClass(cid, cls); - - KBObject inProp = kb.getProperty(this.pcns + "hasInput"); - KBObject outProp = kb.getProperty(this.pcns + "hasOutput"); - - for (ComponentRole role : comp.getInputs()) { - role.setID(cid + "_" + role.getRoleName()); // HACK: role id is _ - KBObject roleobj = this.createRole(role, writerkb); - if(roleobj == null) - return false; - writerkb.addTriple(cobj, inProp, roleobj); - } - for (ComponentRole role : comp.getOutputs()) { - role.setID(cid + "_" + role.getRoleName()); - KBObject roleobj = this.createRole(role, writerkb); - if(roleobj == null) - return false; - writerkb.addTriple(cobj, outProp, roleobj); - } - - if(comp.getSource() != null){ - this.setModelCatalogIdentifier(cid, comp.getSource(), writerkb); - } - if(comp.getDocumentation() != null) - this.setComponentDocumentation(cobj, comp.getDocumentation(), writerkb); - - if(comp.getComponentRequirement() != null) - this.setComponentRequirements(cobj, comp.getComponentRequirement(), writerkb); - - if(comp.getLocation() != null) - this.setComponentLocation(cid, comp.getLocation(), writerkb); - - if(comp.getRulesText() != null) { - this.setComponentRules(cid, comp.getRulesText(), writerkb); - } - - this.setComponentVersion(cid, comp.getVersion(), writerkb); - + return false; + } + + @Override + public boolean save() { + boolean ok1 = false, ok2 = false; + if (abs_writerkb != null) ok1 = this.save(abs_writerkb); + if (lib_writerkb != null) ok2 = this.save(lib_writerkb); + return ok1 && ok2; + } + + @Override + public boolean addComponent(Component comp, String pholderid) { + KBAPI writerkb = this.getWriterKB((comp.getType() == Component.CONCRETE)); + return this.addComponent(comp, pholderid, writerkb); + } + + protected boolean addComponent( + Component comp, + String pholderid, + KBAPI writerkb + ) { + // Check for uniqueness of the role names passed in + HashSet unique = new HashSet(); + for (ComponentRole role : comp.getInputs()) unique.add(role.getRoleName()); + for (ComponentRole role : comp.getOutputs()) unique.add(role.getRoleName()); + + // If there are some duplicate role ids, return false + if ( + unique.size() < (comp.getInputs().size() + comp.getOutputs().size()) + ) return false; + + String cid = comp.getID(); + String cholderid = this.getComponentHolderId(cid); + + try { + this.start_write(); + + // If parent holder passed in, create a holder as subclass of parent holder + // Else assume that current holder already exists and fetch that + KBObject cls; + if (pholderid != null) cls = + writerkb.createClass(cholderid, pholderid); else cls = + kb.getConcept(cholderid); + + KBObject cobj = writerkb.createObjectOfClass(cid, cls); + + KBObject inProp = kb.getProperty(this.pcns + "hasInput"); + KBObject outProp = kb.getProperty(this.pcns + "hasOutput"); + + for (ComponentRole role : comp.getInputs()) { + role.setID(cid + "_" + role.getRoleName()); // HACK: role id is _ + KBObject roleobj = this.createRole(role, writerkb); + if (roleobj == null) return false; + writerkb.addTriple(cobj, inProp, roleobj); + } + for (ComponentRole role : comp.getOutputs()) { + role.setID(cid + "_" + role.getRoleName()); + KBObject roleobj = this.createRole(role, writerkb); + if (roleobj == null) return false; + writerkb.addTriple(cobj, outProp, roleobj); + } + + if (comp.getSource() != null) { + this.setModelCatalogIdentifier(cid, comp.getSource(), writerkb); + } + if (comp.getDocumentation() != null) this.setComponentDocumentation( + cobj, + comp.getDocumentation(), + writerkb + ); + + if (comp.getComponentRequirement() != null) this.setComponentRequirements( + cobj, + comp.getComponentRequirement(), + writerkb + ); + + if (comp.getLocation() != null) this.setComponentLocation( + cid, + comp.getLocation(), + writerkb + ); + + if (comp.getRulesText() != null) { + this.setComponentRules(cid, comp.getRulesText(), writerkb); + } + + this.setComponentVersion(cid, comp.getVersion(), writerkb); + KBObject isConcreteProp = kb.getProperty(this.pcns + "isConcrete"); - KBObject isConcreteVal = writerkb.createLiteral(comp.getType() == Component.CONCRETE); - writerkb.setPropertyValue(cobj, isConcreteProp, isConcreteVal); - - if(this.externalCatalog != null) - this.externalCatalog.addComponent(comp, pholderid); - - return this.save(); - } - finally { + KBObject isConcreteVal = writerkb.createLiteral( + comp.getType() == Component.CONCRETE + ); + writerkb.setPropertyValue(cobj, isConcreteProp, isConcreteVal); + + if (this.externalCatalog != null) this.externalCatalog.addComponent( + comp, + pholderid + ); + + return this.save(); + } finally { this.end(); } - } + } - @Override - public boolean addComponentHolder(String holderid, String pholderid, boolean is_concrete) { + @Override + public boolean addComponentHolder( + String holderid, + String pholderid, + boolean is_concrete + ) { KBAPI writerkb = this.getWriterKB(is_concrete); return this.addComponentHolder(holderid, pholderid, is_concrete, writerkb); - } - - protected boolean addComponentHolder(String holderid, String pholderid, boolean is_concrete, KBAPI writerkb) { - try { - this.start_write(); - writerkb.createClass(holderid, pholderid); - if(this.externalCatalog != null) - this.externalCatalog.addComponentHolder(holderid, pholderid, is_concrete); - return this.save(); - } - finally { + } + + protected boolean addComponentHolder( + String holderid, + String pholderid, + boolean is_concrete, + KBAPI writerkb + ) { + try { + this.start_write(); + writerkb.createClass(holderid, pholderid); + if (this.externalCatalog != null) this.externalCatalog.addComponentHolder( + holderid, + pholderid, + is_concrete + ); + return this.save(); + } finally { this.end(); } - } - + } + private void setSubclassParentsToClass(String oldclsid, String newclsid) { KBObject oldcls = this.kb.getConcept(oldclsid); - for(KBObject subcls: kb.getSubClasses(oldcls, true)) { + for (KBObject subcls : kb.getSubClasses(oldcls, true)) { KBObject dum = this.lib_writerkb.getConcept(subcls.getID()); - if(dum != null) - this.lib_writerkb.setSuperClass(subcls.getID(), newclsid); - else - this.abs_writerkb.setSuperClass(subcls.getID(), newclsid); + if (dum != null) this.lib_writerkb.setSuperClass( + subcls.getID(), + newclsid + ); else this.abs_writerkb.setSuperClass(subcls.getID(), newclsid); } } - + @Override public boolean moveChildComponentsTo(String oldid, String newid) { try { @@ -438,228 +475,297 @@ public boolean moveChildComponentsTo(String oldid, String newid) { String newclsid = this.getComponentHolderId(newid); this.setSubclassParentsToClass(oldclsid, newclsid); return this.save(); - } - finally { + } finally { this.end(); } } - - @Override - public boolean removeComponentHolder(String ctype, boolean is_concrete) { + + @Override + public boolean removeComponentHolder(String ctype, boolean is_concrete) { KBAPI writerkb = this.getWriterKB(is_concrete); return this.removeComponentHolder(ctype, is_concrete, writerkb); - } - - protected boolean removeComponentHolder(String ctype, boolean is_concrete, KBAPI writerkb) { - try { - this.start_write(); - //this.setSubclassParentsToGrandparents(ctype); - KBUtils.removeAllTriplesWith(writerkb, ctype, false); - if(this.externalCatalog != null) - this.externalCatalog.removeComponentHolder(ctype, is_concrete); - return this.save(); - } - finally { + } + + protected boolean removeComponentHolder( + String ctype, + boolean is_concrete, + KBAPI writerkb + ) { + try { + this.start_write(); + //this.setSubclassParentsToGrandparents(ctype); + KBUtils.removeAllTriplesWith(writerkb, ctype, false); + if ( + this.externalCatalog != null + ) this.externalCatalog.removeComponentHolder(ctype, is_concrete); + return this.save(); + } finally { this.end(); } - } - - @Override - public boolean removeComponent(String cid, boolean remove_holder, boolean unlink, boolean remove_children) { - KBAPI writerkb = this.getWriterKB(cid); - return this.removeComponent(cid, remove_holder, unlink, remove_children, writerkb); - } - - protected boolean removeComponent(String cid, boolean remove_holder, boolean unlink, boolean remove_children, KBAPI writerkb) { - if(remove_children) { + } + + @Override + public boolean removeComponent( + String cid, + boolean remove_holder, + boolean unlink, + boolean remove_children + ) { + KBAPI writerkb = this.getWriterKB(cid); + return this.removeComponent( + cid, + remove_holder, + unlink, + remove_children, + writerkb + ); + } + + protected boolean removeComponent( + String cid, + boolean remove_holder, + boolean unlink, + boolean remove_children, + KBAPI writerkb + ) { + if (remove_children) { String holderid = this.getComponentHolderId(cid); - ComponentTree cnode = createComponentHierarchy(holderid, false); - for(ComponentTreeNode tn: cnode.getRoot().getChildren()) { - String childid = tn.getCls().getComponent().getID(); - this.removeComponent(childid, remove_holder, unlink, remove_children); - } - } - - ArrayList inputobjs, outputobjs, sdobjs, hdobjs; - String loc; - try { - this.start_read(); - KBObject compobj = kb.getIndividual(cid); - if(compobj == null) { - return false; - } - inputobjs = this.getComponentInputs(compobj); - outputobjs = this.getComponentOutputs(compobj); - sdobjs = this.kb.getPropertyValues(compobj, - this.objPropMap.get("hasSoftwareDependency")); - hdobjs = this.kb.getPropertyValues(compobj, - this.objPropMap.get("hasHardwareDependency")); + ComponentTree cnode = createComponentHierarchy(holderid, false); + for (ComponentTreeNode tn : cnode.getRoot().getChildren()) { + String childid = tn.getCls().getComponent().getID(); + this.removeComponent(childid, remove_holder, unlink, remove_children); + } + } + + ArrayList inputobjs, outputobjs, sdobjs, hdobjs; + String loc; + try { + this.start_read(); + KBObject compobj = kb.getIndividual(cid); + if (compobj == null) { + return false; + } + inputobjs = this.getComponentInputs(compobj); + outputobjs = this.getComponentOutputs(compobj); + sdobjs = + this.kb.getPropertyValues( + compobj, + this.objPropMap.get("hasSoftwareDependency") + ); + hdobjs = + this.kb.getPropertyValues( + compobj, + this.objPropMap.get("hasHardwareDependency") + ); loc = this.getComponentLocation(cid); this.end(); - } - catch(Exception e) { - e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); this.end(); return false; } - - try { + + try { // Remove items from KB this.start_write(); - + // - Remove component class (holder) - if(remove_holder) { + if (remove_holder) { String holderid = this.getComponentHolderId(cid); //this.setSubclassParentsToGrandparents(holderid); KBUtils.removeAllTriplesWith(writerkb, holderid, false); } // - Remove inputs - for (KBObject obj : inputobjs) { - KBUtils.removeAllTriplesWith(writerkb, obj.getID(), false); - } - // - Remove outputs - for (KBObject obj : outputobjs) { - KBUtils.removeAllTriplesWith(writerkb, obj.getID(), false); - } - // - Remove software requirement items - for(KBObject obj : sdobjs) - for(KBTriple t : writerkb.genericTripleQuery(obj, null, null)) - writerkb.removeTriple(t); - // - Remove hardware requirement items - for(KBObject obj : hdobjs) - for (KBTriple t : writerkb.genericTripleQuery(obj, null, null)) - writerkb.removeTriple(t); + for (KBObject obj : inputobjs) { + KBUtils.removeAllTriplesWith(writerkb, obj.getID(), false); + } + // - Remove outputs + for (KBObject obj : outputobjs) { + KBUtils.removeAllTriplesWith(writerkb, obj.getID(), false); + } + // - Remove software requirement items + for (KBObject obj : sdobjs) for (KBTriple t : writerkb.genericTripleQuery( + obj, + null, + null + )) writerkb.removeTriple(t); + // - Remove hardware requirement items + for (KBObject obj : hdobjs) for (KBTriple t : writerkb.genericTripleQuery( + obj, + null, + null + )) writerkb.removeTriple(t); // - Remove the component itself - KBUtils.removeAllTriplesWith(writerkb, cid, false); - - this.save(); - this.end(); - } - catch(Exception e) { + KBUtils.removeAllTriplesWith(writerkb, cid, false); + + this.save(); + this.end(); + } catch (Exception e) { e.printStackTrace(); this.end(); return false; } - - try { - // Remove the component directory - if (unlink) { - // Remove component if it is in the catalog's component directory - if(loc != null) { - File f = new File(loc); - if(f.getParentFile().getAbsolutePath().equals(this.codedir)) { - if(f.isDirectory()) - try { - FileUtils.deleteDirectory(f); - } catch (IOException e) { - e.printStackTrace(); - } - else - f.delete(); - } - } - } - if(this.externalCatalog != null) - this.externalCatalog.removeComponent(cid, remove_holder, unlink, remove_children); - } - catch(Exception e) { + + try { + // Remove the component directory + if (unlink) { + // Remove component if it is in the catalog's component directory + if (loc != null) { + File f = new File(loc); + if (f.getParentFile().getAbsolutePath().equals(this.codedir)) { + if (f.isDirectory()) try { + FileUtils.deleteDirectory(f); + } catch (IOException e) { + e.printStackTrace(); + } else f.delete(); + } + } + } + if (this.externalCatalog != null) this.externalCatalog.removeComponent( + cid, + remove_holder, + unlink, + remove_children + ); + } catch (Exception e) { e.printStackTrace(); return false; } return true; - } - - @Override - public boolean renameComponent(String oldid, String newid) { - try { - this.start_write(); - KBAPI writerkb = this.getWriterKB(oldid); - KBUtils.renameAllTriplesWith(writerkb, this.getComponentHolderId(oldid), - this.getComponentHolderId(newid), false); - KBUtils.renameAllTriplesWith(writerkb, oldid, newid, false); - - if(this.externalCatalog != null) - this.externalCatalog.renameComponent(oldid, newid); - + } + + @Override + public boolean renameComponent(String oldid, String newid) { + try { + this.start_write(); + KBAPI writerkb = this.getWriterKB(oldid); + KBUtils.renameAllTriplesWith( + writerkb, + this.getComponentHolderId(oldid), + this.getComponentHolderId(newid), + false + ); + KBUtils.renameAllTriplesWith(writerkb, oldid, newid, false); + + if (this.externalCatalog != null) this.externalCatalog.renameComponent( + oldid, + newid + ); + return this.save(); - } - finally { + } finally { this.end(); } - } - - - @Override - public void copyFrom(ComponentCreationAPI dc) { - ComponentCreationKB dckb = (ComponentCreationKB)dc; - - try { - this.start_write(); - dckb.start_read(); - - this.abs_writerkb.copyFrom(dckb.abs_writerkb); - this.lib_writerkb.copyFrom(dckb.lib_writerkb); - + } + + @Override + public void copyFrom(ComponentCreationAPI dc) { + ComponentCreationKB dckb = (ComponentCreationKB) dc; + + try { + this.start_write(); + dckb.start_read(); + + this.abs_writerkb.copyFrom(dckb.abs_writerkb); + this.lib_writerkb.copyFrom(dckb.lib_writerkb); + // Namespace rename maps HashMap nsmap = new HashMap(); nsmap.put(dckb.dcns, this.dcns); nsmap.put(dckb.pcns, this.pcns); nsmap.put(dckb.dcdomns, this.dcdomns); nsmap.put(dckb.pcdomns, this.pcdomns); - KBUtils.renameTripleNamespaces(abs_writerkb, nsmap); - KBUtils.renameTripleNamespaces(lib_writerkb, nsmap); - - KBUtils.renameAllTriplesWith(abs_writerkb, dckb.pcurl, this.pcurl, false); - KBUtils.renameAllTriplesWith(abs_writerkb, dckb.absurl, this.absurl, false); - KBUtils.renameAllTriplesWith(abs_writerkb, dckb.dconturl, this.dconturl, false); - + KBUtils.renameTripleNamespaces(abs_writerkb, nsmap); + KBUtils.renameTripleNamespaces(lib_writerkb, nsmap); + + KBUtils.renameAllTriplesWith(abs_writerkb, dckb.pcurl, this.pcurl, false); + KBUtils.renameAllTriplesWith( + abs_writerkb, + dckb.absurl, + this.absurl, + false + ); + KBUtils.renameAllTriplesWith( + abs_writerkb, + dckb.dconturl, + this.dconturl, + false + ); + KBUtils.renameAllTriplesWith(lib_writerkb, dckb.pcurl, this.pcurl, false); - KBUtils.renameAllTriplesWith(lib_writerkb, dckb.absurl, this.absurl, false); - KBUtils.renameAllTriplesWith(lib_writerkb, dckb.liburl, this.liburl, false); - KBUtils.renameAllTriplesWith(lib_writerkb, dckb.dconturl, this.dconturl, false); - + KBUtils.renameAllTriplesWith( + lib_writerkb, + dckb.absurl, + this.absurl, + false + ); + KBUtils.renameAllTriplesWith( + lib_writerkb, + dckb.liburl, + this.liburl, + false + ); + KBUtils.renameAllTriplesWith( + lib_writerkb, + dckb.dconturl, + this.dconturl, + false + ); + // Change any specified locations of data - KBObject locProp = lib_writerkb.getProperty(this.pcns+"hasLocation"); - ArrayList triples = - lib_writerkb.genericTripleQuery(null, locProp, null); - for(KBTriple t : triples) { + KBObject locProp = lib_writerkb.getProperty(this.pcns + "hasLocation"); + ArrayList triples = lib_writerkb.genericTripleQuery( + null, + locProp, + null + ); + for (KBTriple t : triples) { lib_writerkb.removeTriple(t); - if(t.getObject() == null || t.getObject().getValue() == null) - continue; + if (t.getObject() == null || t.getObject().getValue() == null) continue; KBObject comp = t.getSubject(); String loc = (String) t.getObject().getValue(); File f = new File(loc); loc = this.codedir + File.separator + f.getName(); - lib_writerkb.setPropertyValue(comp, locProp, lib_writerkb.createLiteral(loc)); + lib_writerkb.setPropertyValue( + comp, + locProp, + lib_writerkb.createLiteral(loc) + ); } - - //FIXME: A hack to get the imported domain's resource namespace. Should be explicit - String dcreslibns = dckb.liburl.replaceAll("\\/export\\/users\\/.+$", - "/export/common/resource/library.owl#"); - KBUtils.renameTripleNamespace(lib_writerkb, dcreslibns, this.resliburl+"#"); - this.save(); - this.end(); - dckb.end(); - - this.start_read(); - this.initializeAPI(true, true, true); - } - finally { + + //FIXME: A hack to get the imported domain's resource namespace. Should be explicit + String dcreslibns = dckb.liburl.replaceAll( + "\\/export\\/users\\/.+$", + "/export/common/resource/library.owl#" + ); + KBUtils.renameTripleNamespace( + lib_writerkb, + dcreslibns, + this.resliburl + "#" + ); + this.save(); + this.end(); + dckb.end(); + + this.start_read(); + this.initializeAPI(true, true, true); + } finally { this.end(); dckb.end(); } - } - - @Override - public boolean delete() { - return - this.start_write() && - this.abs_writerkb.delete() && - this.lib_writerkb.delete() && - this.save() && - this.end(); - } - + } + + @Override + public boolean delete() { + return ( + this.start_write() && + this.abs_writerkb.delete() && + this.lib_writerkb.delete() && + this.save() && + this.end() + ); + } + @Override public ComponentCreationAPI getExternalCatalog() { return this.externalCatalog; @@ -670,56 +776,66 @@ public void setExternalCatalog(ComponentCreationAPI cc) { this.externalCatalog = cc; this.externalCatalog.copyFrom(this); } - - /* - * Private helper functions - */ - private void setComponentDocumentation(KBObject compobj, String doc, KBAPI writerkb) { - KBObject docProp = kb.getProperty(this.pcns + "hasDocumentation"); - writerkb.setPropertyValue(compobj, docProp, writerkb.createLiteral(doc)); - } - - private KBObject createRole(ComponentRole role, KBAPI writerkb) { - KBObject argidProp = kb.getProperty(this.pcns + "hasArgumentID"); - KBObject dimProp = kb.getProperty(this.pcns + "hasDimensionality"); - KBObject pfxProp = kb.getProperty(this.pcns + "hasArgumentName"); - KBObject valProp = kb.getProperty(this.pcns + "hasValue"); - - String roletype = this.pcns + (role.isParam() ? "ParameterArgument" : "DataArgument"); - KBObject roletypeobj = this.kb.getConcept(roletype); - KBObject roleobj = writerkb.createObjectOfClass(role.getID(), roletypeobj); - writerkb.setPropertyValue(roleobj, argidProp, writerkb.createLiteral(role.getRoleName())); - writerkb.setPropertyValue(roleobj, dimProp, writerkb.createLiteral(role.getDimensionality())); - writerkb.setPropertyValue(roleobj, pfxProp, writerkb.createLiteral(role.getPrefix())); - - if (role.isParam() && role.getType() != null) { - // Write the parameter default value - String xsdtype = role.getType(); - Object val = role.getParamDefaultalue(); - String valstr = ""; - if (val != null) { - if (xsdtype.matches(".*int.*") && val.getClass() == Double.class) - val = ((Double)val).intValue(); // HACK: gson sometimes converts ints into double - valstr = val.toString(); - } - else if (xsdtype.matches(".*int.*")) - valstr = "0"; - else if (xsdtype.matches(".*bool.*")) - valstr = "false"; - else if (xsdtype.matches(".*float.*")) - valstr = "0.0"; - - KBObject defobj = writerkb.createXSDLiteral(valstr, xsdtype); - if(defobj == null) - return null; - writerkb.setPropertyValue(roleobj, valProp, defobj); - } else if (!role.isParam() && role.getType() != null) { - // Write the role type - KBObject typeobj = kb.getConcept(role.getType()); - if(typeobj != null) - writerkb.addClassForInstance(roleobj, typeobj); - } - return roleobj; - } + /* + * Private helper functions + */ + private void setComponentDocumentation( + KBObject compobj, + String doc, + KBAPI writerkb + ) { + KBObject docProp = kb.getProperty(this.pcns + "hasDocumentation"); + writerkb.setPropertyValue(compobj, docProp, writerkb.createLiteral(doc)); + } + + private KBObject createRole(ComponentRole role, KBAPI writerkb) { + KBObject argidProp = kb.getProperty(this.pcns + "hasArgumentID"); + KBObject dimProp = kb.getProperty(this.pcns + "hasDimensionality"); + KBObject pfxProp = kb.getProperty(this.pcns + "hasArgumentName"); + KBObject valProp = kb.getProperty(this.pcns + "hasValue"); + + String roletype = + this.pcns + (role.isParam() ? "ParameterArgument" : "DataArgument"); + KBObject roletypeobj = this.kb.getConcept(roletype); + KBObject roleobj = writerkb.createObjectOfClass(role.getID(), roletypeobj); + writerkb.setPropertyValue( + roleobj, + argidProp, + writerkb.createLiteral(role.getRoleName()) + ); + writerkb.setPropertyValue( + roleobj, + dimProp, + writerkb.createLiteral(role.getDimensionality()) + ); + writerkb.setPropertyValue( + roleobj, + pfxProp, + writerkb.createLiteral(role.getPrefix()) + ); + + if (role.isParam() && role.getType() != null) { + // Write the parameter default value + String xsdtype = role.getType(); + Object val = role.getParamDefaultalue(); + String valstr = ""; + if (val != null) { + if (xsdtype.matches(".*int.*") && val.getClass() == Double.class) val = + ((Double) val).intValue(); // HACK: gson sometimes converts ints into double + valstr = val.toString(); + } else if (xsdtype.matches(".*int.*")) valstr = "0"; else if ( + xsdtype.matches(".*bool.*") + ) valstr = "false"; else if (xsdtype.matches(".*float.*")) valstr = "0.0"; + + KBObject defobj = writerkb.createXSDLiteral(valstr, xsdtype); + if (defobj == null) return null; + writerkb.setPropertyValue(roleobj, valProp, defobj); + } else if (!role.isParam() && role.getType() != null) { + // Write the role type + KBObject typeobj = kb.getConcept(role.getType()); + if (typeobj != null) writerkb.addClassForInstance(roleobj, typeobj); + } + return roleobj; + } } diff --git a/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentKB.java b/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentKB.java index 208d5f8a..b2dfd20b 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentKB.java +++ b/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentKB.java @@ -17,13 +17,6 @@ package edu.isi.wings.catalog.component.api.impl.kb; -import java.io.File; -import java.io.FileNotFoundException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Properties; -import java.util.Scanner; - import edu.isi.kcap.ontapi.KBAPI; import edu.isi.kcap.ontapi.KBObject; import edu.isi.kcap.ontapi.KBTriple; @@ -40,171 +33,222 @@ import edu.isi.wings.catalog.component.classes.requirements.ComponentRequirement; import edu.isi.wings.catalog.resource.api.ResourceAPI; import edu.isi.wings.common.kb.KBUtils; +import java.io.File; +import java.io.FileNotFoundException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Properties; +import java.util.Scanner; public class ComponentKB extends TransactionsJena { - protected KBAPI kb; - protected KBAPI abs_writerkb, lib_writerkb; - protected String dcns; - protected String pcns; - protected String pcdomns; - protected String dcdomns; - protected String wflowns; - protected String pcurl; - protected String liburl; - protected String absurl; - protected String dconturl; - protected String resonturl; - protected String resliburl; - protected String codedir; - protected String topclass; - - protected ResourceAPI resapi; - - protected HashMap objPropMap; - protected HashMap dataPropMap; - protected HashMap conceptMap; - - protected ArrayList domainKnowledge; - protected HashMap rulePrefixes; - - protected Properties props; - - protected String componentClassSuffix = "Class"; - - /** - * Constructor - * - * @param props - * The properties should contain: lib.concrete.url, - * lib.concrete.rules.path lib.abstract.url, - * lib.abstract.rules.path ont.domain.component.url, - * ont.domain.data.url, ont.component.url, ont.data.url, - * ont.workflow.url tdb.repository.dir (optional) - */ - public ComponentKB(Properties props, boolean create_writers, boolean create_if_empty, boolean plainkb) { - this.props = props; - - String hash = "#"; - this.dcns = props.getProperty("ont.data.url") + hash; - this.pcns = props.getProperty("ont.component.url") + hash; - this.dcdomns = props.getProperty("ont.domain.data.url") + hash; - this.wflowns = props.getProperty("ont.workflow.url") + hash; - this.pcurl = props.getProperty("ont.component.url"); - this.dconturl = props.getProperty("ont.domain.data.url"); - this.pcdomns = props.getProperty("ont.domain.component.ns"); - this.absurl = props.getProperty("lib.abstract.url"); - this.liburl = props.getProperty("lib.concrete.url"); - this.codedir = props.getProperty("lib.domain.code.storage"); + + protected KBAPI kb; + protected KBAPI abs_writerkb, lib_writerkb; + protected String dcns; + protected String pcns; + protected String pcdomns; + protected String dcdomns; + protected String wflowns; + protected String pcurl; + protected String liburl; + protected String absurl; + protected String dconturl; + protected String resonturl; + protected String resliburl; + protected String codedir; + protected String topclass; + + protected ResourceAPI resapi; + + protected HashMap objPropMap; + protected HashMap dataPropMap; + protected HashMap conceptMap; + + protected ArrayList domainKnowledge; + protected HashMap rulePrefixes; + + protected Properties props; + + protected String componentClassSuffix = "Class"; + + /** + * Constructor + * + * @param props + * The properties should contain: lib.concrete.url, + * lib.concrete.rules.path lib.abstract.url, + * lib.abstract.rules.path ont.domain.component.url, + * ont.domain.data.url, ont.component.url, ont.data.url, + * ont.workflow.url tdb.repository.dir (optional) + */ + public ComponentKB( + Properties props, + boolean create_writers, + boolean create_if_empty, + boolean plainkb + ) { + this.props = props; + + String hash = "#"; + this.dcns = props.getProperty("ont.data.url") + hash; + this.pcns = props.getProperty("ont.component.url") + hash; + this.dcdomns = props.getProperty("ont.domain.data.url") + hash; + this.wflowns = props.getProperty("ont.workflow.url") + hash; + this.pcurl = props.getProperty("ont.component.url"); + this.dconturl = props.getProperty("ont.domain.data.url"); + this.pcdomns = props.getProperty("ont.domain.component.ns"); + this.absurl = props.getProperty("lib.abstract.url"); + this.liburl = props.getProperty("lib.concrete.url"); + this.codedir = props.getProperty("lib.domain.code.storage"); this.resonturl = props.getProperty("ont.resource.url"); this.resliburl = props.getProperty("lib.resource.url"); - this.topclass = this.pcns + "Component"; - - String tdbRepository = props.getProperty("tdb.repository.dir"); - if (tdbRepository == null) { - this.ontologyFactory = new OntFactory(OntFactory.JENA); - } else { - this.ontologyFactory = new OntFactory(OntFactory.JENA, tdbRepository); - } - KBUtils.createLocationMappings(props, this.ontologyFactory); - - this.initializeAPI(create_writers, create_if_empty, plainkb); - } - - protected void initializeAPI(boolean create_writers, boolean create_if_empty, boolean plainkb) { - try { - this.kb = this.ontologyFactory.getKB(absurl, - plainkb ? OntSpec.PLAIN : OntSpec.PELLET, create_if_empty); - } - catch(Exception e) { - // Legacy Porting: - // - Absurl is missing. Load liburl and extract abstract components from it - this.createAbstractFromConcrete(); - } - finally { - // Legacy Porting: - // - If rule files are present, move rules as property values into components - this.saveRulesInComponents(props); - } - - try { - this.kb.importFrom(this.ontologyFactory.getKB(liburl, OntSpec.PLAIN, create_if_empty)); - this.kb.importFrom(this.ontologyFactory.getKB(props.getProperty("ont.domain.data.url"), - OntSpec.PLAIN, create_if_empty)); - this.kb.importFrom(this.ontologyFactory.getKB(props.getProperty("ont.component.url"), - OntSpec.PLAIN, create_if_empty, true)); -// this.kb.importFrom(this.ontologyFactory.getKB(props.getProperty("ont.data.url"), -// OntSpec.PLAIN, true, true)); - this.kb.importFrom(this.ontologyFactory.getKB(this.resonturl, - OntSpec.PLAIN, create_if_empty, true)); -// this.kb.importFrom(this.ontologyFactory.getKB(this.resliburl, -// OntSpec.PLAIN, create_if_empty, true)); - - if (create_writers) { - this.lib_writerkb = this.ontologyFactory.getKB(liburl, OntSpec.PLAIN); - this.abs_writerkb = this.ontologyFactory.getKB(absurl, OntSpec.PLAIN); - } - - this.start_read(); - this.initializeMaps(this.kb); - this.initDomainKnowledge(); - this.setRuleMappings(this.kb); - this.end(); - - this.start_write(); - this.addMissingProperties(); - this.save(lib_writerkb); - this.end(); - } - catch (Exception e) { - e.printStackTrace(); - this.end(); - } - } - - private void addMissingProperties() { + this.topclass = this.pcns + "Component"; + + String tdbRepository = props.getProperty("tdb.repository.dir"); + if (tdbRepository == null) { + this.ontologyFactory = new OntFactory(OntFactory.JENA); + } else { + this.ontologyFactory = new OntFactory(OntFactory.JENA, tdbRepository); + } + KBUtils.createLocationMappings(props, this.ontologyFactory); + + this.initializeAPI(create_writers, create_if_empty, plainkb); + } + + protected void initializeAPI( + boolean create_writers, + boolean create_if_empty, + boolean plainkb + ) { + try { + this.kb = + this.ontologyFactory.getKB( + absurl, + plainkb ? OntSpec.PLAIN : OntSpec.PELLET, + create_if_empty + ); + } catch (Exception e) { + // Legacy Porting: + // - Absurl is missing. Load liburl and extract abstract components from it + this.createAbstractFromConcrete(); + } finally { + // Legacy Porting: + // - If rule files are present, move rules as property values into components + this.saveRulesInComponents(props); + } + + try { + this.kb.importFrom( + this.ontologyFactory.getKB(liburl, OntSpec.PLAIN, create_if_empty) + ); + this.kb.importFrom( + this.ontologyFactory.getKB( + props.getProperty("ont.domain.data.url"), + OntSpec.PLAIN, + create_if_empty + ) + ); + this.kb.importFrom( + this.ontologyFactory.getKB( + props.getProperty("ont.component.url"), + OntSpec.PLAIN, + create_if_empty, + true + ) + ); + // this.kb.importFrom(this.ontologyFactory.getKB(props.getProperty("ont.data.url"), + // OntSpec.PLAIN, true, true)); + this.kb.importFrom( + this.ontologyFactory.getKB( + this.resonturl, + OntSpec.PLAIN, + create_if_empty, + true + ) + ); + // this.kb.importFrom(this.ontologyFactory.getKB(this.resliburl, + // OntSpec.PLAIN, create_if_empty, true)); + + if (create_writers) { + this.lib_writerkb = this.ontologyFactory.getKB(liburl, OntSpec.PLAIN); + this.abs_writerkb = this.ontologyFactory.getKB(absurl, OntSpec.PLAIN); + } + + this.start_read(); + this.initializeMaps(this.kb); + this.initDomainKnowledge(); + this.setRuleMappings(this.kb); + this.end(); + + this.start_write(); + this.addMissingProperties(); + this.save(lib_writerkb); + this.end(); + } catch (Exception e) { + e.printStackTrace(); + this.end(); + } + } + + private void addMissingProperties() { // Legacy ontologies don't have some properties. Add them in here - if(!dataPropMap.containsKey("hasLocation")) - dataPropMap.put("hasLocation", lib_writerkb.createDatatypeProperty(this.pcns+"hasLocation")); - if(!dataPropMap.containsKey("hasVersion")) - dataPropMap.put("hasVersion", lib_writerkb.createDatatypeProperty(this.pcns+"hasVersion")); - if(!dataPropMap.containsKey("hasRule")) - dataPropMap.put("hasRule", lib_writerkb.createDatatypeProperty(this.pcns+"hasRule")); - if(!dataPropMap.containsKey("hasDocumentation")) - dataPropMap.put("hasDocumentation", lib_writerkb.createDatatypeProperty(this.pcns+"hasDocumentation")); - if(!dataPropMap.containsKey("isNoOperation")) - dataPropMap.put("isNoOperation", lib_writerkb.createDatatypeProperty(this.pcns+"isNoOperation")); - } - - private void initializeMaps(KBAPI kb) { - this.objPropMap = new HashMap(); - this.dataPropMap = new HashMap(); - this.conceptMap = new HashMap(); - - for (KBObject prop : kb.getAllObjectProperties()) { - this.objPropMap.put(prop.getName(), prop); - this.conceptMap.put(prop.getID(), prop); - } - for (KBObject con : kb.getAllClasses()) { - this.conceptMap.put(con.getName(), con); - this.conceptMap.put(con.getID(), con); - } - for (KBObject odp : kb.getAllDatatypeProperties()) { - this.dataPropMap.put(odp.getName(), odp); - this.dataPropMap.put(odp.getID(), odp); - } - } - - private void initDomainKnowledge() { - // Create general domain knowledge data for use in rules - domainKnowledge = new ArrayList(); - KBObject rdfsSubProp = this.kb.getProperty(KBUtils.RDFS + "subPropertyOf"); - KBObject dcMetricsProp = this.kb.getProperty(this.dcns + "hasMetrics"); - KBObject dcDataMetricsProp = this.kb.getProperty(this.dcns + "hasDataMetrics"); - domainKnowledge.addAll(kb.genericTripleQuery(null, rdfsSubProp, dcMetricsProp)); - domainKnowledge.addAll(kb.genericTripleQuery(null, rdfsSubProp, dcDataMetricsProp)); - } - - protected int getCatalogVersion() { + if (!dataPropMap.containsKey("hasLocation")) dataPropMap.put( + "hasLocation", + lib_writerkb.createDatatypeProperty(this.pcns + "hasLocation") + ); + if (!dataPropMap.containsKey("hasVersion")) dataPropMap.put( + "hasVersion", + lib_writerkb.createDatatypeProperty(this.pcns + "hasVersion") + ); + if (!dataPropMap.containsKey("hasRule")) dataPropMap.put( + "hasRule", + lib_writerkb.createDatatypeProperty(this.pcns + "hasRule") + ); + if (!dataPropMap.containsKey("hasDocumentation")) dataPropMap.put( + "hasDocumentation", + lib_writerkb.createDatatypeProperty(this.pcns + "hasDocumentation") + ); + if (!dataPropMap.containsKey("isNoOperation")) dataPropMap.put( + "isNoOperation", + lib_writerkb.createDatatypeProperty(this.pcns + "isNoOperation") + ); + } + + private void initializeMaps(KBAPI kb) { + this.objPropMap = new HashMap(); + this.dataPropMap = new HashMap(); + this.conceptMap = new HashMap(); + + for (KBObject prop : kb.getAllObjectProperties()) { + this.objPropMap.put(prop.getName(), prop); + this.conceptMap.put(prop.getID(), prop); + } + for (KBObject con : kb.getAllClasses()) { + this.conceptMap.put(con.getName(), con); + this.conceptMap.put(con.getID(), con); + } + for (KBObject odp : kb.getAllDatatypeProperties()) { + this.dataPropMap.put(odp.getName(), odp); + this.dataPropMap.put(odp.getID(), odp); + } + } + + private void initDomainKnowledge() { + // Create general domain knowledge data for use in rules + domainKnowledge = new ArrayList(); + KBObject rdfsSubProp = this.kb.getProperty(KBUtils.RDFS + "subPropertyOf"); + KBObject dcMetricsProp = this.kb.getProperty(this.dcns + "hasMetrics"); + KBObject dcDataMetricsProp = + this.kb.getProperty(this.dcns + "hasDataMetrics"); + domainKnowledge.addAll( + kb.genericTripleQuery(null, rdfsSubProp, dcMetricsProp) + ); + domainKnowledge.addAll( + kb.genericTripleQuery(null, rdfsSubProp, dcDataMetricsProp) + ); + } + + protected int getCatalogVersion() { boolean new_transaction = false; if (!this.is_in_transaction()) { this.start_read(); @@ -212,108 +256,118 @@ protected int getCatalogVersion() { } try { KBObject catalog = this.lib_writerkb.getResource(this.liburl); - KBObject versionProp = this.lib_writerkb.getProperty(KBUtils.OWL + "versionInfo"); - for(KBTriple triple : this.lib_writerkb.genericTripleQuery(catalog, versionProp, null)) { - KBObject versionObj = triple.getObject(); - if(versionObj != null && versionObj.getValue() != null) { - return Integer.valueOf(versionObj.getValue().toString()); - } - } - return 0; + KBObject versionProp = + this.lib_writerkb.getProperty(KBUtils.OWL + "versionInfo"); + for (KBTriple triple : this.lib_writerkb.genericTripleQuery( + catalog, + versionProp, + null + )) { + KBObject versionObj = triple.getObject(); + if (versionObj != null && versionObj.getValue() != null) { + return Integer.valueOf(versionObj.getValue().toString()); + } + } + return 0; + } finally { + if (new_transaction) this.end(); + } + } + + protected void setCatalogVersion(int version) { + boolean new_transaction = false; + if (!this.is_in_transaction()) { + this.start_write(); + new_transaction = true; } - finally { - if(new_transaction) + try { + KBObject catalog = this.lib_writerkb.getResource(this.liburl); + KBObject versionProp = this.kb.getProperty(KBUtils.OWL + "versionInfo"); + KBObject versionObj = this.lib_writerkb.createLiteral("" + version); + this.lib_writerkb.deleteObject(catalog, false, false); + for (KBTriple triple : this.lib_writerkb.genericTripleQuery( + catalog, + versionProp, + null + )) { + this.lib_writerkb.removeTriple(triple); + } + this.lib_writerkb.addTriple(catalog, versionProp, versionObj); + } finally { + if (new_transaction) { + this.save(this.lib_writerkb); this.end(); + } } - } - - protected void setCatalogVersion(int version) { - boolean new_transaction = false; - if (!this.is_in_transaction()) { - this.start_write(); - new_transaction = true; - } - try { - KBObject catalog = this.lib_writerkb.getResource(this.liburl); - KBObject versionProp = this.kb.getProperty(KBUtils.OWL + "versionInfo"); - KBObject versionObj = this.lib_writerkb.createLiteral(""+version); - this.lib_writerkb.deleteObject(catalog, false, false); - for(KBTriple triple : this.lib_writerkb.genericTripleQuery(catalog, versionProp, null)) { - this.lib_writerkb.removeTriple(triple); - } - this.lib_writerkb.addTriple(catalog, versionProp, versionObj); - } - finally { - if(new_transaction) { - this.save(this.lib_writerkb); - this.end(); - } - } - } - - public String getComponentLocation(String cid) { - boolean new_transaction = false; - if(!this.is_in_transaction()) { - this.start_read(); - new_transaction = true; - } - try { - KBObject locprop = this.kb.getProperty(this.pcns + "hasLocation"); - KBObject cobj = this.kb.getIndividual(cid); - KBObject locobj = this.kb.getPropertyValue(cobj, locprop); - if (locobj != null && locobj.getValue() != null) - return locobj.getValueAsString(); - else if (cobj != null) { - String location = this.codedir + File.separator + cobj.getName(); - File f = new File(location); - if(f.exists()) - return location; - } - return null; - } - finally { - if(new_transaction) { - this.end(); - } - } - } - - public String getDefaultComponentLocation(String cid) { - boolean new_transaction = false; - if(!this.is_in_transaction()) { + } + + public String getComponentLocation(String cid) { + boolean new_transaction = false; + if (!this.is_in_transaction()) { this.start_read(); new_transaction = true; } try { - KBObject cobj = this.kb.getIndividual(cid); - return this.codedir + File.separator + cobj.getName(); + KBObject locprop = this.kb.getProperty(this.pcns + "hasLocation"); + KBObject cobj = this.kb.getIndividual(cid); + KBObject locobj = this.kb.getPropertyValue(cobj, locprop); + if ( + locobj != null && locobj.getValue() != null + ) return locobj.getValueAsString(); else if (cobj != null) { + String location = this.codedir + File.separator + cobj.getName(); + File f = new File(location); + if (f.exists()) return location; + } + return null; + } finally { + if (new_transaction) { + this.end(); + } } - finally { - if(new_transaction) { + } + + public String getDefaultComponentLocation(String cid) { + boolean new_transaction = false; + if (!this.is_in_transaction()) { + this.start_read(); + new_transaction = true; + } + try { + KBObject cobj = this.kb.getIndividual(cid); + return this.codedir + File.separator + cobj.getName(); + } finally { + if (new_transaction) { this.end(); } } - } - - public ArrayList getParentComponentTypes(String cid) { - ArrayList ctypes = new ArrayList(); + } + + public ArrayList getParentComponentTypes(String cid) { + ArrayList ctypes = new ArrayList(); try { - this.read(()->{ - KBObject compobj = this.kb.getIndividual(cid); - for(KBObject clsobj : this.kb.getAllClassesOfInstance(compobj, true)) { - for(KBObject superclsobj : this.kb.getSuperClasses(clsobj, false)) { - for(KBObject supercompobj : this.kb.getInstancesOfClass(superclsobj, true)) - ctypes.add(supercompobj.getID()); + this.read(() -> { + KBObject compobj = this.kb.getIndividual(cid); + for (KBObject clsobj : this.kb.getAllClassesOfInstance( + compobj, + true + )) { + for (KBObject superclsobj : this.kb.getSuperClasses( + clsobj, + false + )) { + for (KBObject supercompobj : this.kb.getInstancesOfClass( + superclsobj, + true + )) ctypes.add(supercompobj.getID()); + } } - } - }); - } - catch (Exception e) { + }); + } catch (Exception e) { e.printStackTrace(); } return ctypes; - } - + } + protected KBAPI getWriterKB(String cid) { boolean new_transaction = false; if (!this.is_in_transaction()) { @@ -322,376 +376,428 @@ protected KBAPI getWriterKB(String cid) { } try { KBObject compobj = this.kb.getIndividual(cid); - KBObject concobj = kb.getDatatypePropertyValue(compobj, this.dataPropMap.get("isConcrete")); + KBObject concobj = kb.getDatatypePropertyValue( + compobj, + this.dataPropMap.get("isConcrete") + ); boolean isConcrete = false; - if(concobj != null && concobj.getValue() != null) - isConcrete = ((Boolean) concobj.getValue()).booleanValue(); - + if (concobj != null && concobj.getValue() != null) isConcrete = + ((Boolean) concobj.getValue()).booleanValue(); + return (isConcrete ? this.lib_writerkb : this.abs_writerkb); + } finally { + if (new_transaction) this.end(); } - finally { - if(new_transaction) - this.end(); + } + + protected KBAPI getWriterKB(boolean isConcrete) { + if (isConcrete) { + return this.lib_writerkb; + } + return this.abs_writerkb; + } + + protected void setComponentRules(String cid, String text, KBAPI writerkb) { + KBObject compobj = writerkb.getIndividual(cid); + KBObject ruleProp = this.dataPropMap.get("hasRule"); + try { + for (KBTriple t : writerkb.genericTripleQuery(compobj, ruleProp, null)) { + writerkb.removeTriple(t); + } + for (KBRule rule : ontologyFactory.parseRules(text).getRules()) { + KBObject ruleobj = writerkb.createLiteral(rule.toString()); + writerkb.addPropertyValue(compobj, ruleProp, ruleobj); + } + } catch (Exception e) { + e.printStackTrace(); } } - - protected KBAPI getWriterKB(boolean isConcrete) { - if(isConcrete) { - return this.lib_writerkb; - } - return this.abs_writerkb; - } - - protected void setComponentRules(String cid, String text, KBAPI writerkb) { - KBObject compobj = writerkb.getIndividual(cid); - KBObject ruleProp = this.dataPropMap.get("hasRule"); - try { - for(KBTriple t : writerkb.genericTripleQuery(compobj, ruleProp, null)) { - writerkb.removeTriple(t); - } - for(KBRule rule : ontologyFactory.parseRules(text).getRules()) { - KBObject ruleobj = writerkb.createLiteral(rule.toString()); - writerkb.addPropertyValue(compobj, ruleProp, ruleobj); - } - } - catch (Exception e) { - e.printStackTrace(); - } - } - - protected KBRuleList getComponentRules(String cid) { - KBRuleList comprules = this.getDirectComponentRules(cid); - comprules.mergeRules(this.getInheritedComponentRules(cid)); - return comprules; - } - - protected KBRuleList getDirectComponentRules(String cid) { - String rulestr = ""; - KBObject compobj = this.kb.getIndividual(cid); - KBObject ruleProp = this.dataPropMap.get("hasRule"); - for(KBObject ruleobj : this.kb.getPropertyValues(compobj, ruleProp)) - rulestr += ruleobj.getValueAsString(); - try { - return ontologyFactory.parseRules(rulestr); - } - catch (Exception e) { - return ontologyFactory.createEmptyRuleList(); - } - } - - protected KBRuleList getInheritedComponentRules(String cid) { - try { - String rulestr = ""; - KBObject compobj = this.kb.getIndividual(cid); - KBObject ruleProp = this.dataPropMap.get("hasRule"); - for(KBObject clsobj : this.kb.getAllClassesOfInstance(compobj, true)) { - for(KBObject superclsobj : this.kb.getSuperClasses(clsobj, false)) { - for(KBObject supercompobj : this.kb.getInstancesOfClass(superclsobj, true)) - for(KBObject ruleobj : this.kb.getPropertyValues(supercompobj, ruleProp)) - rulestr += ruleobj.getValueAsString(); - } - } - return ontologyFactory.parseRules(rulestr); + + protected KBRuleList getComponentRules(String cid) { + KBRuleList comprules = this.getDirectComponentRules(cid); + comprules.mergeRules(this.getInheritedComponentRules(cid)); + return comprules; + } + + protected KBRuleList getDirectComponentRules(String cid) { + String rulestr = ""; + KBObject compobj = this.kb.getIndividual(cid); + KBObject ruleProp = this.dataPropMap.get("hasRule"); + for (KBObject ruleobj : this.kb.getPropertyValues( + compobj, + ruleProp + )) rulestr += ruleobj.getValueAsString(); + try { + return ontologyFactory.parseRules(rulestr); + } catch (Exception e) { + return ontologyFactory.createEmptyRuleList(); } - catch (Exception e) { + } + + protected KBRuleList getInheritedComponentRules(String cid) { + try { + String rulestr = ""; + KBObject compobj = this.kb.getIndividual(cid); + KBObject ruleProp = this.dataPropMap.get("hasRule"); + for (KBObject clsobj : this.kb.getAllClassesOfInstance(compobj, true)) { + for (KBObject superclsobj : this.kb.getSuperClasses(clsobj, false)) { + for (KBObject supercompobj : this.kb.getInstancesOfClass( + superclsobj, + true + )) for (KBObject ruleobj : this.kb.getPropertyValues( + supercompobj, + ruleProp + )) rulestr += ruleobj.getValueAsString(); + } + } + return ontologyFactory.parseRules(rulestr); + } catch (Exception e) { return ontologyFactory.createEmptyRuleList(); } - } - - - protected ComponentRequirement getComponentRequirements(KBObject compobj, - KBAPI tkb) { + } + + protected ComponentRequirement getComponentRequirements( + KBObject compobj, + KBAPI tkb + ) { ComponentRequirement requirement = new ComponentRequirement(); - + KBObject sdprop = this.objPropMap.get("hasSoftwareDependency"); KBObject hdprop = this.objPropMap.get("hasHardwareDependency"); KBObject minver = this.objPropMap.get("requiresMinimumVersion"); //KBObject exver = this.objPropMap.get("requiresExactVersion"); - for(KBObject sdobj : tkb.getPropertyValues(compobj, sdprop)) { + for (KBObject sdobj : tkb.getPropertyValues(compobj, sdprop)) { KBObject verobj = tkb.getPropertyValue(sdobj, minver); - if(verobj != null) - requirement.addSoftwareId(verobj.getID()); + if (verobj != null) requirement.addSoftwareId(verobj.getID()); } KBObject hdobj = tkb.getPropertyValue(compobj, hdprop); - if(hdobj != null) { - KBObject memobj = tkb.getPropertyValue(hdobj, - this.dataPropMap.get("requiresMemoryGB")); - KBObject stobj = tkb.getPropertyValue(hdobj, - this.dataPropMap.get("requiresStorageGB")); - KBObject s4bitobj = tkb.getPropertyValue(hdobj, - this.dataPropMap.get("needs64bit")); - - if(memobj != null && memobj.getValue() != null) - requirement.setMemoryGB(Float.parseFloat(memobj.getValueAsString())); - if(stobj != null && stobj.getValue() != null) - requirement.setStorageGB(Float.parseFloat(stobj.getValueAsString())); - if(s4bitobj != null) - requirement.setNeed64bit((Boolean)s4bitobj.getValue()); + if (hdobj != null) { + KBObject memobj = tkb.getPropertyValue( + hdobj, + this.dataPropMap.get("requiresMemoryGB") + ); + KBObject stobj = tkb.getPropertyValue( + hdobj, + this.dataPropMap.get("requiresStorageGB") + ); + KBObject s4bitobj = tkb.getPropertyValue( + hdobj, + this.dataPropMap.get("needs64bit") + ); + + if (memobj != null && memobj.getValue() != null) requirement.setMemoryGB( + Float.parseFloat(memobj.getValueAsString()) + ); + if (stobj != null && stobj.getValue() != null) requirement.setStorageGB( + Float.parseFloat(stobj.getValueAsString()) + ); + if (s4bitobj != null) requirement.setNeed64bit( + (Boolean) s4bitobj.getValue() + ); } - + return requirement; } - - protected void setComponentRequirements(KBObject compobj, ComponentRequirement requirement, KBAPI writerkb) { + protected void setComponentRequirements( + KBObject compobj, + ComponentRequirement requirement, + KBAPI writerkb + ) { KBObject sdprop = this.objPropMap.get("hasSoftwareDependency"); KBObject sdcls = this.conceptMap.get("SoftwareDependency"); KBObject hdprop = this.objPropMap.get("hasHardwareDependency"); KBObject hdcls = this.conceptMap.get("HardwareDependency"); KBObject minver = this.objPropMap.get("requiresMinimumVersion"); - //KBObject exver = this.objPropMap.get("requiresExactVersion"); + //KBObject exver = this.objPropMap.get("requiresExactVersion"); - if(requirement.getSoftwareIds() != null) { + if (requirement.getSoftwareIds() != null) { for (String softwareId : requirement.getSoftwareIds()) { KBObject sdobj = writerkb.createObjectOfClass(null, sdcls); KBObject verobj = this.kb.getResource(softwareId); - if(verobj != null) - writerkb.setPropertyValue(sdobj, minver, verobj); + if (verobj != null) writerkb.setPropertyValue(sdobj, minver, verobj); writerkb.addPropertyValue(compobj, sdprop, sdobj); } } - + KBObject hdobj = writerkb.createObjectOfClass(null, hdcls); - writerkb.setPropertyValue(hdobj, - this.dataPropMap.get("requiresMemoryGB"), - writerkb.createLiteral(requirement.getMemoryGB())); - writerkb.setPropertyValue(hdobj, - this.dataPropMap.get("requiresStorageGB"), - writerkb.createLiteral(requirement.getStorageGB())); - writerkb.setPropertyValue(hdobj, - this.dataPropMap.get("needs64bit"), - writerkb.createLiteral(requirement.isNeed64bit())); + writerkb.setPropertyValue( + hdobj, + this.dataPropMap.get("requiresMemoryGB"), + writerkb.createLiteral(requirement.getMemoryGB()) + ); + writerkb.setPropertyValue( + hdobj, + this.dataPropMap.get("requiresStorageGB"), + writerkb.createLiteral(requirement.getStorageGB()) + ); + writerkb.setPropertyValue( + hdobj, + this.dataPropMap.get("needs64bit"), + writerkb.createLiteral(requirement.isNeed64bit()) + ); writerkb.setPropertyValue(compobj, hdprop, hdobj); } - /* - * Legacy component library porting functions - */ - - private void saveRulesInComponents(Properties props) { - File absrulesfile = null, librulesfile = null; - if(props.containsKey("lib.abstract.rules.path")) - absrulesfile = new File(props.getProperty("lib.abstract.rules.path")); - if(props.containsKey("lib.concrete.rules.path")) - librulesfile = new File(props.getProperty("lib.concrete.rules.path")); - - if((absrulesfile != null && absrulesfile.exists()) || - (librulesfile != null && librulesfile.exists())) { - String rulestr = this.readRulesFromFile(absrulesfile); - rulestr += this.readRulesFromFile(librulesfile); - if(rulestr.equals("")) - return; - - try { - this.start_write(); - - KBAPI abskb = this.ontologyFactory.getKB(absurl, OntSpec.PLAIN); - KBAPI libkb = this.ontologyFactory.getKB(liburl, OntSpec.PLAIN); - - KBAPI kb = this.ontologyFactory.getKB(absurl, OntSpec.PLAIN); - kb.importFrom(this.ontologyFactory.getKB( - this.props.getProperty("ont.component.url"), OntSpec.PLAIN)); - KBObject ruleProp = kb.getProperty(this.pcns+"hasRule"); - - this.setRuleMappings(kb); - KBRuleList rulelist = ontologyFactory.parseRules(rulestr); - for (KBRule rule : rulelist.getRules()) { - boolean found = false; - KBObject ruleobj = - ontologyFactory.getDataObject(rule.toString()); - - for (KBRuleClause clause : rule.getRuleBody()) { - if (clause.isTriple()) { - KBRuleTriple triple = clause.getTriple(); - KBRuleObject subj = triple.getSubject(); - KBRuleObject pred = triple.getPredicate(); - KBRuleObject obj = triple.getObject(); - if (subj == null || pred == null || obj == null) - continue; - if (subj.isVariable() && !pred.isVariable() && !obj.isVariable()) { - if (pred.getKBObject().getID().equals(KBUtils.RDF + "type") - && (obj.getKBObject().getID().endsWith(componentClassSuffix))) { - KBObject cls = abskb.getConcept(obj.getKBObject().getID()); - if(cls != null) { - for(KBObject compobj : abskb.getInstancesOfClass(cls, true)) - abskb.addPropertyValue(compobj, ruleProp, ruleobj); - found = true; - break; - } - else { - cls = libkb.getConcept(obj.getKBObject().getID()); - if(cls != null) { - for(KBObject compobj : libkb.getInstancesOfClass(cls, true)) - libkb.addPropertyValue(compobj, ruleProp, ruleobj); - } - found = true; - break; - } - } - } - } - } - - if(!found) { - // Miscellaneous rule. Add to all topmost objects - for(KBObject subcls : kb.getSubClasses(kb.getConcept(this.topclass), true)) { - for(KBObject kbcobj : kb.getInstancesOfClass(subcls, true)) { - KBObject compobj = abskb.getIndividual(kbcobj.getID()); - if(compobj != null) - abskb.addPropertyValue(compobj, ruleProp, ruleobj); - } - } - } - } - this.save(abskb); - this.save(libkb); - this.end(); - - this.kb = this.ontologyFactory.getKB(this.absurl, OntSpec.PELLET); - - if (absrulesfile.exists()) - absrulesfile.delete(); - if (librulesfile.exists()) - librulesfile.delete(); - - } catch (Exception e) { - this.end(); - e.printStackTrace(); - } - } - } - - - // Legacy Porting code - private void createAbstractFromConcrete() { - try { - this.start_read(); - KBAPI abskb = this.ontologyFactory.getKB(OntSpec.PLAIN); - KBAPI libkb = this.ontologyFactory.getKB(liburl, OntSpec.PLAIN); - KBAPI kb = this.ontologyFactory.getKB(liburl, OntSpec.PELLET); - kb.importFrom(this.ontologyFactory.getKB(this.props.getProperty("ont.component.url"), OntSpec.PLAIN)); - this.end(); - - this.start_write(); - this.initializeMaps(kb); - this.end(); - - HashMap cmap = this.conceptMap; - HashMap dpmap = this.dataPropMap; - HashMap opmap = this.objPropMap; - - ArrayList triples = new ArrayList(); - - this.start_read(); - ArrayList compobjs = kb.getInstancesOfClass(cmap.get("Component"), false); - for(KBObject compobj : compobjs) { - KBObject dval = kb.getDatatypePropertyValue(compobj, dpmap.get("isConcrete")); - if(dval != null && dval.getValue() != null && !(Boolean)dval.getValue()) { - // Get Component Class - KBObject compcls = libkb.getClassOfInstance(compobj); - - // Read and write from plain kb so as not to get any entailments - triples = libkb.genericTripleQuery(compobj, null, null); - triples.addAll(libkb.genericTripleQuery(compcls, null, null)); - - for(KBObject inputobj : libkb.getPropertyValues(compobj, opmap.get("hasInput"))) - triples.addAll(libkb.genericTripleQuery(inputobj, null, null)); - for(KBObject outputobj : libkb.getPropertyValues(compobj, opmap.get("hasOutput"))) - triples.addAll(libkb.genericTripleQuery(outputobj, null, null)); - } - } - this.end(); - - this.start_write(); - for(KBTriple triple : triples) - libkb.removeTriple(triple); + /* + * Legacy component library porting functions + */ + + private void saveRulesInComponents(Properties props) { + File absrulesfile = null, librulesfile = null; + if (props.containsKey("lib.abstract.rules.path")) absrulesfile = + new File(props.getProperty("lib.abstract.rules.path")); + if (props.containsKey("lib.concrete.rules.path")) librulesfile = + new File(props.getProperty("lib.concrete.rules.path")); + + if ( + (absrulesfile != null && absrulesfile.exists()) || + (librulesfile != null && librulesfile.exists()) + ) { + String rulestr = this.readRulesFromFile(absrulesfile); + rulestr += this.readRulesFromFile(librulesfile); + if (rulestr.equals("")) return; + + try { + this.start_write(); + + KBAPI abskb = this.ontologyFactory.getKB(absurl, OntSpec.PLAIN); + KBAPI libkb = this.ontologyFactory.getKB(liburl, OntSpec.PLAIN); + + KBAPI kb = this.ontologyFactory.getKB(absurl, OntSpec.PLAIN); + kb.importFrom( + this.ontologyFactory.getKB( + this.props.getProperty("ont.component.url"), + OntSpec.PLAIN + ) + ); + KBObject ruleProp = kb.getProperty(this.pcns + "hasRule"); + + this.setRuleMappings(kb); + KBRuleList rulelist = ontologyFactory.parseRules(rulestr); + for (KBRule rule : rulelist.getRules()) { + boolean found = false; + KBObject ruleobj = ontologyFactory.getDataObject(rule.toString()); + + for (KBRuleClause clause : rule.getRuleBody()) { + if (clause.isTriple()) { + KBRuleTriple triple = clause.getTriple(); + KBRuleObject subj = triple.getSubject(); + KBRuleObject pred = triple.getPredicate(); + KBRuleObject obj = triple.getObject(); + if (subj == null || pred == null || obj == null) continue; + if ( + subj.isVariable() && !pred.isVariable() && !obj.isVariable() + ) { + if ( + pred.getKBObject().getID().equals(KBUtils.RDF + "type") && + (obj.getKBObject().getID().endsWith(componentClassSuffix)) + ) { + KBObject cls = abskb.getConcept(obj.getKBObject().getID()); + if (cls != null) { + for (KBObject compobj : abskb.getInstancesOfClass( + cls, + true + )) abskb.addPropertyValue(compobj, ruleProp, ruleobj); + found = true; + break; + } else { + cls = libkb.getConcept(obj.getKBObject().getID()); + if (cls != null) { + for (KBObject compobj : libkb.getInstancesOfClass( + cls, + true + )) libkb.addPropertyValue(compobj, ruleProp, ruleobj); + } + found = true; + break; + } + } + } + } + } + + if (!found) { + // Miscellaneous rule. Add to all topmost objects + for (KBObject subcls : kb.getSubClasses( + kb.getConcept(this.topclass), + true + )) { + for (KBObject kbcobj : kb.getInstancesOfClass(subcls, true)) { + KBObject compobj = abskb.getIndividual(kbcobj.getID()); + if (compobj != null) abskb.addPropertyValue( + compobj, + ruleProp, + ruleobj + ); + } + } + } + } + this.save(abskb); + this.save(libkb); + this.end(); + + this.kb = this.ontologyFactory.getKB(this.absurl, OntSpec.PELLET); + + if (absrulesfile.exists()) absrulesfile.delete(); + if (librulesfile.exists()) librulesfile.delete(); + } catch (Exception e) { + this.end(); + e.printStackTrace(); + } + } + } + + // Legacy Porting code + private void createAbstractFromConcrete() { + try { + this.start_read(); + KBAPI abskb = this.ontologyFactory.getKB(OntSpec.PLAIN); + KBAPI libkb = this.ontologyFactory.getKB(liburl, OntSpec.PLAIN); + KBAPI kb = this.ontologyFactory.getKB(liburl, OntSpec.PELLET); + kb.importFrom( + this.ontologyFactory.getKB( + this.props.getProperty("ont.component.url"), + OntSpec.PLAIN + ) + ); + this.end(); + + this.start_write(); + this.initializeMaps(kb); + this.end(); + + HashMap cmap = this.conceptMap; + HashMap dpmap = this.dataPropMap; + HashMap opmap = this.objPropMap; + + ArrayList triples = new ArrayList(); + + this.start_read(); + ArrayList compobjs = kb.getInstancesOfClass( + cmap.get("Component"), + false + ); + for (KBObject compobj : compobjs) { + KBObject dval = kb.getDatatypePropertyValue( + compobj, + dpmap.get("isConcrete") + ); + if ( + dval != null && dval.getValue() != null && !(Boolean) dval.getValue() + ) { + // Get Component Class + KBObject compcls = libkb.getClassOfInstance(compobj); + + // Read and write from plain kb so as not to get any entailments + triples = libkb.genericTripleQuery(compobj, null, null); + triples.addAll(libkb.genericTripleQuery(compcls, null, null)); + + for (KBObject inputobj : libkb.getPropertyValues( + compobj, + opmap.get("hasInput") + )) triples.addAll(libkb.genericTripleQuery(inputobj, null, null)); + for (KBObject outputobj : libkb.getPropertyValues( + compobj, + opmap.get("hasOutput") + )) triples.addAll(libkb.genericTripleQuery(outputobj, null, null)); + } + } + this.end(); + + this.start_write(); + for (KBTriple triple : triples) libkb.removeTriple(triple); libkb.save(); - abskb.addTriples(triples); - abskb.createImport(this.absurl, props.getProperty("ont.component.url")); - abskb.saveAs(this.absurl); - this.end(); - - this.kb = this.ontologyFactory.getKB(this.absurl, OntSpec.PELLET); - } - catch (Exception e) { - this.end(); - e.printStackTrace(); - return; - } - } - - private String readRulesFromFile(File f) { - String str = ""; - try { - if(!f.canRead()) return str; - Scanner sc = new Scanner(f); - while (sc.hasNextLine()) { - String line = ignoreComments(sc.nextLine()); - if (line != null) - str += line + "\n"; - } - sc.close(); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } - return str; - } - - private String ignoreComments(String line) { - String result_line = null; - int upto = line.indexOf('#'); - if (upto != 0 && upto > 0) { - result_line = line.substring(0, upto); - } else if (upto < 0) { - result_line = line; - } - return result_line; - } - + abskb.addTriples(triples); + abskb.createImport(this.absurl, props.getProperty("ont.component.url")); + abskb.saveAs(this.absurl); + this.end(); + + this.kb = this.ontologyFactory.getKB(this.absurl, OntSpec.PELLET); + } catch (Exception e) { + this.end(); + e.printStackTrace(); + return; + } + } + + private String readRulesFromFile(File f) { + String str = ""; + try { + if (!f.canRead()) return str; + Scanner sc = new Scanner(f); + while (sc.hasNextLine()) { + String line = ignoreComments(sc.nextLine()); + if (line != null) str += line + "\n"; + } + sc.close(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + return str; + } + + private String ignoreComments(String line) { + String result_line = null; + int upto = line.indexOf('#'); + if (upto != 0 && upto > 0) { + result_line = line.substring(0, upto); + } else if (upto < 0) { + result_line = line; + } + return result_line; + } + public String getComponentHolderId(String cid) { //Component holder id is Class return cid + componentClassSuffix; } - + public String getComponentTypeFromHolder(String holderid) { //Component holder id is Class return holderid.replace(componentClassSuffix, ""); - } - + } + public Component getComponent(String cid, boolean details) { this.start_read(); - + try { KBObject compobj = kb.getIndividual(cid); - if(compobj == null) return null; - - KBObject concobj = kb.getDatatypePropertyValue(compobj, this.dataPropMap.get("isConcrete")); + if (compobj == null) return null; + + KBObject concobj = kb.getDatatypePropertyValue( + compobj, + this.dataPropMap.get("isConcrete") + ); boolean isConcrete = false; - if(concobj != null && concobj.getValue() != null) - isConcrete = ((Boolean) concobj.getValue()).booleanValue(); + if (concobj != null && concobj.getValue() != null) isConcrete = + ((Boolean) concobj.getValue()).booleanValue(); int ctype = isConcrete ? Component.CONCRETE : Component.ABSTRACT; - + Component comp = new Component(compobj.getID(), ctype); - if (isConcrete) - comp.setLocation(this.getComponentLocation(cid)); - + if (isConcrete) comp.setLocation(this.getComponentLocation(cid)); + if (details) { - comp.setDocumentation(this.getComponentDocumentation(compobj)); - //obtain component type - KBObject classComponent = this.kb.getClassOfInstance(this.kb.getIndividual(cid)); - if(classComponent != null && classComponent.getID() != null) { - ArrayList clss = this.kb.getSuperClasses(classComponent, true); - if (clss != null && clss.size() > 0){ - KBObject cls = clss.get(0); - String comptype = this.getComponentTypeFromHolder(cls.getName()); - comp.setComponentType(comptype); - } - } + comp.setDocumentation(this.getComponentDocumentation(compobj)); + //obtain component type + KBObject classComponent = + this.kb.getClassOfInstance(this.kb.getIndividual(cid)); + if (classComponent != null && classComponent.getID() != null) { + ArrayList clss = + this.kb.getSuperClasses(classComponent, true); + if (clss != null && clss.size() > 0) { + KBObject cls = clss.get(0); + String comptype = this.getComponentTypeFromHolder(cls.getName()); + comp.setComponentType(comptype); + } + } comp.setComponentRequirement( - this.getComponentRequirements(compobj, this.kb)); - + this.getComponentRequirements(compobj, this.kb) + ); + ArrayList inobjs = this.getComponentInputs(compobj); for (KBObject inobj : inobjs) { comp.addInput(this.getRole(inobj)); @@ -706,18 +812,21 @@ public Component getComponent(String cid, boolean details) { comp.setVersion(this.getComponentVersion(compobj)); } return comp; + } finally { + this.end(); } - finally { - this.end(); - } - } - + } + protected int getComponentVersion(KBObject compobj) { KBObject versionProp = kb.getProperty(this.pcns + "hasVersion"); KBObject versionVal = kb.getPropertyValue(compobj, versionProp); - return (Integer) ((versionVal != null && versionVal.getValue() != null) ? versionVal.getValue() : 0); + return (Integer) ( + (versionVal != null && versionVal.getValue() != null) + ? versionVal.getValue() + : 0 + ); } - + protected ArrayList getComponentInputs(KBObject compobj) { KBObject inProp = kb.getProperty(this.pcns + "hasInput"); return kb.getPropertyValues(compobj, inProp); @@ -731,19 +840,17 @@ protected ArrayList getComponentOutputs(KBObject compobj) { protected String getComponentDocumentation(KBObject compobj) { KBObject docProp = kb.getProperty(this.pcns + "hasDocumentation"); KBObject doc = kb.getPropertyValue(compobj, docProp); - if(doc != null && doc.getValue() != null) - return doc.getValueAsString(); + if (doc != null && doc.getValue() != null) return doc.getValueAsString(); + return null; + } + + protected String getComponentModelCatalog(KBObject compobj) { + KBObject docProp = kb.getProperty(this.pcns + "source"); + KBObject doc = kb.getPropertyValue(compobj, docProp); + if (doc != null && doc.getValue() != null) return doc.getValueAsString(); return null; } - protected String getComponentModelCatalog(KBObject compobj) { - KBObject docProp = kb.getProperty(this.pcns + "source"); - KBObject doc = kb.getPropertyValue(compobj, docProp); - if(doc != null && doc.getValue() != null) - return doc.getValueAsString(); - return null; - } - protected ComponentRole getRole(KBObject argobj) { ComponentRole arg = new ComponentRole(argobj.getID()); KBObject argidProp = kb.getProperty(this.pcns + "hasArgumentID"); @@ -754,13 +861,14 @@ protected ComponentRole getRole(KBObject argobj) { ArrayList alltypes = kb.getAllClassesOfInstance(argobj, true); for (KBObject type : alltypes) { - if (type.getID().equals(this.pcns + "ParameterArgument")) - arg.setParam(true); - else if (type.getID().equals(this.pcns + "DataArgument")) - arg.setParam(false); - else if (type.getNamespace().equals(this.dcdomns) - || type.getNamespace().equals(this.dcns)) - arg.setType(type.getID()); + if (type.getID().equals(this.pcns + "ParameterArgument")) arg.setParam( + true + ); else if (type.getID().equals(this.pcns + "DataArgument")) arg.setParam( + false + ); else if ( + type.getNamespace().equals(this.dcdomns) || + type.getNamespace().equals(this.dcns) + ) arg.setType(type.getID()); } KBObject role = kb.getPropertyValue(argobj, argidProp); KBObject dim = kb.getPropertyValue(argobj, dimProp); @@ -773,37 +881,40 @@ else if (type.getNamespace().equals(this.dcdomns) arg.setParamDefaultalue(val.getValue()); } } - if (role != null && role.getValue() != null) - arg.setRoleName(role.getValueAsString()); - if (dim != null && dim.getValue() != null) - arg.setDimensionality((Integer) dim.getValue()); - if (pfx != null && pfx.getValue() != null) - arg.setPrefix(pfx.getValueAsString()); - + if (role != null && role.getValue() != null) arg.setRoleName( + role.getValueAsString() + ); + if (dim != null && dim.getValue() != null) arg.setDimensionality( + (Integer) dim.getValue() + ); + if (pfx != null && pfx.getValue() != null) arg.setPrefix( + pfx.getValueAsString() + ); + return arg; } - - /** - * Set Rule Prefix-Namespace Mappings Prefixes allowed in Rules: rdf, rdfs, - * owl, xsd -- usual dc, dcdom -- data catalog pc, pcdom -- component - * catalog ac, acdom -- synonyms for pc, pcdom - */ - private void setRuleMappings(KBAPI kb) { - rulePrefixes = new HashMap(); - rulePrefixes.put("rdf", KBUtils.RDF); - rulePrefixes.put("rdfs", KBUtils.RDFS); - rulePrefixes.put("owl", KBUtils.OWL); - rulePrefixes.put("xsd", KBUtils.XSD); - rulePrefixes.put("", this.wflowns); - rulePrefixes.put("dcdom", this.dcdomns); - rulePrefixes.put("dc", this.dcns); - rulePrefixes.put("pcdom", this.pcdomns); - rulePrefixes.put("pc", this.pcns); - rulePrefixes.put("acdom", this.pcdomns); // Legacy - rulePrefixes.put("ac", this.pcns); // Legacy - rulePrefixes.put("wflow", this.wflowns); - rulePrefixes.put("res", this.resonturl+"#"); - rulePrefixes.put("reslib", this.resliburl+"#"); - kb.setRulePrefixes(rulePrefixes); - } + + /** + * Set Rule Prefix-Namespace Mappings Prefixes allowed in Rules: rdf, rdfs, + * owl, xsd -- usual dc, dcdom -- data catalog pc, pcdom -- component + * catalog ac, acdom -- synonyms for pc, pcdom + */ + private void setRuleMappings(KBAPI kb) { + rulePrefixes = new HashMap(); + rulePrefixes.put("rdf", KBUtils.RDF); + rulePrefixes.put("rdfs", KBUtils.RDFS); + rulePrefixes.put("owl", KBUtils.OWL); + rulePrefixes.put("xsd", KBUtils.XSD); + rulePrefixes.put("", this.wflowns); + rulePrefixes.put("dcdom", this.dcdomns); + rulePrefixes.put("dc", this.dcns); + rulePrefixes.put("pcdom", this.pcdomns); + rulePrefixes.put("pc", this.pcns); + rulePrefixes.put("acdom", this.pcdomns); // Legacy + rulePrefixes.put("ac", this.pcns); // Legacy + rulePrefixes.put("wflow", this.wflowns); + rulePrefixes.put("res", this.resonturl + "#"); + rulePrefixes.put("reslib", this.resliburl + "#"); + kb.setRulePrefixes(rulePrefixes); + } } diff --git a/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentReasoningKB.java b/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentReasoningKB.java index 68564880..59b53864 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentReasoningKB.java +++ b/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/ComponentReasoningKB.java @@ -17,9 +17,6 @@ package edu.isi.wings.catalog.component.api.impl.kb; -import org.apache.commons.lang.SystemUtils; -import org.apache.log4j.Logger; - import edu.isi.kcap.ontapi.*; import edu.isi.kcap.ontapi.rules.KBRuleList; import edu.isi.wings.catalog.component.api.ComponentReasoningAPI; @@ -41,102 +38,131 @@ import edu.isi.wings.workflow.template.classes.variables.ComponentVariable; import edu.isi.wings.workflow.template.classes.variables.Variable; import edu.isi.wings.workflow.template.classes.variables.VariableType; - import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; import java.util.*; +import org.apache.commons.lang.SystemUtils; +import org.apache.log4j.Logger; + +public class ComponentReasoningKB + extends ComponentKB + implements ComponentReasoningAPI { -public class ComponentReasoningKB extends ComponentKB implements ComponentReasoningAPI { - private Logger logger = Logger.getLogger(this.getClass()); - private boolean useRules = true; + private Logger logger = Logger.getLogger(this.getClass()); + private boolean useRules = true; - private HashMap ccache = - new HashMap(); - private HashMap rulescache = - new HashMap(); + private HashMap ccache = new HashMap(); + private HashMap rulescache = + new HashMap(); private HashMap> abscache = - new HashMap>(); + new HashMap>(); private HashMap> classcache = - new HashMap>(); - private HashMap> kbcache = - new HashMap>(); - - private ArrayList metricTriples; - private ArrayList metricProps; - - public ComponentReasoningKB(Properties props) { - super(props, true, true, false); - this.useRules = Boolean.parseBoolean(props.getProperty("use_rules", "true")); - this.initializeMetrics(); - } - - private void initializeMetrics() { - this.start_read(); - + new HashMap>(); + private HashMap> kbcache = + new HashMap>(); + + private ArrayList metricTriples; + private ArrayList metricProps; + + public ComponentReasoningKB(Properties props) { + super(props, true, true, false); + this.useRules = + Boolean.parseBoolean(props.getProperty("use_rules", "true")); + this.initializeMetrics(); + } + + private void initializeMetrics() { + this.start_read(); + KBObject rdfsProp = this.kb.getProperty(KBUtils.RDFS + "subPropertyOf"); KBObject dcProp = this.kb.getProperty(this.dcns + "hasMetrics"); KBObject dcPropD = this.kb.getProperty(this.dcns + "hasDataMetrics"); - + // Get all metric properties this.metricTriples = this.kb.genericTripleQuery(null, rdfsProp, dcProp); // Get all data metric properties - this.metricTriples.addAll(this.kb.genericTripleQuery(null, rdfsProp, dcPropD)); - + this.metricTriples.addAll( + this.kb.genericTripleQuery(null, rdfsProp, dcPropD) + ); + // Get a list of all metrics and datametrics properties in the catalog - this.metricProps = this.kb - .getSubPropertiesOf(this.objPropMap.get("hasMetrics"), false); - this.metricProps.addAll(this.kb - .getSubPropertiesOf(this.dataPropMap.get("hasDataMetrics"), false)); - + this.metricProps = + this.kb.getSubPropertiesOf(this.objPropMap.get("hasMetrics"), false); + this.metricProps.addAll( + this.kb.getSubPropertiesOf( + this.dataPropMap.get("hasDataMetrics"), + false + ) + ); + this.end(); - } - - protected KBObject copyObjectIntoKB(String id, String originid, KBAPI tkb, String includeNS, - String excludeNS, boolean direct) { - try { - this.start_read(); - KBObject obj = this.kb.getIndividual(originid); - // Add component to the temporary KB (add all its classes explicitly) - for(KBTriple triple : this.getTriplesForObject(id, obj, includeNS, excludeNS, direct)) { - tkb.addTriple(triple); - } - return tkb.getIndividual(id); - } - catch(Exception e) { - e.printStackTrace(); - } - finally { - this.end(); - } - return null; - } - - protected KBObject copyObjectClassesIntoKB(String id, String originid, KBAPI tkb, String includeNS, - String excludeNS, boolean direct) { - try { + } + + protected KBObject copyObjectIntoKB( + String id, + String originid, + KBAPI tkb, + String includeNS, + String excludeNS, + boolean direct + ) { + try { this.start_read(); KBObject obj = this.kb.getIndividual(originid); - // Add component to the temporary KB (add all its classes explicitly) - for(KBTriple triple : this.getObjectClassTriples(id, obj, includeNS, excludeNS, direct)) { - tkb.addTriple(triple); - } - KBObject ind = tkb.getResource(id); - return ind; - } - catch(Exception e) { + // Add component to the temporary KB (add all its classes explicitly) + for (KBTriple triple : this.getTriplesForObject( + id, + obj, + includeNS, + excludeNS, + direct + )) { + tkb.addTriple(triple); + } + return tkb.getIndividual(id); + } catch (Exception e) { e.printStackTrace(); + } finally { + this.end(); } - finally { + return null; + } + + protected KBObject copyObjectClassesIntoKB( + String id, + String originid, + KBAPI tkb, + String includeNS, + String excludeNS, + boolean direct + ) { + try { + this.start_read(); + KBObject obj = this.kb.getIndividual(originid); + // Add component to the temporary KB (add all its classes explicitly) + for (KBTriple triple : this.getObjectClassTriples( + id, + obj, + includeNS, + excludeNS, + direct + )) { + tkb.addTriple(triple); + } + KBObject ind = tkb.getResource(id); + return ind; + } catch (Exception e) { + e.printStackTrace(); + } finally { this.end(); } - return null; - } - - protected ArrayList getConcreteComponentsForAbstract(String id) { - if(abscache.containsKey(id)) - return abscache.get(id); - + return null; + } + + protected ArrayList getConcreteComponentsForAbstract(String id) { + if (abscache.containsKey(id)) return abscache.get(id); + KBObject cls = this.kb.getClassOfInstance(this.kb.getIndividual(id)); if (cls == null) { this.abscache.put(id, null); @@ -144,467 +170,553 @@ protected ArrayList getConcreteComponentsForAbstract(String id) { } ArrayList insts = this.kb.getInstancesOfClass(cls, false); ArrayList ids = new ArrayList(); - for(KBObject inst : insts) - ids.add(inst.getID()); + for (KBObject inst : insts) ids.add(inst.getID()); this.abscache.put(id, ids); return ids; - } - - protected ArrayList getTriplesForObject(String id, KBObject obj, - String includeNS, String excludeNS, boolean direct) { - ArrayList triples = new ArrayList(); - if(obj != null) { - triples.add(ontologyFactory.getTriple( - ontologyFactory.getObject(id), - ontologyFactory.getObject(KBUtils.RDF+"type"), - this.kb.getClassOfInstance(obj))); - triples.addAll(getObjectClassTriples(id, obj, includeNS, excludeNS, direct)); - } - return triples; - } - - protected ArrayList getObjectClassTriples(String id, KBObject obj, - String includeNS, String excludeNS, boolean direct) { - // Add component to the temporary KB (add all its classes explicitly) - ArrayList triples = new ArrayList(); - - if(kbcache.containsKey(obj.getID())) { - for(KBTriple triple : kbcache.get(obj.getID())) { - triples.add(ontologyFactory.getTriple( - ontologyFactory.getObject(id), - triple.getPredicate(), - triple.getObject())); + } + + protected ArrayList getTriplesForObject( + String id, + KBObject obj, + String includeNS, + String excludeNS, + boolean direct + ) { + ArrayList triples = new ArrayList(); + if (obj != null) { + triples.add( + ontologyFactory.getTriple( + ontologyFactory.getObject(id), + ontologyFactory.getObject(KBUtils.RDF + "type"), + this.kb.getClassOfInstance(obj) + ) + ); + triples.addAll( + getObjectClassTriples(id, obj, includeNS, excludeNS, direct) + ); + } + return triples; + } + + protected ArrayList getObjectClassTriples( + String id, + KBObject obj, + String includeNS, + String excludeNS, + boolean direct + ) { + // Add component to the temporary KB (add all its classes explicitly) + ArrayList triples = new ArrayList(); + + if (kbcache.containsKey(obj.getID())) { + for (KBTriple triple : kbcache.get(obj.getID())) { + triples.add( + ontologyFactory.getTriple( + ontologyFactory.getObject(id), + triple.getPredicate(), + triple.getObject() + ) + ); } return triples; } - - KBObject tobj = ontologyFactory.getObject(id); - ArrayList objclses = - this.getAllCachedClassesOfInstance(obj.getID(), direct); - for (KBObject objcls : objclses) { - if ((includeNS != null && objcls.getNamespace().equals(includeNS)) - || (excludeNS != null && !objcls.getNamespace().equals(excludeNS))) { - triples.add(ontologyFactory.getTriple( - tobj, ontologyFactory.getObject(KBUtils.RDF + "type"), objcls)); - } - } + + KBObject tobj = ontologyFactory.getObject(id); + ArrayList objclses = + this.getAllCachedClassesOfInstance(obj.getID(), direct); + for (KBObject objcls : objclses) { + if ( + (includeNS != null && objcls.getNamespace().equals(includeNS)) || + (excludeNS != null && !objcls.getNamespace().equals(excludeNS)) + ) { + triples.add( + ontologyFactory.getTriple( + tobj, + ontologyFactory.getObject(KBUtils.RDF + "type"), + objcls + ) + ); + } + } kbcache.put(id, triples); - return triples; - } - - private ArrayList getAllClassesOfInstance(KBAPI tkb, String id) { - ArrayList clses = new ArrayList(); - for(KBTriple t : - tkb.genericTripleQuery(tkb.getResource(id), - tkb.getProperty(KBUtils.RDF+"type"), null)) { - clses.add(this.conceptMap.get(t.getObject().getID())); - } - return clses; - } - - private ArrayList getAllCachedClassesOfInstance(String id, boolean direct) { - String key = id + ":" + direct; - if(classcache.containsKey(key)) - return classcache.get(key); - ArrayList clses = kb.getAllClassesOfInstance(kb.getResource(id), direct); - classcache.put(key, clses); - return clses; - } - - protected boolean checkTypeCompatibility(KBAPI tkb, String varid, String argid) { - this.start_read(); - ArrayList varclses = this.getAllClassesOfInstance(tkb, varid); - ArrayList argclses = this.getAllCachedClassesOfInstance(argid, true); - for (KBObject argcls : argclses) { - if (argcls.getNamespace().equals(this.dcdomns)) { - for (KBObject varcls : varclses) { - if (varcls.getNamespace().equals(this.dcdomns)) { - if (!kb.hasSubClass(argcls, varcls) && !kb.hasSubClass(varcls, argcls)) - return false; - } - } - } - } - this.end(); - return true; - } - - protected boolean checkTypeCompatibility(ArrayList varclassids, String argid) { - this.start_read(); - ArrayList argclses = this.getAllCachedClassesOfInstance(argid, true); - for (KBObject argcls : argclses) { - if (argcls.getNamespace().equals(this.dcdomns)) { - for (String varclassid : varclassids) { - KBObject varcl = this.conceptMap.get(varclassid); - if (varcl.getNamespace().equals(this.dcdomns)) { - if (!kb.hasSubClass(argcls, varcl) && !kb.hasSubClass(varcl, argcls)) - return false; - } - } - } - } - this.end(); - return true; - } - - /** - * Query 2.1
- * Get a list of Specialized Components with their IO Data Requirements - * given a component (maybe abstract) and it's IO Data Requirements - * - * @param details - * A ComponentDetails Object which contains: - *
    - *
  • component, - *
  • maps of component input arguments to template variables, - *
  • maps of component output arguments to template variables, - *
  • template variable descriptions (dods) - list of triples - *
- * @return list of ComponentDetails Objects for each specialized component - */ - - public ArrayList specializeAndFindDataDetails(ComponentPacket details) { - return findDataDetails(details, true, this.useRules); - } - - /** - * Query 2.1b
- * Get inferred IO Data Requirements given a component (maybe abstract) and - * it's given IO Data Requirements - * - * @param details - * A ComponentDetails Object which contains: - *
    - *
  • component, - *
  • maps of component input arguments to template variables, - *
  • maps of component output arguments to template variables, - *
  • template variable descriptions (dods) - list of triples - *
- * @return ComponentDetails Object - */ - public ComponentPacket findDataDetails(ComponentPacket details) { - ArrayList list = findDataDetails(details, false, this.useRules); - if (list.size() > 0) - return list.get(0); - - return null; - } - - /** - * Helper function which does the actual data requirement checking - * - * @param details - * @param specialize - * @param useRules - * @return - */ - public ArrayList findDataDetails(ComponentPacket details, boolean specialize, - boolean useRules) { - ArrayList list = new ArrayList(); - - HashMap omap = this.objPropMap; - HashMap dmap = this.dataPropMap; - - // Extract info from details object - ComponentVariable c = details.getComponent(); - HashMap roleMaps = details.getStringRoleMaps(); - HashMap varMaps = details.getStringVariableMap(); - ArrayList redbox = details.getRequirements(); - - // Get All component bindings - String incompid = c.getID(); - Binding cb = c.getBinding(); - ArrayList cbindings = new ArrayList(); - if(!cb.isSet()) - cbindings.add(cb.getID()); - else { - for(WingsSet s : cb) { - cbindings.add(((Binding)s).getID()); - } - } - + return triples; + } + + private ArrayList getAllClassesOfInstance(KBAPI tkb, String id) { + ArrayList clses = new ArrayList(); + for (KBTriple t : tkb.genericTripleQuery( + tkb.getResource(id), + tkb.getProperty(KBUtils.RDF + "type"), + null + )) { + clses.add(this.conceptMap.get(t.getObject().getID())); + } + return clses; + } + + private ArrayList getAllCachedClassesOfInstance( + String id, + boolean direct + ) { + String key = id + ":" + direct; + if (classcache.containsKey(key)) return classcache.get(key); + ArrayList clses = kb.getAllClassesOfInstance( + kb.getResource(id), + direct + ); + classcache.put(key, clses); + return clses; + } + + protected boolean checkTypeCompatibility( + KBAPI tkb, + String varid, + String argid + ) { + this.start_read(); + ArrayList varclses = this.getAllClassesOfInstance(tkb, varid); + ArrayList argclses = + this.getAllCachedClassesOfInstance(argid, true); + for (KBObject argcls : argclses) { + if (argcls.getNamespace().equals(this.dcdomns)) { + for (KBObject varcls : varclses) { + if (varcls.getNamespace().equals(this.dcdomns)) { + if ( + !kb.hasSubClass(argcls, varcls) && !kb.hasSubClass(varcls, argcls) + ) return false; + } + } + } + } + this.end(); + return true; + } + + protected boolean checkTypeCompatibility( + ArrayList varclassids, + String argid + ) { + this.start_read(); + ArrayList argclses = + this.getAllCachedClassesOfInstance(argid, true); + for (KBObject argcls : argclses) { + if (argcls.getNamespace().equals(this.dcdomns)) { + for (String varclassid : varclassids) { + KBObject varcl = this.conceptMap.get(varclassid); + if (varcl.getNamespace().equals(this.dcdomns)) { + if ( + !kb.hasSubClass(argcls, varcl) && !kb.hasSubClass(varcl, argcls) + ) return false; + } + } + } + } + this.end(); + return true; + } + + /** + * Query 2.1
+ * Get a list of Specialized Components with their IO Data Requirements + * given a component (maybe abstract) and it's IO Data Requirements + * + * @param details + * A ComponentDetails Object which contains: + *
    + *
  • component, + *
  • maps of component input arguments to template variables, + *
  • maps of component output arguments to template variables, + *
  • template variable descriptions (dods) - list of triples + *
+ * @return list of ComponentDetails Objects for each specialized component + */ + + public ArrayList specializeAndFindDataDetails( + ComponentPacket details + ) { + return findDataDetails(details, true, this.useRules); + } + + /** + * Query 2.1b
+ * Get inferred IO Data Requirements given a component (maybe abstract) and + * it's given IO Data Requirements + * + * @param details + * A ComponentDetails Object which contains: + *
    + *
  • component, + *
  • maps of component input arguments to template variables, + *
  • maps of component output arguments to template variables, + *
  • template variable descriptions (dods) - list of triples + *
+ * @return ComponentDetails Object + */ + public ComponentPacket findDataDetails(ComponentPacket details) { + ArrayList list = findDataDetails( + details, + false, + this.useRules + ); + if (list.size() > 0) return list.get(0); + + return null; + } + + /** + * Helper function which does the actual data requirement checking + * + * @param details + * @param specialize + * @param useRules + * @return + */ + public ArrayList findDataDetails( + ComponentPacket details, + boolean specialize, + boolean useRules + ) { + ArrayList list = new ArrayList(); + + HashMap omap = this.objPropMap; + HashMap dmap = this.dataPropMap; + + // Extract info from details object + ComponentVariable c = details.getComponent(); + HashMap roleMaps = details.getStringRoleMaps(); + HashMap varMaps = details.getStringVariableMap(); + ArrayList redbox = details.getRequirements(); + + // Get All component bindings + String incompid = c.getID(); + Binding cb = c.getBinding(); + ArrayList cbindings = new ArrayList(); + if (!cb.isSet()) cbindings.add(cb.getID()); else { + for (WingsSet s : cb) { + cbindings.add(((Binding) s).getID()); + } + } + try { - // Get List of all concrete components - ArrayList ccomps = new ArrayList(); - for(String cbid : cbindings) { - Component comp = this.getCachedComponent(cbid); - if (comp == null) { - logger.debug(cbid + " is not a valid component"); - details.addExplanations(cbid + " is not a valid component"); - details.setInvalidFlag(true); - list.add(details); - return list; - } - boolean isConcrete = (comp.getType() == Component.CONCRETE); - if (!specialize) { - // If no specialization required, add component as is - ccomps.add(comp); - } else if (isConcrete) { - // If specialization required, but component is already concrete, - // add as is - ccomps.add(comp); - } else { - /* If the component is abstract, then get all it's concrete - * components. Example of how components are structured in ontology: - * absClass - * - [abs (isConcrete: false)] - * - conc1Class - * - [conc1 (isConcrete: true)] - * - conc2Class - * - [conc2 (isConcrete: true)] - * Note: Only 1 Component Instance per Class - */ - ArrayList concreteids = new ArrayList(); - try { + // Get List of all concrete components + ArrayList ccomps = new ArrayList(); + for (String cbid : cbindings) { + Component comp = this.getCachedComponent(cbid); + if (comp == null) { + logger.debug(cbid + " is not a valid component"); + details.addExplanations(cbid + " is not a valid component"); + details.setInvalidFlag(true); + list.add(details); + return list; + } + boolean isConcrete = (comp.getType() == Component.CONCRETE); + if (!specialize) { + // If no specialization required, add component as is + ccomps.add(comp); + } else if (isConcrete) { + // If specialization required, but component is already concrete, + // add as is + ccomps.add(comp); + } else { + /* If the component is abstract, then get all it's concrete + * components. Example of how components are structured in ontology: + * absClass + * - [abs (isConcrete: false)] + * - conc1Class + * - [conc1 (isConcrete: true)] + * - conc2Class + * - [conc2 (isConcrete: true)] + * Note: Only 1 Component Instance per Class + */ + ArrayList concreteids = new ArrayList(); + try { this.start_read(); - concreteids = this.getConcreteComponentsForAbstract(comp.getID()); - this.end(); - } - catch(Exception e) { - e.printStackTrace(); - this.end(); - } - - for (String concreteid : concreteids) { - Component instcomp = this.getCachedComponent(concreteid); - if (instcomp != null && instcomp.getType() == Component.CONCRETE) { - ccomps.add(instcomp); - } - } - } - } - - logger.debug("Available components to check validity: " + ccomps); - - // For All concrete components : - // - Get mapping of specialized arguments to variables - // - Transfer "relevant" output variable properties to input variables - // - Pass back the specialized component + specialized mappings + - // modified red-box - // - Handle *NEW* Arguments as well -> Create *NEW* DataVariables - - // Get Metrics property hierarchy triples for adding into the temporary - // kb - - for (Component ccomp : ccomps) { - HashMap sRoleMap = new HashMap(); - ArrayList varids = new ArrayList(); - - // Create a new temporary kb - KBAPI tkb = this.ontologyFactory.getKB(OntSpec.PLAIN); - - // Add the redbox (i.e. datavariable constraints) to the temporary - // kb, along with domain knowledge about the data catalog - tkb.addTriples(redbox); - tkb.addTriples(domainKnowledge); - - KBObject tcomp = this.copyObjectIntoKB(incompid, ccomp.getID(), tkb, this.pcdomns, null, false); - boolean typesOk = true; - - // For all argument roles of the specialized component : - ArrayList allArgs = new ArrayList(ccomp.getInputs()); - allArgs.addAll(ccomp.getOutputs()); - - HashSet explanations = new HashSet(); - ComponentPacket cmr; - ComponentVariable concreteComponent = new ComponentVariable(incompid); - Binding componentBinding = new Binding(ccomp.getID()); - componentBinding.setVersion(ccomp.getVersion()); - concreteComponent.setBinding(componentBinding); - if (specialize) - concreteComponent.setConcrete(true); - else - concreteComponent.setConcrete(c.isConcrete()); - - ArrayList inputRoles = new ArrayList(); - - for (ComponentRole arg : allArgs) { - // Get the argument ID for the specialized argument - String argid = arg.getRoleName(); - - Variable var = roleMaps.get(argid); - String varid = null; - String roleid = null; - if (var == null) { - // Create a new Variable for role if none exists currently - // varid = arg.getID()+"_"+ccomp.getName()+"_Variable"; - varid = ccomp.getNamespace() + argid; - roleid = ccomp.getID() + "_" + argid + "_role"; - - short type = 0; - if (arg.isParam()) - type = VariableType.PARAM; - else - type = VariableType.DATA; - var = new Variable(varid, type); - } else { - varid = var.getID(); - roleid = varMaps.get(var.getID()).getID(); - // Make sure that the variable has a type that is either - // subsumed by the argument type, or that the argument type - // is subsumed by the variable type - if (!checkTypeCompatibility(tkb, varid, arg.getID())) { - logger.debug(arg.getID() + " is not type compatible with variable: " - + varid); - explanations.add("INFO "+ccomp + " is not selectable because " + arg.getID() - + " is not type compatible with variable: " + varid); - typesOk = false; - break; - } - } - - // Copy over the argument's classes to the variable - KBObject varobj = this.copyObjectClassesIntoKB(varid, arg.getID(), tkb, this.dcdomns, null, false); - if(varobj == null) - continue; - - // create hasArgumentID property for the variable - tkb.addTriple(varobj, dmap.get("hasArgumentID"), tkb.createLiteral(argid)); - - Role r = new Role(roleid); - r.setRoleId(argid); - r.setDimensionality(arg.getDimensionality()); - - // Set variable data binding - if (var.isDataVariable() && var.getBinding() != null - && var.getBinding().getName() != null) { - tkb.addTriple(varobj, dmap.get("hasBindingID"), - tkb.createLiteral(var.getBinding().getName())); - } else { - tkb.addTriple(varobj, dmap.get("hasBindingID"), - tkb.createLiteral("")); - } - - // Set variable parameter binding (default if none set) - if(var.isParameterVariable()) { - KBObject arg_value = null; - - List paramBindings = new ArrayList(); - if(var.getBinding() != null) { - if(var.getBinding().isSet()) { - for(WingsSet bset : var.getBinding()) { - paramBindings.add((ValueBinding) bset); - } - } - else { - paramBindings.add((ValueBinding) var.getBinding()); - } - for(ValueBinding parambinding: paramBindings) { - if (parambinding != null && parambinding.getValue() != null) { - arg_value = tkb.createXSDLiteral(parambinding.getValueAsString(), - parambinding.getDatatype()); + concreteids = this.getConcreteComponentsForAbstract(comp.getID()); + this.end(); + } catch (Exception e) { + e.printStackTrace(); + this.end(); + } + + for (String concreteid : concreteids) { + Component instcomp = this.getCachedComponent(concreteid); + if (instcomp != null && instcomp.getType() == Component.CONCRETE) { + ccomps.add(instcomp); + } + } + } + } + + logger.debug("Available components to check validity: " + ccomps); + + // For All concrete components : + // - Get mapping of specialized arguments to variables + // - Transfer "relevant" output variable properties to input variables + // - Pass back the specialized component + specialized mappings + + // modified red-box + // - Handle *NEW* Arguments as well -> Create *NEW* DataVariables + + // Get Metrics property hierarchy triples for adding into the temporary + // kb + + for (Component ccomp : ccomps) { + HashMap sRoleMap = new HashMap(); + ArrayList varids = new ArrayList(); + + // Create a new temporary kb + KBAPI tkb = this.ontologyFactory.getKB(OntSpec.PLAIN); + + // Add the redbox (i.e. datavariable constraints) to the temporary + // kb, along with domain knowledge about the data catalog + tkb.addTriples(redbox); + tkb.addTriples(domainKnowledge); + + KBObject tcomp = + this.copyObjectIntoKB( + incompid, + ccomp.getID(), + tkb, + this.pcdomns, + null, + false + ); + boolean typesOk = true; + + // For all argument roles of the specialized component : + ArrayList allArgs = new ArrayList( + ccomp.getInputs() + ); + allArgs.addAll(ccomp.getOutputs()); + + HashSet explanations = new HashSet(); + ComponentPacket cmr; + ComponentVariable concreteComponent = new ComponentVariable(incompid); + Binding componentBinding = new Binding(ccomp.getID()); + componentBinding.setVersion(ccomp.getVersion()); + concreteComponent.setBinding(componentBinding); + if (specialize) concreteComponent.setConcrete( + true + ); else concreteComponent.setConcrete(c.isConcrete()); + + ArrayList inputRoles = new ArrayList(); + + for (ComponentRole arg : allArgs) { + // Get the argument ID for the specialized argument + String argid = arg.getRoleName(); + + Variable var = roleMaps.get(argid); + String varid = null; + String roleid = null; + if (var == null) { + // Create a new Variable for role if none exists currently + // varid = arg.getID()+"_"+ccomp.getName()+"_Variable"; + varid = ccomp.getNamespace() + argid; + roleid = ccomp.getID() + "_" + argid + "_role"; + + short type = 0; + if (arg.isParam()) type = VariableType.PARAM; else type = + VariableType.DATA; + var = new Variable(varid, type); + } else { + varid = var.getID(); + roleid = varMaps.get(var.getID()).getID(); + // Make sure that the variable has a type that is either + // subsumed by the argument type, or that the argument type + // is subsumed by the variable type + if (!checkTypeCompatibility(tkb, varid, arg.getID())) { + logger.debug( + arg.getID() + " is not type compatible with variable: " + varid + ); + explanations.add( + "INFO " + + ccomp + + " is not selectable because " + + arg.getID() + + " is not type compatible with variable: " + + varid + ); + typesOk = false; + break; + } + } + + // Copy over the argument's classes to the variable + KBObject varobj = + this.copyObjectClassesIntoKB( + varid, + arg.getID(), + tkb, + this.dcdomns, + null, + false + ); + if (varobj == null) continue; + + // create hasArgumentID property for the variable + tkb.addTriple( + varobj, + dmap.get("hasArgumentID"), + tkb.createLiteral(argid) + ); + + Role r = new Role(roleid); + r.setRoleId(argid); + r.setDimensionality(arg.getDimensionality()); + + // Set variable data binding + if ( + var.isDataVariable() && + var.getBinding() != null && + var.getBinding().getName() != null + ) { + tkb.addTriple( + varobj, + dmap.get("hasBindingID"), + tkb.createLiteral(var.getBinding().getName()) + ); + } else { + tkb.addTriple( + varobj, + dmap.get("hasBindingID"), + tkb.createLiteral("") + ); + } + + // Set variable parameter binding (default if none set) + if (var.isParameterVariable()) { + KBObject arg_value = null; + + List paramBindings = new ArrayList(); + if (var.getBinding() != null) { + if (var.getBinding().isSet()) { + for (WingsSet bset : var.getBinding()) { + paramBindings.add((ValueBinding) bset); } - else if(arg.getParamDefaultalue() != null) { - arg_value = tkb.createLiteral(arg.getParamDefaultalue()); - } - if (arg_value != null) { - tkb.addPropertyValue(varobj, dmap.get("hasValue"), arg_value); - } - } - } - } - - // assign this variable as an input or output to the component - if (ccomp.getInputs().contains(arg)) { - inputRoles.add(r.getRoleId()); - tkb.addTriple(tcomp, omap.get("hasInput"), varobj); - } else { - tkb.addTriple(tcomp, omap.get("hasOutput"), varobj); - } - sRoleMap.put(r, var); - varids.add(var.getID()); - } - - // Empty triple list returned if errors encountered below - ArrayList empty = new ArrayList(); - - // Return if there was some problem with types - if (!typesOk) { - logger.debug(ccomp + " is not selectable "); - explanations.add("INFO " + ccomp + " is not selectable "); - cmr = new ComponentPacket(concreteComponent, sRoleMap, empty); - cmr.setInputRoles(inputRoles); - cmr.addExplanations(explanations); - cmr.setInvalidFlag(true); - list.add(cmr); - continue; - } - - // ** Run Rules ** - if (useRules && ccomp.hasRules()) { - // Redirect output to a byte stream - ByteArrayOutputStream bost = new ByteArrayOutputStream(); - PrintStream oldout = System.out; - System.setOut(new PrintStream(bost, true)); - - // Run propagation rules on the temporary kb - tkb.setRulePrefixes(this.rulePrefixes); - tkb.applyRules(this.getCachedComponentRules(ccomp)); - //tkb.applyRulesFromString(allrules); - - // Get printouts from Rules and store as Explanations - if (!bost.toString().equals("")) { - for (String exp : bost.toString().split("\\n")) { - explanations.add(exp); - } - } - // Set output back to original System.out - System.setOut(oldout); - } - - // Checking for invalidity - KBObject invalidProp = tkb.getProperty(this.pcns + "isInvalid"); - KBObject isInvalid = tkb.getPropertyValue(tcomp, invalidProp); - if (isInvalid != null && (Boolean) isInvalid.getValue()) { - logger.debug(ccomp + " is not selectable "); - explanations.add("INFO " + ccomp + " is not selectable "); - cmr = new ComponentPacket(concreteComponent, sRoleMap, empty); - cmr.setInputRoles(inputRoles); - cmr.addExplanations(explanations); - cmr.setInvalidFlag(true); - list.add(cmr); - continue; - } - - // Set parameter values (if any) - for (Variable var : roleMaps.values()) { - if (var.isParameterVariable() && var.getBinding() == null) { - KBObject varobj = tkb.getResource(var.getID()); - ArrayList vals = tkb.getPropertyValues(varobj, dmap.get("hasValue")); - if (vals != null && vals.size() > 0) { - Binding b = new Binding(); - for(KBObject val : vals) { - b.add(new ValueBinding(val.getValue(), val.getDataType())); - tkb.addTriple(varobj, - tkb.getResource(this.wflowns + "hasParameterValue"), val); - } + } else { + paramBindings.add((ValueBinding) var.getBinding()); + } + for (ValueBinding parambinding : paramBindings) { + if (parambinding != null && parambinding.getValue() != null) { + arg_value = + tkb.createXSDLiteral( + parambinding.getValueAsString(), + parambinding.getDatatype() + ); + } else if (arg.getParamDefaultalue() != null) { + arg_value = tkb.createLiteral(arg.getParamDefaultalue()); + } + if (arg_value != null) { + tkb.addPropertyValue(varobj, dmap.get("hasValue"), arg_value); + } + } + } + } + + // assign this variable as an input or output to the component + if (ccomp.getInputs().contains(arg)) { + inputRoles.add(r.getRoleId()); + tkb.addTriple(tcomp, omap.get("hasInput"), varobj); + } else { + tkb.addTriple(tcomp, omap.get("hasOutput"), varobj); + } + sRoleMap.put(r, var); + varids.add(var.getID()); + } + + // Empty triple list returned if errors encountered below + ArrayList empty = new ArrayList(); + + // Return if there was some problem with types + if (!typesOk) { + logger.debug(ccomp + " is not selectable "); + explanations.add("INFO " + ccomp + " is not selectable "); + cmr = new ComponentPacket(concreteComponent, sRoleMap, empty); + cmr.setInputRoles(inputRoles); + cmr.addExplanations(explanations); + cmr.setInvalidFlag(true); + list.add(cmr); + continue; + } + + // ** Run Rules ** + if (useRules && ccomp.hasRules()) { + // Redirect output to a byte stream + ByteArrayOutputStream bost = new ByteArrayOutputStream(); + PrintStream oldout = System.out; + System.setOut(new PrintStream(bost, true)); + + // Run propagation rules on the temporary kb + tkb.setRulePrefixes(this.rulePrefixes); + tkb.applyRules(this.getCachedComponentRules(ccomp)); + //tkb.applyRulesFromString(allrules); + + // Get printouts from Rules and store as Explanations + if (!bost.toString().equals("")) { + for (String exp : bost.toString().split("\\n")) { + explanations.add(exp); + } + } + // Set output back to original System.out + System.setOut(oldout); + } + + // Checking for invalidity + KBObject invalidProp = tkb.getProperty(this.pcns + "isInvalid"); + KBObject isInvalid = tkb.getPropertyValue(tcomp, invalidProp); + if (isInvalid != null && (Boolean) isInvalid.getValue()) { + logger.debug(ccomp + " is not selectable "); + explanations.add("INFO " + ccomp + " is not selectable "); + cmr = new ComponentPacket(concreteComponent, sRoleMap, empty); + cmr.setInputRoles(inputRoles); + cmr.addExplanations(explanations); + cmr.setInvalidFlag(true); + list.add(cmr); + continue; + } + + // Set parameter values (if any) + for (Variable var : roleMaps.values()) { + if (var.isParameterVariable() && var.getBinding() == null) { + KBObject varobj = tkb.getResource(var.getID()); + ArrayList vals = tkb.getPropertyValues( + varobj, + dmap.get("hasValue") + ); + if (vals != null && vals.size() > 0) { + Binding b = new Binding(); + for (KBObject val : vals) { + b.add(new ValueBinding(val.getValue(), val.getDataType())); + tkb.addTriple( + varobj, + tkb.getResource(this.wflowns + "hasParameterValue"), + val + ); + } var.setBinding(b.size() > 1 ? b : (ValueBinding) b.get(0)); - } - } - } - - // Create a constraint engine and get Relevant Constraints here - ConstraintEngine cons = new ConstraintEngineKB(tkb, ""); - cons.addWhitelistedNamespace(this.dcdomns); - cons.addWhitelistedNamespace(this.dcns); - cons.addWhitelistedNamespace(this.wflowns); - ArrayList blacklistedIds = new ArrayList(); - blacklistedIds.add(dmap.get("hasArgumentID").getID()); - blacklistedIds.add(dmap.get("hasBindingID").getID()); - blacklistedIds.add(this.dcns + "hasMetrics"); - blacklistedIds.add(this.dcns + "hasDataMetrics"); - blacklistedIds.add(this.pcns + "hasValue"); - - for (String id : blacklistedIds) - cons.addBlacklistedId(id); - ArrayList constraints = cons.getConstraints(varids); - for (String id : blacklistedIds) - cons.removeBlacklistedId(id); - - cmr = new ComponentPacket(concreteComponent, sRoleMap, constraints); - cmr.setInputRoles(inputRoles); - cmr.addExplanations(explanations); - + } + } + } + + // Create a constraint engine and get Relevant Constraints here + ConstraintEngine cons = new ConstraintEngineKB(tkb, ""); + cons.addWhitelistedNamespace(this.dcdomns); + cons.addWhitelistedNamespace(this.dcns); + cons.addWhitelistedNamespace(this.wflowns); + ArrayList blacklistedIds = new ArrayList(); + blacklistedIds.add(dmap.get("hasArgumentID").getID()); + blacklistedIds.add(dmap.get("hasBindingID").getID()); + blacklistedIds.add(this.dcns + "hasMetrics"); + blacklistedIds.add(this.dcns + "hasDataMetrics"); + blacklistedIds.add(this.pcns + "hasValue"); + + for (String id : blacklistedIds) cons.addBlacklistedId(id); + ArrayList constraints = cons.getConstraints(varids); + for (String id : blacklistedIds) cons.removeBlacklistedId(id); + + cmr = new ComponentPacket(concreteComponent, sRoleMap, constraints); + cmr.setInputRoles(inputRoles); + cmr.addExplanations(explanations); + // Checking for NoOperation KBObject noOpProp = tkb.getProperty(this.pcns + "isNoOperation"); KBObject isNoOperation = tkb.getPropertyValue(tcomp, noOpProp); @@ -613,123 +725,131 @@ else if(arg.getParamDefaultalue() != null) { explanations.add("INFO " + ccomp + " is going to be skipped"); cmr.setNoOperationFlag(true); } - - list.add(cmr); - } - return list; - } - catch(Exception e) { + + list.add(cmr); + } + return list; + } catch (Exception e) { e.printStackTrace(); } return null; - } - - private Component getCachedComponent(String compid) { + } + + private Component getCachedComponent(String compid) { // Get Component Component comp = null; - if(ccache.containsKey(compid)) { + if (ccache.containsKey(compid)) { comp = ccache.get(compid); - } - else { + } else { comp = this.getComponent(compid, true); ccache.put(compid, comp); } return comp; - } - - private KBRuleList getCachedComponentRules(Component comp) { - if(rulescache.containsKey(comp.getID())) - return rulescache.get(comp.getID()); - String rulestr = ""; - for(String str : comp.getRules()) - rulestr += str + "\n"; - for(String str : comp.getInheritedRules()) - rulestr += str + "\n"; - KBRuleList rules = this.ontologyFactory.parseRules(rulestr); - rulescache.put(comp.getID(), rules); - return rules; - } - - /** - * Query 4.2
- * This function is supposed to SET the DataSet Metrics, or Parameter - * Values for the Variables that are passed in via the input/output maps as - * part of details.
- * Variables will already be bound to dataObjects, so the function will have - * to do something like the following : - * - * 
-	 * If Variable.isParameterVariable() Variable.setParameterValue(value)
-	 * If Variable.isDataVariable() Variable.getDataObjectBinding().setDataMetrics(xml)
-	 *
- * - * @param details - * A ComponentDetails Object which contains: - *
    - *
  • component, - *
  • maps of component input arguments to template variables, - *
  • maps of component output arguments to template variables, - *
  • template variable descriptions (dods) - list of triples - *
- * @return List of extra template variable descriptions (will mostly be - * empty in Q4.2 though) - */ - public ArrayList findOutputDataPredictedDescriptions(ComponentPacket details) { - ArrayList list = new ArrayList(); - - HashMap omap = this.objPropMap; - HashMap dmap = this.dataPropMap; - - // If the component has no rules, then simplify !! - - // Extract info from details object - ComponentVariable c = details.getComponent(); - HashMap sRoleMap = details.getStringRoleMaps(); - HashMap noParamBindings = new HashMap(); - - ArrayList redbox = details.getRequirements(); - - Component comp = this.getCachedComponent(c.getBinding().getID()); - if (comp == null) { - logger.debug(c.getBinding().getID() + " is not a valid component"); - details.addExplanations(c.getBinding().getID() + " is not a valid component"); - details.setInvalidFlag(true); - list.add(details); - return list; - } - - c.setRequirements(comp.getComponentRequirement()); - - boolean typesOk = true; - + } + + private KBRuleList getCachedComponentRules(Component comp) { + if (rulescache.containsKey(comp.getID())) return rulescache.get( + comp.getID() + ); + String rulestr = ""; + for (String str : comp.getRules()) rulestr += str + "\n"; + for (String str : comp.getInheritedRules()) rulestr += str + "\n"; + KBRuleList rules = this.ontologyFactory.parseRules(rulestr); + rulescache.put(comp.getID(), rules); + return rules; + } + + /** + * Query 4.2
+ * This function is supposed to SET the DataSet Metrics, or Parameter + * Values for the Variables that are passed in via the input/output maps as + * part of details.
+ * Variables will already be bound to dataObjects, so the function will have + * to do something like the following : + * + * 
+   * If Variable.isParameterVariable() Variable.setParameterValue(value)
+   * If Variable.isDataVariable() Variable.getDataObjectBinding().setDataMetrics(xml)
+   *
+ * + * @param details + * A ComponentDetails Object which contains: + *
    + *
  • component, + *
  • maps of component input arguments to template variables, + *
  • maps of component output arguments to template variables, + *
  • template variable descriptions (dods) - list of triples + *
+ * @return List of extra template variable descriptions (will mostly be + * empty in Q4.2 though) + */ + public ArrayList findOutputDataPredictedDescriptions( + ComponentPacket details + ) { + ArrayList list = new ArrayList(); + + HashMap omap = this.objPropMap; + HashMap dmap = this.dataPropMap; + + // If the component has no rules, then simplify !! + + // Extract info from details object + ComponentVariable c = details.getComponent(); + HashMap sRoleMap = details.getStringRoleMaps(); + HashMap noParamBindings = new HashMap(); + + ArrayList redbox = details.getRequirements(); + + Component comp = this.getCachedComponent(c.getBinding().getID()); + if (comp == null) { + logger.debug(c.getBinding().getID() + " is not a valid component"); + details.addExplanations( + c.getBinding().getID() + " is not a valid component" + ); + details.setInvalidFlag(true); + list.add(details); + return list; + } + + c.setRequirements(comp.getComponentRequirement()); + + boolean typesOk = true; + // Set default parameter values (if not already set) - // - Also recheck type compatibility + // - Also recheck type compatibility ArrayList inputRoles = new ArrayList(); - for(ComponentRole role : comp.getInputs()) { + for (ComponentRole role : comp.getInputs()) { inputRoles.add(role.getRoleName()); Variable v = sRoleMap.get(role.getRoleName()); - if(role.isParam()) { - if(v.getBinding() == null) { + if (role.isParam()) { + if (v.getBinding() == null) { v.setBinding(new ValueBinding(role.getParamDefaultalue())); noParamBindings.put(v.getID(), true); - } - else if(v.getBinding().getValue() == null) { + } else if (v.getBinding().getValue() == null) { v.getBinding().setValue(role.getParamDefaultalue()); noParamBindings.put(v.getID(), true); } - } - else { - ArrayList varclassids = new ArrayList(); - ArrayList vartypes = - v.getBinding().getMetrics().getMetrics().get(KBUtils.RDF + "type"); - if(vartypes != null) { - for(Metric m : vartypes) { + } else { + ArrayList varclassids = new ArrayList(); + ArrayList vartypes = v + .getBinding() + .getMetrics() + .getMetrics() + .get(KBUtils.RDF + "type"); + if (vartypes != null) { + for (Metric m : vartypes) { varclassids.add(m.getValueAsString()); } // Check type compatibility of roles if (!checkTypeCompatibility(varclassids, role.getID())) { - details.addExplanations("INFO "+comp + " is not selectable because " + role.getID() - + " is not type compatible with variable binding: " + v.getBinding()); + details.addExplanations( + "INFO " + + comp + + " is not selectable because " + + role.getID() + + " is not type compatible with variable binding: " + + v.getBinding() + ); typesOk = false; break; } @@ -738,36 +858,48 @@ else if(v.getBinding().getValue() == null) { } details.setInputRoles(inputRoles); - if(!typesOk) { + if (!typesOk) { details.setInvalidFlag(true); list.add(details); return list; } - - if(!comp.hasRules()) { - // No rules. Just set default parameter values (if not already set) - - // Also set output type metrics - for(ComponentRole role : comp.getOutputs()) { - Variable v = sRoleMap.get(role.getRoleName()); - v.getBinding().getMetrics().addMetric( - KBUtils.RDF + "type", - new Metric(Metric.URI, role.getType())); - } - list.add(details); - return list; - } - - // Create a new temporary KB store to run rules on - KBAPI tkb = this.ontologyFactory.getKB(OntSpec.PLAIN); - - try { - - KBObject tcomp = this.copyObjectIntoKB(comp.getID(), comp.getID(), tkb, this.pcdomns, null, false); - - // Keep a map of variable object to variable name - HashMap variableNameMap = new HashMap(); - + + if (!comp.hasRules()) { + // No rules. Just set default parameter values (if not already set) + + // Also set output type metrics + for (ComponentRole role : comp.getOutputs()) { + Variable v = sRoleMap.get(role.getRoleName()); + v + .getBinding() + .getMetrics() + .addMetric( + KBUtils.RDF + "type", + new Metric(Metric.URI, role.getType()) + ); + } + list.add(details); + return list; + } + + // Create a new temporary KB store to run rules on + KBAPI tkb = this.ontologyFactory.getKB(OntSpec.PLAIN); + + try { + KBObject tcomp = + this.copyObjectIntoKB( + comp.getID(), + comp.getID(), + tkb, + this.pcdomns, + null, + false + ); + + // Keep a map of variable object to variable name + HashMap variableNameMap = + new HashMap(); + for (String rolestr : sRoleMap.keySet()) { Variable var = sRoleMap.get(rolestr); // Map template variable to a temporary variable for running rules @@ -787,16 +919,15 @@ else if(v.getBinding().getValue() == null) { for (KBTriple t : redbox) { KBObject subj = varIDObjMap.get(t.getSubject().getID()); KBObject obj = varIDObjMap.get(t.getObject().getID()); - if (subj == null) - subj = t.getSubject(); - if (obj == null) - obj = t.getObject(); + if (subj == null) subj = t.getSubject(); + if (obj == null) obj = t.getObject(); tkb.addTriple(subj, t.getPredicate(), obj); } - + // Get a mapping of ArgID's to arg for the Component // Also note which roles are inputs - HashMap argMaps = new HashMap(); + HashMap argMaps = + new HashMap(); HashMap sInputRoles = new HashMap(); for (ComponentRole role : comp.getInputs()) { argMaps.put(role.getRoleName(), role); @@ -805,148 +936,189 @@ else if(v.getBinding().getValue() == null) { for (ComponentRole role : comp.getOutputs()) { argMaps.put(role.getRoleName(), role); } - - // Convert metrics to Property assertions in the Temporary KB - for (String rolestr : sRoleMap.keySet()) { - Variable var = sRoleMap.get(rolestr); - ComponentRole arg = argMaps.get(rolestr); - if(arg == null) { - details.addExplanations("ERROR Component catalog cannot recognize role id "+rolestr); - continue; - } - String variableName = variableNameMap.get(var); - - // Get a KBObject for the temporary variable - KBObject varobj = tkb.getResource(variableName); - - if (var.isDataVariable()) { - // If the variable is a data variable (& is bound) - if (var.getBinding() != null) { - // Convert Metrics to PC properties in order to run rules - Metrics metrics = var.getBinding().getMetrics(); - HashMap> propValMap = metrics.getMetrics(); - for (String propid : propValMap.keySet()) { - for(Metric tmp : propValMap.get(propid)) { - Object val = tmp.getValue(); - String valstring = tmp.getValueAsString(); - int type = tmp.getType(); - String dtype = tmp.getDatatype(); - KBObject metricProp = this.kb.getProperty(propid); - if (metricProp != null) { - //System.out.println(var.getName()+": " + propid + " = " +valstring); - if (type == Metric.URI) { - // Object Property - KBObject valobj = this.kb.getResource(valstring); - if (valobj == null) { - // TODO: Log and explain (make a utility - // function) - details.addExplanations("ERROR Cannot Recognize Metrics Value " + valstring); - continue; - } - // Copy over the object class into kb as well - // (except where the object itself is a class) - if (!metricProp.getID().equals(KBUtils.RDF + "type")) { - valobj = this.copyObjectIntoKB(valobj.getID(), valobj.getID(), tkb, - null, null, true); - // Remove any existing values first - for(KBTriple t : tkb.genericTripleQuery(varobj, metricProp, null)) - tkb.removeTriple(t); - } - // Add a Triple for the metric property value - tkb.addTriple(varobj, metricProp, valobj); - } else if (type == Metric.LITERAL && val != null) { - // Literal value - KBObject tobj = dtype != null ? tkb.createXSDLiteral(valstring, dtype) : - tkb.createLiteral(val); - if (tobj != null) { - // Remove any existing values first - for(KBTriple t : tkb.genericTripleQuery(varobj, metricProp, null)) - tkb.removeTriple(t); - // Add a Triple for the metric propertyvalue - tkb.addTriple(varobj, metricProp, tobj); - } else { - details.addExplanations("ERROR Cannot Convert Metrics Value " + valstring); - continue; - } - } - } else { - // TODO: Log and explain (make a utility function) - details.addExplanations("ERROR No Such Metrics Property Known to Component Catalog : " - + propid); - continue; - } - } - } - - // Create other standard PC properties on variable - // - hasDimensionSizes - // - hasBindingID - if (var.getBinding().isSet()) { - String dimensionSizes = ""; - ArrayList vbs = new ArrayList(); - vbs.add(var.getBinding()); - while (!vbs.isEmpty()) { - Binding vb = vbs.remove(0); - if (vb.isSet()) { - for (WingsSet vs : vb) { - vbs.add((Binding) vs); - } - if (!dimensionSizes.equals("")) - dimensionSizes += ","; - dimensionSizes += vb.getSize(); - } - } - tkb.setPropertyValue(varobj, dmap.get("hasDimensionSizes"), - tkb.createLiteral(dimensionSizes)); - } - - if (var.getBinding().getID() != null) - tkb.addTriple(varobj, dmap.get("hasBindingID"), - tkb.createLiteral(var.getBinding().getName())); - else - tkb.addTriple(varobj, dmap.get("hasBindingID"), - tkb.createLiteral("")); - - // end if (var.getDataBinding() != null) - } - // end if (var.isDataVariable()) - } else if (var.isParameterVariable()) { - // If the Variable/Argument is a Parameter - ValueBinding parambinding = (ValueBinding) var.getBinding(); - if (parambinding != null && parambinding.getValue() != null) { - // If the template has any value specified, use that instead - //arg_value = tkb.createLiteral(var.getBinding().getValue()); - KBObject arg_value = tkb.createXSDLiteral(parambinding.getValueAsString(), - parambinding.getDatatype()); - tkb.setPropertyValue(varobj, dmap.get("hasValue"), arg_value); - } - if(dmap.containsKey("hasBindingID")) - // Set the hasBindingID term - tkb.addTriple(varobj, dmap.get("hasBindingID"), - tkb.createLiteral("Param" + arg.getName())); - } - - // Copy argument classes from Catalog as classes for the temporary - // variable in the temporary kb store - this.copyObjectClassesIntoKB(varobj.getID(), arg.getID(), tkb, null, null, true); - - // Set the temporary variable's argumentID so rules can get/set - // triples based on the argument - tkb.addTriple(varobj, dmap.get("hasArgumentID"), tkb.createLiteral(rolestr)); - - // Set hasInput or hasOutput for the temporary Variable - if (sInputRoles.containsKey(rolestr)) { - tkb.addTriple(tcomp, omap.get("hasInput"), varobj); - } else { - tkb.addTriple(tcomp, omap.get("hasOutput"), varobj); - } - // end of for (String rolestr : sRoleMap.keySet()) - } - - // Add all metrics and datametrics properties to temporary store + + // Convert metrics to Property assertions in the Temporary KB + for (String rolestr : sRoleMap.keySet()) { + Variable var = sRoleMap.get(rolestr); + ComponentRole arg = argMaps.get(rolestr); + if (arg == null) { + details.addExplanations( + "ERROR Component catalog cannot recognize role id " + rolestr + ); + continue; + } + String variableName = variableNameMap.get(var); + + // Get a KBObject for the temporary variable + KBObject varobj = tkb.getResource(variableName); + + if (var.isDataVariable()) { + // If the variable is a data variable (& is bound) + if (var.getBinding() != null) { + // Convert Metrics to PC properties in order to run rules + Metrics metrics = var.getBinding().getMetrics(); + HashMap> propValMap = + metrics.getMetrics(); + for (String propid : propValMap.keySet()) { + for (Metric tmp : propValMap.get(propid)) { + Object val = tmp.getValue(); + String valstring = tmp.getValueAsString(); + int type = tmp.getType(); + String dtype = tmp.getDatatype(); + KBObject metricProp = this.kb.getProperty(propid); + if (metricProp != null) { + //System.out.println(var.getName()+": " + propid + " = " +valstring); + if (type == Metric.URI) { + // Object Property + KBObject valobj = this.kb.getResource(valstring); + if (valobj == null) { + // TODO: Log and explain (make a utility + // function) + details.addExplanations( + "ERROR Cannot Recognize Metrics Value " + valstring + ); + continue; + } + // Copy over the object class into kb as well + // (except where the object itself is a class) + if (!metricProp.getID().equals(KBUtils.RDF + "type")) { + valobj = + this.copyObjectIntoKB( + valobj.getID(), + valobj.getID(), + tkb, + null, + null, + true + ); + // Remove any existing values first + for (KBTriple t : tkb.genericTripleQuery( + varobj, + metricProp, + null + )) tkb.removeTriple(t); + } + // Add a Triple for the metric property value + tkb.addTriple(varobj, metricProp, valobj); + } else if (type == Metric.LITERAL && val != null) { + // Literal value + KBObject tobj = dtype != null + ? tkb.createXSDLiteral(valstring, dtype) + : tkb.createLiteral(val); + if (tobj != null) { + // Remove any existing values first + for (KBTriple t : tkb.genericTripleQuery( + varobj, + metricProp, + null + )) tkb.removeTriple(t); + // Add a Triple for the metric propertyvalue + tkb.addTriple(varobj, metricProp, tobj); + } else { + details.addExplanations( + "ERROR Cannot Convert Metrics Value " + valstring + ); + continue; + } + } + } else { + // TODO: Log and explain (make a utility function) + details.addExplanations( + "ERROR No Such Metrics Property Known to Component Catalog : " + + propid + ); + continue; + } + } + } + + // Create other standard PC properties on variable + // - hasDimensionSizes + // - hasBindingID + if (var.getBinding().isSet()) { + String dimensionSizes = ""; + ArrayList vbs = new ArrayList(); + vbs.add(var.getBinding()); + while (!vbs.isEmpty()) { + Binding vb = vbs.remove(0); + if (vb.isSet()) { + for (WingsSet vs : vb) { + vbs.add((Binding) vs); + } + if (!dimensionSizes.equals("")) dimensionSizes += ","; + dimensionSizes += vb.getSize(); + } + } + tkb.setPropertyValue( + varobj, + dmap.get("hasDimensionSizes"), + tkb.createLiteral(dimensionSizes) + ); + } + + if (var.getBinding().getID() != null) tkb.addTriple( + varobj, + dmap.get("hasBindingID"), + tkb.createLiteral(var.getBinding().getName()) + ); else tkb.addTriple( + varobj, + dmap.get("hasBindingID"), + tkb.createLiteral("") + ); + // end if (var.getDataBinding() != null) + } + // end if (var.isDataVariable()) + } else if (var.isParameterVariable()) { + // If the Variable/Argument is a Parameter + ValueBinding parambinding = (ValueBinding) var.getBinding(); + if (parambinding != null && parambinding.getValue() != null) { + // If the template has any value specified, use that instead + //arg_value = tkb.createLiteral(var.getBinding().getValue()); + KBObject arg_value = tkb.createXSDLiteral( + parambinding.getValueAsString(), + parambinding.getDatatype() + ); + tkb.setPropertyValue(varobj, dmap.get("hasValue"), arg_value); + } + if (dmap.containsKey("hasBindingID")) tkb.addTriple( // Set the hasBindingID term + varobj, + dmap.get("hasBindingID"), + tkb.createLiteral("Param" + arg.getName()) + ); + } + + // Copy argument classes from Catalog as classes for the temporary + // variable in the temporary kb store + this.copyObjectClassesIntoKB( + varobj.getID(), + arg.getID(), + tkb, + null, + null, + true + ); + + // Set the temporary variable's argumentID so rules can get/set + // triples based on the argument + tkb.addTriple( + varobj, + dmap.get("hasArgumentID"), + tkb.createLiteral(rolestr) + ); + + // Set hasInput or hasOutput for the temporary Variable + if (sInputRoles.containsKey(rolestr)) { + tkb.addTriple(tcomp, omap.get("hasInput"), varobj); + } else { + tkb.addTriple(tcomp, omap.get("hasOutput"), varobj); + } + // end of for (String rolestr : sRoleMap.keySet()) + } + + // Add all metrics and datametrics properties to temporary store tkb.addTriples(metricTriples); - - // Set current output variable metrics to do a diff with later + + // Set current output variable metrics to do a diff with later for (String rolestr : sRoleMap.keySet()) { Variable var = sRoleMap.get(rolestr); if (var.isDataVariable() && !sInputRoles.containsKey(rolestr)) { @@ -955,54 +1127,56 @@ else if(v.getBinding().getValue() == null) { // Create Metrics from PC Properties for (KBObject metricProp : metricProps) { KBObject val = tkb.getPropertyValue(varobj, metricProp); - if (val == null) - continue; + if (val == null) continue; // Add value - if (val.isLiteral()) - metrics.addMetric(metricProp.getID(), new Metric(Metric.LITERAL, - val.getValue(), val.getDataType())); - else - metrics.addMetric(metricProp.getID(), - new Metric(Metric.URI, val.getID())); + if (val.isLiteral()) metrics.addMetric( + metricProp.getID(), + new Metric(Metric.LITERAL, val.getValue(), val.getDataType()) + ); else metrics.addMetric( + metricProp.getID(), + new Metric(Metric.URI, val.getID()) + ); } var.getBinding().setMetrics(metrics); } } - + KBRuleList rules = this.getCachedComponentRules(comp); - if(rules.getRules().size() > 0) { - // Redirect Standard output to a byte stream - ByteArrayOutputStream bost = new ByteArrayOutputStream(); - PrintStream oldout = System.out; - System.setOut(new PrintStream(bost, true)); - - // *** Run propagation rules on the temporary ontmodel *** - tkb.setRulePrefixes(this.rulePrefixes); - tkb.applyRules(rules); - //tkb.applyRulesFromString(allrules); - - // Add printouts from rules as explanations - if (!bost.toString().equals("")) { - for (String exp : bost.toString().split("\\n")) { - details.addExplanations(exp); - } - } - // Reset the Standard output - System.setOut(oldout); + if (rules.getRules().size() > 0) { + // Redirect Standard output to a byte stream + ByteArrayOutputStream bost = new ByteArrayOutputStream(); + PrintStream oldout = System.out; + System.setOut(new PrintStream(bost, true)); + + // *** Run propagation rules on the temporary ontmodel *** + tkb.setRulePrefixes(this.rulePrefixes); + tkb.applyRules(rules); + //tkb.applyRulesFromString(allrules); + + // Add printouts from rules as explanations + if (!bost.toString().equals("")) { + for (String exp : bost.toString().split("\\n")) { + details.addExplanations(exp); + } + } + // Reset the Standard output + System.setOut(oldout); + } + + // Check if the rules marked this component as invalid for + // the current component details packet + KBObject invalidProp = this.dataPropMap.get("isInvalid"); + KBObject isInvalid = tkb.getPropertyValue(tcomp, invalidProp); + if (isInvalid != null && (Boolean) isInvalid.getValue()) { + details.addExplanations( + "INFO " + tcomp + " is not valid for its inputs" + ); + logger.debug(tcomp + " is not valid for its inputs"); + details.setInvalidFlag(true); + list.add(details); + return list; } - - // Check if the rules marked this component as invalid for - // the current component details packet - KBObject invalidProp = this.dataPropMap.get("isInvalid"); - KBObject isInvalid = tkb.getPropertyValue(tcomp, invalidProp); - if (isInvalid != null && (Boolean) isInvalid.getValue()) { - details.addExplanations("INFO "+tcomp + " is not valid for its inputs"); - logger.debug(tcomp + " is not valid for its inputs"); - details.setInvalidFlag(true); - list.add(details); - return list; - } - + // Checking for NoOperation KBObject noOpProp = tkb.getProperty(this.pcns + "isNoOperation"); KBObject isNoOperation = tkb.getPropertyValue(tcomp, noOpProp); @@ -1010,337 +1184,390 @@ else if(v.getBinding().getValue() == null) { logger.debug(tcomp + " is going to be skipped"); details.addExplanations("INFO " + tcomp + " is going to be skipped"); details.setNoOperationFlag(true); - - // Set compatible input/outputs - - for(ComponentRole outrole : comp.getOutputs()) { + + // Set compatible input/outputs + + for (ComponentRole outrole : comp.getOutputs()) { Variable outvar = sRoleMap.get(outrole.getRoleName()); ArrayList varclasses = new ArrayList(); - for(KBObject clsobj : this.getAllCachedClassesOfInstance(outrole.getID(), true)) { + for (KBObject clsobj : this.getAllCachedClassesOfInstance( + outrole.getID(), + true + )) { varclasses.add(clsobj.getID()); } - - // Marking all compatible inputs to be passed through to the output. + + // Marking all compatible inputs to be passed through to the output. // Note: If not all outputs have a matching input, then component is still run and not skipped - ArrayList matchingInputs = new ArrayList(); - for(ComponentRole inrole : comp.getInputs()) { - if(!inrole.isParam()) { - if(checkTypeCompatibility(varclasses, inrole.getID())) { + ArrayList matchingInputs = + new ArrayList(); + for (ComponentRole inrole : comp.getInputs()) { + if (!inrole.isParam()) { + if (checkTypeCompatibility(varclasses, inrole.getID())) { matchingInputs.add(inrole); } } } - + ComponentRole matchingInput = null; if (matchingInputs.size() > 0) { // If some outputs have multiple matching inputs, then we match by prefix (-i1 matches with -o1) // - If no match by prefix, then just choose the first match int outPrefix = Integer.parseInt(outrole.getPrefix().substring(2)); - + matchingInput = matchingInputs.get(0); if (matchingInputs.size() > 1) { for (ComponentRole mInput : matchingInputs) { - int inPrefix = Integer.parseInt(mInput.getPrefix().substring(2)); + int inPrefix = Integer.parseInt( + mInput.getPrefix().substring(2) + ); if (inPrefix == outPrefix) { matchingInput = mInput; } } } } - + if (matchingInput != null) { // Marking the compatible input to be passed through to the output. - details.addNoOperationIOPassthrough(outrole.getRoleName(), matchingInput.getRoleName()); - + details.addNoOperationIOPassthrough( + outrole.getRoleName(), + matchingInput.getRoleName() + ); + Variable invar = sRoleMap.get(matchingInput.getRoleName()); - logger.debug("Passing through " + invar.getName() + " to " + outvar.getName()); + logger.debug( + "Passing through " + invar.getName() + " to " + outvar.getName() + ); } } } - - - // Check component dependencies - // If set, overwrite the component dependencies with these - ComponentRequirement req = comp.getComponentRequirement(); - if(req != null) { - if(req.getMemoryGB() != 0) - c.getRequirements().setMemoryGB(req.getMemoryGB()); - if(req.getStorageGB() != 0) - c.getRequirements().setStorageGB(req.getStorageGB()); - } - - // Set values of variables by looking at values set by rules - // in temporary kb store - // - Only set if there isn't already a binding value for the variable - for (Variable var : sRoleMap.values()) { - if (var.isParameterVariable() - && (noParamBindings.containsKey(var.getID()) || - var.getBinding() == null || - var.getBinding().getValue() == null)) { - KBObject varobj = tkb.getResource(variableNameMap.get(var)); - KBObject origvarobj = tkb.getResource(var.getID()); - KBObject val = tkb.getPropertyValue(varobj, dmap.get("hasValue")); - if (val != null && val.getValue() != null) { - tkb.addTriple(origvarobj, - tkb.getResource(this.wflowns + "hasParameterValue"), val); - var.setBinding(new ValueBinding(val.getValue(), val.getDataType())); - } - } - } - - // To create the output Variable metrics, we go through the metrics - // property of the output data variables and get their metrics property - // values - for (String rolestr : sRoleMap.keySet()) { - Variable var = sRoleMap.get(rolestr); - if (var.isDataVariable() && !sInputRoles.containsKey(rolestr)) { - Metrics curmetrics = var.getBinding().getMetrics(); - Metrics metrics = new Metrics(); - KBObject varobj = tkb.getResource(variableNameMap.get(var)); - - // Create Metrics from PC Properties - for (KBObject metricProp : metricProps) { - ArrayList vals = tkb.getPropertyValues(varobj, metricProp); - if(vals == null) - continue; - for(KBObject val : vals) { - if(vals.size() > 1) { - if(!curmetrics.getMetrics().containsKey(metricProp.getID())) - continue; - // If multiple values present, ignore value that is equal to current value - for(Metric mval : curmetrics.getMetrics().get(metricProp.getID())) { - if(!val.isLiteral() && val.getID().equals(mval.getValue())) - continue; - else if(val.isLiteral() && val.getValue().equals(mval.getValue())) - continue; - } - } - // Add value - if (val.isLiteral()) - metrics.addMetric(metricProp.getID(), new Metric(Metric.LITERAL, - val.getValue(), val.getDataType())); - else - metrics.addMetric(metricProp.getID(), - new Metric(Metric.URI, val.getID())); - } - } - ArrayList clses = this.getAllClassesOfInstance(tkb, varobj.getID()); - for (KBObject cls : clses) - metrics.addMetric(KBUtils.RDF + "type", new Metric(Metric.URI, cls.getID())); - - // Set metrics for the Binding - if (var.getBinding() != null) - var.getBinding().setMetrics(metrics); - - // -- Dealing with Collections -- - // User other Properties for creating output binding collections - // and setting the collection item metrics as well - // PC Properties used: - // - hasDimensionSizes - // - hasDimensionIndexProperties - int dim = 0; - final int maxdims = 10; // not more than 10 dimensions - int[] dimSizes = new int[maxdims]; - String[] dimIndexProps = new String[maxdims]; - KBObject dimSizesObj = tkb.getPropertyValue(varobj, dmap.get("hasDimensionSizes")); - KBObject dimIndexPropsObj = tkb.getPropertyValue(varobj, - dmap.get("hasDimensionIndexProperties")); - - // Parse dimension sizes string (can be given as a comma-separated list) - // Example 2,3 - // - This will create a 2x3 matrix - if (dimSizesObj != null && dimSizesObj.getValue() != null) { - if (dimSizesObj.getValue().getClass().getName().equals("java.lang.Integer")) { - dimSizes[0] = (Integer) dimSizesObj.getValue(); - dim = 1; - } else { - String dimSizesStr = (String) dimSizesObj.getValue(); - for (String dimSize : dimSizesStr.split(",")) { - try { - int size = Integer.parseInt(dimSize); - dimSizes[dim] = size; - dim++; - } catch (Exception e) { - } - } - } - } - - // Parse dimension index string (can be given as a comma - // separated list) - // Example hasXIndex, hasYIndex - // - This will set each output item's - // - first dimension index using property hasXIndex - // - second dimension index using property hasYIndex - // Example output: - // - output - // - output0 (hasXIndex 0) - // - output00 (hasXIndex 0, hasYIndex 0) - // - output01 (hasXIndex 0, hasYIndex 1) - // - output0 (hasXIndex 1) - // - output10 (hasXIndex 1, hasYIndex 0) - // - output11 (hasXIndex 1, hasYIndex 1) - - if (dimIndexPropsObj != null && dimIndexPropsObj.getValue() != null) { - int xdim = 0; - String dimIndexPropsStr = (String) dimIndexPropsObj.getValue(); - for (String dimIndexProp : dimIndexPropsStr.split(",")) { - try { - dimIndexProps[xdim] = dimIndexProp; - xdim++; - } catch (Exception e) { - } - } - } - - // If the output is a collection - // dim = 1 is a List - // dim = 2 is a Matrix - // dim = 3 is a Cube - // .. and so on - if (dim > 0) { - int[] dimCounters = new int[dim]; - dimCounters[0] = 1; - for (int k = 1; k < dim; k++) { - int perms = 1; - for (int l = k - 1; l >= 0; l--) - perms *= dimSizes[l]; - dimCounters[k] = dimCounters[k - 1] + perms; - } - - Binding b = var.getBinding(); - ArrayList vbs = new ArrayList(); - vbs.add(b); - int counter = 0; - while (!vbs.isEmpty()) { - Binding vb = vbs.remove(0); - if (vb.getMetrics() == null) - continue; - int vdim = 0; - for (vdim = 0; vdim < dim; vdim++) { - if (counter < dimCounters[vdim]) - break; - } - if (vdim < dim) { - for (int i = 0; i < dimSizes[vdim]; i++) { - Binding cvb = new Binding(b.getNamespace() - + UuidGen.generateAUuid("" + i)); - // Copy over metrics from parent variable binding + + // Check component dependencies + // If set, overwrite the component dependencies with these + ComponentRequirement req = comp.getComponentRequirement(); + if (req != null) { + if (req.getMemoryGB() != 0) c + .getRequirements() + .setMemoryGB(req.getMemoryGB()); + if (req.getStorageGB() != 0) c + .getRequirements() + .setStorageGB(req.getStorageGB()); + } + + // Set values of variables by looking at values set by rules + // in temporary kb store + // - Only set if there isn't already a binding value for the variable + for (Variable var : sRoleMap.values()) { + if ( + var.isParameterVariable() && + ( + noParamBindings.containsKey(var.getID()) || + var.getBinding() == null || + var.getBinding().getValue() == null + ) + ) { + KBObject varobj = tkb.getResource(variableNameMap.get(var)); + KBObject origvarobj = tkb.getResource(var.getID()); + KBObject val = tkb.getPropertyValue(varobj, dmap.get("hasValue")); + if (val != null && val.getValue() != null) { + tkb.addTriple( + origvarobj, + tkb.getResource(this.wflowns + "hasParameterValue"), + val + ); + var.setBinding(new ValueBinding(val.getValue(), val.getDataType())); + } + } + } + + // To create the output Variable metrics, we go through the metrics + // property of the output data variables and get their metrics property + // values + for (String rolestr : sRoleMap.keySet()) { + Variable var = sRoleMap.get(rolestr); + if (var.isDataVariable() && !sInputRoles.containsKey(rolestr)) { + Metrics curmetrics = var.getBinding().getMetrics(); + Metrics metrics = new Metrics(); + KBObject varobj = tkb.getResource(variableNameMap.get(var)); + + // Create Metrics from PC Properties + for (KBObject metricProp : metricProps) { + ArrayList vals = tkb.getPropertyValues( + varobj, + metricProp + ); + if (vals == null) continue; + for (KBObject val : vals) { + if (vals.size() > 1) { + if ( + !curmetrics.getMetrics().containsKey(metricProp.getID()) + ) continue; + // If multiple values present, ignore value that is equal to current value + for (Metric mval : curmetrics + .getMetrics() + .get(metricProp.getID())) { + if ( + !val.isLiteral() && val.getID().equals(mval.getValue()) + ) continue; else if ( + val.isLiteral() && val.getValue().equals(mval.getValue()) + ) continue; + } + } + // Add value + if (val.isLiteral()) metrics.addMetric( + metricProp.getID(), + new Metric(Metric.LITERAL, val.getValue(), val.getDataType()) + ); else metrics.addMetric( + metricProp.getID(), + new Metric(Metric.URI, val.getID()) + ); + } + } + ArrayList clses = + this.getAllClassesOfInstance(tkb, varobj.getID()); + for (KBObject cls : clses) metrics.addMetric( + KBUtils.RDF + "type", + new Metric(Metric.URI, cls.getID()) + ); + + // Set metrics for the Binding + if (var.getBinding() != null) var.getBinding().setMetrics(metrics); + + // -- Dealing with Collections -- + // User other Properties for creating output binding collections + // and setting the collection item metrics as well + // PC Properties used: + // - hasDimensionSizes + // - hasDimensionIndexProperties + int dim = 0; + final int maxdims = 10; // not more than 10 dimensions + int[] dimSizes = new int[maxdims]; + String[] dimIndexProps = new String[maxdims]; + KBObject dimSizesObj = tkb.getPropertyValue( + varobj, + dmap.get("hasDimensionSizes") + ); + KBObject dimIndexPropsObj = tkb.getPropertyValue( + varobj, + dmap.get("hasDimensionIndexProperties") + ); + + // Parse dimension sizes string (can be given as a comma-separated list) + // Example 2,3 + // - This will create a 2x3 matrix + if (dimSizesObj != null && dimSizesObj.getValue() != null) { + if ( + dimSizesObj + .getValue() + .getClass() + .getName() + .equals("java.lang.Integer") + ) { + dimSizes[0] = (Integer) dimSizesObj.getValue(); + dim = 1; + } else { + String dimSizesStr = (String) dimSizesObj.getValue(); + for (String dimSize : dimSizesStr.split(",")) { + try { + int size = Integer.parseInt(dimSize); + dimSizes[dim] = size; + dim++; + } catch (Exception e) {} + } + } + } + + // Parse dimension index string (can be given as a comma + // separated list) + // Example hasXIndex, hasYIndex + // - This will set each output item's + // - first dimension index using property hasXIndex + // - second dimension index using property hasYIndex + // Example output: + // - output + // - output0 (hasXIndex 0) + // - output00 (hasXIndex 0, hasYIndex 0) + // - output01 (hasXIndex 0, hasYIndex 1) + // - output0 (hasXIndex 1) + // - output10 (hasXIndex 1, hasYIndex 0) + // - output11 (hasXIndex 1, hasYIndex 1) + + if (dimIndexPropsObj != null && dimIndexPropsObj.getValue() != null) { + int xdim = 0; + String dimIndexPropsStr = (String) dimIndexPropsObj.getValue(); + for (String dimIndexProp : dimIndexPropsStr.split(",")) { + try { + dimIndexProps[xdim] = dimIndexProp; + xdim++; + } catch (Exception e) {} + } + } + + // If the output is a collection + // dim = 1 is a List + // dim = 2 is a Matrix + // dim = 3 is a Cube + // .. and so on + if (dim > 0) { + int[] dimCounters = new int[dim]; + dimCounters[0] = 1; + for (int k = 1; k < dim; k++) { + int perms = 1; + for (int l = k - 1; l >= 0; l--) perms *= dimSizes[l]; + dimCounters[k] = dimCounters[k - 1] + perms; + } + + Binding b = var.getBinding(); + ArrayList vbs = new ArrayList(); + vbs.add(b); + int counter = 0; + while (!vbs.isEmpty()) { + Binding vb = vbs.remove(0); + if (vb.getMetrics() == null) continue; + int vdim = 0; + for (vdim = 0; vdim < dim; vdim++) { + if (counter < dimCounters[vdim]) break; + } + if (vdim < dim) { + for (int i = 0; i < dimSizes[vdim]; i++) { + Binding cvb = new Binding( + b.getNamespace() + UuidGen.generateAUuid("" + i) + ); + // Copy over metrics from parent variable binding Metrics tmpMetrics = new Metrics(vb.getMetrics()); - // Add dimension index (if property set) - String prop = dimIndexProps[vdim]; - if (prop != null && !prop.equals("")) { - Metric nm = new Metric(Metric.LITERAL, i, KBUtils.XSD+"integer"); - tmpMetrics.addMetric(this.dcdomns + prop, nm); - } + // Add dimension index (if property set) + String prop = dimIndexProps[vdim]; + if (prop != null && !prop.equals("")) { + Metric nm = new Metric( + Metric.LITERAL, + i, + KBUtils.XSD + "integer" + ); + tmpMetrics.addMetric(this.dcdomns + prop, nm); + } cvb.setMetrics(tmpMetrics); - vb.add(cvb); - vbs.add(cvb); - } - } - counter++; - } - } - // end if(dim > 0) - } - } - // FIXME: Handle multiple configurations - list.add(details); - return list; - } - catch(Exception e) { - e.printStackTrace(); - } - return null; - } - - public ComponentInvocation getComponentInvocation(ComponentPacket details) { - HashMap omap = this.objPropMap; - HashMap dmap = this.dataPropMap; - - // Extract info from details object - ComponentVariable c = details.getComponent(); - Map roleMap = details.getRoleMap(); - - // Get Component - ComponentInvocation invocation = new ComponentInvocation(); - try { - this.start_read(); - KBObject comp = this.kb.getResource(c.getBinding().getID()); - String exepath = this.getComponentLocation(comp.getID()); - - - String exedir = null; - if(exepath != null) { - File f = new File(exepath); - if(f.isDirectory()) { - exedir = exepath; - File shexef = new File(exepath + File.separator + "run"); - File winexef = new File(exepath + File.separator + "run.bat"); - if(SystemUtils.IS_OS_WINDOWS && winexef.exists()) - exepath = winexef.getAbsolutePath(); - else - exepath = shexef.getAbsolutePath(); - } - } - - invocation.setComponentId(comp.getID()); - invocation.setComponentLocation(exepath); - invocation.setComponentDirectory(exedir); - - ArrayList inputs = this.kb.getPropertyValues(comp, omap.get("hasInput")); - ArrayList outputs = this.kb.getPropertyValues(comp, omap.get("hasOutput")); - ArrayList args = new ArrayList(inputs); - args.addAll(outputs); - - for (KBObject arg : args) { - KBObject argid = this.kb.getDatatypePropertyValue(arg, dmap.get("hasArgumentID")); - String role = (String) argid.getValue(); - for(Role r : roleMap.keySet()) { - Variable var = roleMap.get(r); - if(r.getRoleId().equals(role)) - setInvocationArguments(invocation, arg, var, inputs.contains(arg)); - } - } - this.end(); + vb.add(cvb); + vbs.add(cvb); + } + } + counter++; + } + } + // end if(dim > 0) + } + } + // FIXME: Handle multiple configurations + list.add(details); + return list; + } catch (Exception e) { + e.printStackTrace(); } - catch (Exception e) { + return null; + } + + public ComponentInvocation getComponentInvocation(ComponentPacket details) { + HashMap omap = this.objPropMap; + HashMap dmap = this.dataPropMap; + + // Extract info from details object + ComponentVariable c = details.getComponent(); + Map roleMap = details.getRoleMap(); + + // Get Component + ComponentInvocation invocation = new ComponentInvocation(); + try { + this.start_read(); + KBObject comp = this.kb.getResource(c.getBinding().getID()); + String exepath = this.getComponentLocation(comp.getID()); + + String exedir = null; + if (exepath != null) { + File f = new File(exepath); + if (f.isDirectory()) { + exedir = exepath; + File shexef = new File(exepath + File.separator + "run"); + File winexef = new File(exepath + File.separator + "run.bat"); + if (SystemUtils.IS_OS_WINDOWS && winexef.exists()) exepath = + winexef.getAbsolutePath(); else exepath = shexef.getAbsolutePath(); + } + } + + invocation.setComponentId(comp.getID()); + invocation.setComponentLocation(exepath); + invocation.setComponentDirectory(exedir); + + ArrayList inputs = + this.kb.getPropertyValues(comp, omap.get("hasInput")); + ArrayList outputs = + this.kb.getPropertyValues(comp, omap.get("hasOutput")); + ArrayList args = new ArrayList(inputs); + args.addAll(outputs); + + for (KBObject arg : args) { + KBObject argid = + this.kb.getDatatypePropertyValue(arg, dmap.get("hasArgumentID")); + String role = (String) argid.getValue(); + for (Role r : roleMap.keySet()) { + Variable var = roleMap.get(r); + if (r.getRoleId().equals(role)) setInvocationArguments( + invocation, + arg, + var, + inputs.contains(arg) + ); + } + } + this.end(); + } catch (Exception e) { e.printStackTrace(); this.end(); } - - return invocation; - } - - private void setInvocationArguments(ComponentInvocation invocation, KBObject arg, - Variable var, boolean isInput) { - - HashMap dmap = this.dataPropMap; - KBObject defaultValue = this.kb.getPropertyValue(arg, dmap.get("hasValue")); - KBObject prefixobj = this.kb.getPropertyValue(arg, dmap.get("hasArgumentName")); - String prefix = (String) prefixobj.getValue(); - - ArrayList bindings = new ArrayList(); - bindings.add(var.getBinding()); - while (!bindings.isEmpty()) { - Binding b = bindings.remove(0); - if (b.isSet()) { - for (WingsSet s : b) { - bindings.add((Binding) s); - } - } else { - if (var.isDataVariable()) { - invocation.addArgument(prefix, b, var.getID(), isInput); - } else if (var.isParameterVariable()) { - if (var.getBinding() != null) { - invocation.addArgument(prefix, ((ValueBinding)var.getBinding()).getValueAsString(), - var.getID(), isInput); - } else if (defaultValue != null && defaultValue.getValue() != null) { - invocation.addArgument(prefix, defaultValue.getValueAsString(), - var.getID(), isInput); - } - } - } - } - } + + return invocation; + } + + private void setInvocationArguments( + ComponentInvocation invocation, + KBObject arg, + Variable var, + boolean isInput + ) { + HashMap dmap = this.dataPropMap; + KBObject defaultValue = this.kb.getPropertyValue(arg, dmap.get("hasValue")); + KBObject prefixobj = + this.kb.getPropertyValue(arg, dmap.get("hasArgumentName")); + String prefix = (String) prefixobj.getValue(); + + ArrayList bindings = new ArrayList(); + bindings.add(var.getBinding()); + while (!bindings.isEmpty()) { + Binding b = bindings.remove(0); + if (b.isSet()) { + for (WingsSet s : b) { + bindings.add((Binding) s); + } + } else { + if (var.isDataVariable()) { + invocation.addArgument(prefix, b, var.getID(), isInput); + } else if (var.isParameterVariable()) { + if (var.getBinding() != null) { + invocation.addArgument( + prefix, + ((ValueBinding) var.getBinding()).getValueAsString(), + var.getID(), + isInput + ); + } else if (defaultValue != null && defaultValue.getValue() != null) { + invocation.addArgument( + prefix, + defaultValue.getValueAsString(), + var.getID(), + isInput + ); + } + } + } + } + } } diff --git a/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/TemplateReasoningKB.java b/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/TemplateReasoningKB.java index 7aecab05..a7d5e067 100644 --- a/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/TemplateReasoningKB.java +++ b/planner/src/main/java/edu/isi/wings/catalog/component/api/impl/kb/TemplateReasoningKB.java @@ -15,10 +15,7 @@ * limitations under the License. */ -package edu.isi.wings.catalog.component.api.impl.kb; - -import java.util.ArrayList; -import java.util.HashMap; +package edu.isi.wings.catalog.component.api.impl.kb; import edu.isi.kcap.ontapi.KBObject; import edu.isi.kcap.ontapi.KBTriple; @@ -33,206 +30,229 @@ import edu.isi.wings.workflow.template.classes.sets.Binding; import edu.isi.wings.workflow.template.classes.variables.ComponentVariable; import edu.isi.wings.workflow.template.classes.variables.Variable; - -public class TemplateReasoningKB extends TransactionsJena implements ComponentReasoningAPI { - - WorkflowGenerationKB swg; - String requestId; - OntFactory ontfac; - - public TemplateReasoningKB(WorkflowGenerationKB swg) { - this.swg = swg; - setRequestId(requestId); - this.ontfac = new OntFactory(OntFactory.JENA); - } - - public boolean componentSubsumes(String subsumerClassID, String subsumedClassID) { - // TODO: Maybe can be done later - return false; - } - - public ComponentPacket findDataDetails(ComponentPacket details) { - // TODO: Should run a light backward,forward sweep ? - return null; - } - - public ComponentPacket findDataTypeDetails(ComponentPacket details) { - // TODO: Should run a type-only light backward,forward sweep ? - return null; - } - - // TODO/FIXME: Ignore Variable constraints for now - public ArrayList findOutputDataPredictedDescriptions(ComponentPacket cmr) { - // Do a forward sweep - ArrayList list = new ArrayList(); - - ComponentVariable c = cmr.getComponent(); - HashMap sRoleMap = cmr.getStringRoleMaps(); - - Template t = (Template) c.getBinding().getValue(); - if (t == null) - return null; - - HashMap tRoles = new HashMap(t.getInputRoles()); - tRoles.putAll(t.getOutputRoles()); - - // TODO: Role variables current do not equate to Template Variables - - // Transfer bindings - for (String varid : tRoles.keySet()) { - Role r = tRoles.get(varid); - Variable v = sRoleMap.get(r.getID()); - if (v != null) - t.getVariable(varid).setBinding(v.getBinding()); - } - - // Do forward sweep on the sub-template - ArrayList