Skip to content

Commit

Permalink
Added code to generate DOI list for any object alignment download
Browse files Browse the repository at this point in the history
  • Loading branch information
paulopinheiro1234 committed Mar 24, 2020
1 parent 003ab08 commit 0189360
Show file tree
Hide file tree
Showing 15 changed files with 509 additions and 23 deletions.
2 changes: 1 addition & 1 deletion app/org/hadatac/console/controllers/Version.java
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
public class Version extends Controller {

public Result index() {
String code_version = "1.1.23";
String code_version = "1.1.24";
String base_ontology = ConfigFactory.load().getString("hadatac.community.ont_prefix");
String loaded_base_ontology = NameSpaces.getInstance().getNameByAbbreviation(base_ontology);
String loaded_base_ontology_version = Ontology.getVersionFromAbbreviation(base_ontology);
Expand Down
54 changes: 54 additions & 0 deletions app/org/hadatac/console/controllers/triplestore/Clean.java
Original file line number Diff line number Diff line change
@@ -1,18 +1,25 @@
package org.hadatac.console.controllers.triplestore;

import java.util.List;
import java.io.File;

import play.mvc.*;

import org.hadatac.console.controllers.AuthApplication;
import org.hadatac.console.controllers.annotator.AutoAnnotator;
import org.hadatac.console.views.html.triplestore.*;
import org.hadatac.data.loader.DataContext;
import org.hadatac.metadata.loader.MetadataContext;
import org.hadatac.metadata.loader.PermissionsContext;
import org.hadatac.metadata.loader.URIUtils;
import org.hadatac.utils.Feedback;
import org.hadatac.utils.NameSpace;
import org.hadatac.utils.NameSpaces;

import com.typesafe.config.ConfigFactory;

import org.hadatac.entity.pojo.DataFile;
import org.hadatac.entity.pojo.Measurement;
import be.objectify.deadbolt.java.actions.Group;
import be.objectify.deadbolt.java.actions.Restrict;

Expand Down Expand Up @@ -73,9 +80,56 @@ public static String playClean(String oper) {
ConfigFactory.load().getString("hadatac.solr.data"),
false);
result = acquisition.cleanAcquisitionData(Feedback.WEB);
} else if (oper.equals("unprocessed")) {
List<DataFile> selected = DataFile.findByStatus(DataFile.UNPROCESSED);
String message = "0 unprocessed datafiles deleted";
if (selected != null && selected.size() > 0) {
message = selected.size() + " unprocessed datafiles deleted";
for (DataFile df : selected) {
Clean.deleteDataFile(df);
}
}
result = message;
} else if (oper.equals("processed")) {
List<DataFile> selected = DataFile.findByStatus(DataFile.PROCESSED);
String message = "0 processed datafiles deleted";
if (selected != null && selected.size() > 0) {
message = selected.size() + " processed datafiles deleted";
for (DataFile df : selected) {
Clean.deleteDataFile(df);
}
}
result = message;
} else if (oper.equals("working")) {
List<DataFile> selected = DataFile.findByStatus(DataFile.WORKING);
String message = "0 working datafiles deleted";
if (selected != null && selected.size() > 0) {
message = selected.size() + " working datafiles deleted";
for (DataFile df : selected) {
Clean.deleteDataFile(df);
}
}
result = message;
}

return result;
}

private static void deleteDataFile(DataFile dataFile) {
File file = new File(dataFile.getAbsolutePath());

if (dataFile.getPureFileName().startsWith("DA-")) {
Measurement.deleteFromSolr(dataFile.getDatasetUri());
NameSpace.deleteTriplesByNamedGraph(URIUtils.replacePrefixEx(dataFile.getDataAcquisitionUri()));
} else {
try {
AutoAnnotator.deleteAddedTriples(file, dataFile);
} catch (Exception e) {
System.out.print("Can not delete triples ingested by " + dataFile.getFileName() + " ..");
}
}
file.delete();
dataFile.delete();
}

}
25 changes: 23 additions & 2 deletions app/org/hadatac/console/views/triplestore/clean.scala.html
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
@import org.hadatac.data.loader._
@import org.hadatac.metadata.loader._
@import org.hadatac.utils._
@import org.hadatac.entity.pojo._
@import java.net._
@import play._
@import com.typesafe.config.ConfigFactory
Expand Down Expand Up @@ -56,14 +57,17 @@ <h4>Data Repository</h4>
@if(Repository.operational(Repository.DATA)) {
<b><font color="green">OPERATIONAL</font></b>
<br>
Data Acquisitions: <b>@DataContext.playTotalDataAcquisitions() documents.</b> Data Content: <b>@DataContext.playTotalMeasurements() documents.</b>
Data Acquisitions: <b>@DataContext.playTotalDataAcquisitions() documents.</b>
<!-- Data Files: <b>@DataContext.playTotalDataFiles() documents.</b> -->
Data Content: <b>@DataContext.playTotalMeasurements() documents.</b>
} else {
<b><font color="red">DOWN</font></b>
}
<br><br>

<div id="question-box">
<a href="@org.hadatac.console.controllers.triplestore.routes.Clean.clean("collections")" class="btn btn-primary" role="button">Clean Data Acquisitions</a>
<!-- <a href="@org.hadatac.console.controllers.triplestore.routes.Clean.clean("files")" class="btn btn-primary" role="button">Clean Data Files</a> -->
<a href="@org.hadatac.console.controllers.triplestore.routes.Clean.clean("acquisitions")" class="btn btn-primary" role="button">Clean Data</a>
</div>
</div>
Expand Down Expand Up @@ -91,6 +95,22 @@ <h4>Metadata Repository</h4>

<hr>

<h4>File Repository</h4>
<div id="rcorners">
Status: <b><font color="green">OPERATIONAL</font></b>
<br>
Unprocessed: <b> @DataFile.totalByStatus(DataFile.UNPROCESSED) files.</b> Processed: <b> @DataFile.totalByStatus(DataFile.PROCESSED) files. </b> Working: <b> @DataFile.totalByStatus(DataFile.WORKING) files.</b>
<br><br>

<div id="question-box">
<a href="@org.hadatac.console.controllers.triplestore.routes.Clean.clean("unprocessed")" class="btn btn-primary" role="button">Clean Unprocessed Files</a>
<a href="@org.hadatac.console.controllers.triplestore.routes.Clean.clean("processed")" class="btn btn-primary" role="button">Clean Processed Files</a>
<a href="@org.hadatac.console.controllers.triplestore.routes.Clean.clean("working")" class="btn btn-primary" role="button">Clean Working Files</a>
</div>
</div>

<hr>

}

<p style='font-family: "Lucida Console", Monaco, monospace;' id='msgarea'></p>
Expand All @@ -100,7 +120,8 @@ <h4>Clean complete -- check the results above to see if the clean was successful
<a href="@org.hadatac.console.controllers.triplestore.routes.Clean.clean("init")" class="btn btn-primary" role="button">Back to Repository Management</a>
</div>

@if((oper=="collections") || (oper=="acquisitions") || (oper=="metadata") || (oper=="usergraph")) {
@if((oper=="collections") || (oper=="acquisitions") || (oper=="files") || (oper=="metadata") || (oper=="usergraph") ||
(oper=="unprocessed") || (oper=="processed") || (oper=="working")) {
<script>
var msg = "@Clean.playClean(oper)";
$('#msgarea').html(msg.replace(/&lt;br&gt;/g, '<br>'));
Expand Down
68 changes: 67 additions & 1 deletion app/org/hadatac/data/loader/AnnotationWorker.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.hadatac.data.api.DataFactory;
import org.hadatac.entity.pojo.STR;
import org.hadatac.entity.pojo.DataFile;
import org.hadatac.entity.pojo.DOI;
import org.hadatac.entity.pojo.DPL;
import org.hadatac.entity.pojo.DataAcquisitionSchema;
import org.hadatac.entity.pojo.DataAcquisitionSchemaAttribute;
Expand Down Expand Up @@ -76,6 +77,9 @@ public static GeneratorChain getGeneratorChain(DataFile dataFile) {
} else if (fileName.startsWith("SSD-")) {
chain = annotateSSDFile(dataFile);

} else if (fileName.startsWith("DOI-")) {
chain = annotateDOIFile(dataFile);

} else {
dataFile.getLogger().printExceptionById("GBL_00001");
return null;
Expand Down Expand Up @@ -371,6 +375,66 @@ public static GeneratorChain annotateStudyIdFile(DataFile dataFile) {
return chain;
}

public static GeneratorChain annotateDOIFile(DataFile dataFile) {
System.out.println("Processing DOI file ...");
RecordFile recordFile = new SpreadsheetRecordFile(dataFile.getFile(), "InfoSheet");
if (!recordFile.isValid()) {
dataFile.getLogger().printExceptionById("DOI_00001");
return null;
} else {
dataFile.setRecordFile(recordFile);
}

DOI doi = new DOI(dataFile);
Map<String, String> mapCatalog = doi.getCatalog();
GeneratorChain chain = new GeneratorChain();

String studyId = doi.getStudyId();
if (studyId == null || studyId.isEmpty()) {
dataFile.getLogger().printExceptionByIdWithArgs("DOI_00002", studyId);
return null;
} else {
Study study = Study.findById(studyId);
if (study != null) {
chain.setStudyUri(study.getUri());
dataFile.getLogger().println("DOI ingestion: Found study id [" + studyId + "]");
} else {
dataFile.getLogger().printExceptionByIdWithArgs("DOI_00003", studyId);
return null;
}
}

chain.setDataFile(dataFile);

String doiVersion = doi.getVersion();
if (doiVersion != null && !doiVersion.isEmpty()) {
dataFile.getLogger().println("DOI ingestion: version is [" + doiVersion + "]");
} else {
dataFile.getLogger().printExceptionById("DOI_00004");
return null;
}

if (mapCatalog.get("Filenames") == null) {
dataFile.getLogger().printExceptionById("DOI_00005");
return null;
}

String sheetName = mapCatalog.get("Filenames").replace("#", "");
RecordFile sheet = new SpreadsheetRecordFile(dataFile.getFile(), sheetName);

try {
DataFile dataFileForSheet = (DataFile)dataFile.clone();
dataFileForSheet.setRecordFile(sheet);
chain.addGenerator(new DOIGenerator(dataFileForSheet));
} catch (CloneNotSupportedException e) {
e.printStackTrace();
dataFile.getLogger().printExceptionById("DOI_00006");
return null;
}

return chain;
}

public static GeneratorChain annotateDPLFile(DataFile dataFile) {
RecordFile recordFile = new SpreadsheetRecordFile(dataFile.getFile(), "InfoSheet");
if (!recordFile.isValid()) {
Expand Down Expand Up @@ -428,9 +492,11 @@ public static GeneratorChain annotateSDDFile(DataFile dataFile) {
String sddVersion = sdd.getVersion();
if (sddName == "") {
dataFile.getLogger().printExceptionById("SDD_00003");
return null;
}
if (sddVersion == "") {
dataFile.getLogger().printExceptionById("SDD_00018");
return null;
}
Map<String, String> mapCatalog = sdd.getCatalog();

Expand Down Expand Up @@ -554,7 +620,7 @@ public static GeneratorChain annotateSSDFile(DataFile dataFile) {
//System.out.println("added SSDGenerator for " + dataFile.getAbsolutePath());

String studyUri = socgen.getStudyUri();
if (studyUri == null || studyUri == "") {
if (studyUri == null || studyUri.isEmpty()) {
return null;
} else {
chain.setStudyUri(studyUri);
Expand Down
67 changes: 67 additions & 0 deletions app/org/hadatac/data/loader/DOIGenerator.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package org.hadatac.data.loader;

import java.lang.String;
import java.util.HashMap;
import java.util.Map;
import java.util.List;
import java.util.ArrayList;
import java.util.StringTokenizer;

import org.hadatac.entity.pojo.DataFile;
import org.hadatac.metadata.loader.URIUtils;

public class DOIGenerator extends BaseGenerator {

String fileName;

public DOIGenerator(DataFile dataFile) {
super(dataFile);
this.fileName = dataFile.getFileName();
}

@Override
public void initMapping() {
mapCol.clear();
mapCol.put("filename", "Filename");
mapCol.put("wasDerivedFrom", "prov:wasDerivedFrom");
}

private String getFilename(Record rec) {
return rec.getValueByColumnName(mapCol.get("filename"));
}

private String getWasDerivedFrom(Record rec) {
return rec.getValueByColumnName(mapCol.get("wasDerivedFrom"));
}

@Override
public Map<String, Object> createRow(Record rec, int rowNumber) throws Exception {
String da_filename = getFilename(rec);
String was_derived_from = getWasDerivedFrom(rec);
List<String> derivedList = new ArrayList<String>();
StringTokenizer st = new StringTokenizer(was_derived_from, ",");
while (st.hasMoreTokens()) {
derivedList.add(st.nextToken().trim());
}
DataFile da = DataFile.findByNameAndStatus(da_filename, DataFile.PROCESSED);
if (da == null) {
dataFile.getLogger().printWarningByIdWithArgs("DOI_00007",da_filename);
} else {
da.setWasDerivedFrom(derivedList);
da.save();
dataFile.getLogger().println("DOI ingestion: updating DOI for [" + da_filename + "]");
}
return null;
}

@Override
public String getTableName() {
return "DOI";
}

@Override
public String getErrorMsg(Exception e) {
return "Error in DOIGenerator: " + e.getMessage();
}
}

18 changes: 18 additions & 0 deletions app/org/hadatac/data/loader/DataContext.java
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,15 @@ public static Long playTotalDataAcquisitions() {
return data.totalDataAcquisitions();
}

public static Long playTotalDataFiles() {
DataContext data = new DataContext( "user",
"password",
ConfigFactory.load().getString("hadatac.solr.data"),
false);

return data.totalDataFiles();
}

private Long totalDocuments(String solrCoreName) {
SolrClient solr = new HttpSolrClient.Builder(kbURL + solrCoreName).build();
SolrQuery parameters = new SolrQuery();
Expand Down Expand Up @@ -79,6 +88,10 @@ public Long totalDataAcquisitions() {
return totalDocuments(CollectionUtil.getCollectionName(Collection.DATA_COLLECTION.get()));
}

public Long totalDataFiles() {
return totalDocuments(CollectionUtil.getCollectionName(Collection.CSV_DATASET.get()));
}

private String cleanAllDocuments(int mode, Collection solrCoreName) {
String message = "";
String straux = "";
Expand Down Expand Up @@ -136,6 +149,7 @@ private String cleanSpecifiedStudy(int mode, String studyURI) {

String url1;
String url2;

try {
url1 = CollectionUtil.getCollectionPath(solrCoreName) + "/update?stream.body=" + URLEncoder.encode(query1, "UTF-8");
url2 = CollectionUtil.getCollectionPath(solrCoreName) + "/update?stream.body=" + URLEncoder.encode(query2, "UTF-8");
Expand Down Expand Up @@ -184,6 +198,10 @@ public String cleanAcquisitionData(int mode) {
return cleanAllDocuments(mode, Collection.DATA_ACQUISITION);
}

public String cleanDataFiles(int mode) {
return cleanAllDocuments(mode, Collection.CSV_DATASET);
}

public String cleanStudy(int mode, String studyURI) {
return cleanSpecifiedStudy(mode, studyURI);
}
Expand Down
Loading

0 comments on commit 0189360

Please sign in to comment.