diff --git a/.travis.yml b/.travis.yml index ee0cd75..f3ff1bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,4 @@ sudo: required -dist: trusty language: java jdk: - openjdk11 @@ -9,9 +8,7 @@ branches: - master notifications: slack: - on_success: change - on_failure: always - secure: xSIYhhwtzd+DV5oHEiT3ZVVCKsVlabQ7DMjx6xrq5XZtNBlsLSq6FhHJY37rbfI//6jv2PPZS2OUgNvhmDeiCrWbN6cIXZWkOVHcizfkKLvENLwnqvY+RU4gbwRvTfYEs+WPsl7iRRIahjKHGJHNuSnPSOwQ68eMr60WCSvakRX3ZOA0NgDW6v+MwUL9oTK25ArPhd0D98q5PVuvHmPTK/nc/CHjAJKIhrxqtgMb2PXYwX1YgEnHxbVarbVUqJsJ1Wi0t5PWvf6nLGPSp97f5mt7E68G+J+i18gGh9d/LWSpz8b1qOkxqB7xZwFOmp2Z0OyCNP0PeXmEKozdXMaEa5oi1eeIbot1VjjWrX9eL8b4JGLCR7cTLwZ1Zw7mZi3vTVSyqe9UEBZgGQAkixC0Y93TBRgstsv5/wDu2M4tIn68vKtCFRBnViJmau5UikYl0EukzzLB+LuJLZ2HgkhngwTxMdNIpXVDNRvMgRXMBfKYUmIneGNEAS2DJHlm6LDkGUpD4nBVJs1JJ07lRv/w9ithypLHgfOIWWtthr3JLuq0jvjJaVT91vxGcrUhLsXYnQbcy2l5yDXOLi+PA9UY0O/51m2t2hNojsZg2Eu0dNSipH7wZgGI7peO5hqAEx4F5IqlWBopSfHVd2M3CpyRp6yMMGbvZam1zE5RSg093F0= + secure: H9IzhdrqF6O+SS4gy/U1jL9WW4kvGFKW/PbTWLd11zI79oz3j+bTnX9E+Dev8ZigCMdd7BTanepE8kPruKT642MKIIhV5EArLLpSFiOXIZ4jmAlD8EoJ5xOf1+Z6cIW1TpkZSYp8GnZjlbRjUIQQvc5lXRcoiD26dz6MbvfWtZTcu1o+n+QGXDRjsg/XDNddAB3GEXXCZc+JJLBsGB+cQ5Ae0ph/rIVbRKByNSg86qrIYVd0D9OnxRXbahR3v+k3xcAFVXT6nrL6Jbw1IY8ARECVOxoxBPVeIJEaFKHor5cd3doapQ8IJjkT1aKb0XTtDXsutjLJdiV0opdeNDvdC8AUxQ8fFxGYS464o5wON//LsaQqLhyfat1ULZ/8y7+QH239FDOFnM/MuLD91OyK9wDK/HsA7n772srJRSDUMoBQvihf68/dDdsSZlSnjYyDMPxiKWCreFMnKCjiBrYvTKiDLpsQgUjz3M9t5PCSqw8Y7agKIRkfVnN6UHn48HY835ZvFDkJI4+/F+HCa9hkGyVxepXN1mnm30NBLcL3FSg9/51WhYPEw3V6izrO0qX/4pRyQswFiqkzOnJf6dA2csMkmayYGh2qkeOINi3EP4bJdURSHTGCb14FEwGnGyiMYS0b82FowyqugvqgmSXPmPL2S7ReztIn9BHv13DawHU= install: "[ ${TRAVIS_PULL_REQUEST} = 'false' ] && mvn -DskipTests=false clean deploy --settings .m2/travis-settings.xml" script: "[ ${TRAVIS_PULL_REQUEST} = 'false' ] && mvn -DskipTests=true clean --settings @@ -20,12 +17,6 @@ env: global: - CEDAR_HOST=metadatacenter.orgx - CEDAR_ADMIN_USER_API_KEY=1234 - - CEDAR_NCBI_SRA_FTP_HOST=ftpHost - - CEDAR_NCBI_SRA_FTP_USER=ftpUser - - CEDAR_NCBI_SRA_FTP_PASSWORD=ftpPassword - - CEDAR_NCBI_SRA_FTP_DIRECTORY=ftpDirectory - - CEDAR_IMMPORT_SUBMISSION_USER=submissionUser - - CEDAR_IMMPORT_SUBMISSION_PASSWORD=submissionPassword - CEDAR_NEO4J_HOST=127.0.0.1 - CEDAR_NEO4J_BOLT_PORT=7687 - CEDAR_NEO4J_USER_NAME=neo4j @@ -34,9 +25,8 @@ env: - CEDAR_MONGO_APP_USER_PASSWORD=password - CEDAR_REDIS_PERSISTENT_HOST=127.0.0.1 - CEDAR_REDIS_PERSISTENT_PORT=6379 - - CEDAR_SUBMISSION_HTTP_PORT=9010 - - CEDAR_SUBMISSION_ADMIN_PORT=9110 - - CEDAR_SUBMISSION_STOP_PORT=9210 - - CEDAR_MESSAGING_HTTP_PORT=9012 - - secure: BNTqaXeFugZh15ZvfViVAn4FlM4Ux+X4sKEoam6OADn8WHh2gpcd/akDhMMW3ztTwFXM3Ytqir1f7YkldV0OC3N+1bzjG6zzq9Svc/m14QIZGDXhp/1JMYUFgvTsM4VUa6t0Xz6qYL3TSE6eccGmEOYM5XVizDGjXvc58CdOi7DLuD2uS4CIddOe6iZV2G4v24era6TTWa+Nmru750qawBwvMncTI5BaY7WRa0GQ3MizE4PbNOt8EI2d451fNp1HopsQks162gELPgoPLLk5S0sGrq/XV+ofrhTa9xduvcsiAdzios4hLS9BDMHp2Yocf9aM3T2amFHKlZ8eTAsolsJmr3S0FqYiJUoSgHEuj0OzF4BJg+M5n98Atw8SnQmLJrEJ9MbnwiW/CI5RXlNEXU56gFfU1ttBoVGnILC+PS11nY3rUiMRLOsi5U4Genn0gupyTQH4XSd1mavkgYnQFwiV1nHqFBGixOXGsLE5BgaQtKCCrUEXFiSIHvwULZzhT6+7yBUmOZY8JUPoFUCYsOvL6Ux9ed875lDFNIEiRVe9Ge1xUGR1GZmG4dGkIxAJKCL56Wbxv4eBJzzgTB/WyRL01GdHzgei+TmT/hQwohGzcivnwrqIMCIZETLtLy4ufSzaNUCqMzVGKwpiSjTyqrCbSSkFa3QM6hHDXgZBH48= - - secure: JancKdf9FDgYESY2bLfRrf+HRiw12c2nWNGz9i1VqjEA2LzBgXziagEz8gzBDTERk2hBBrPRQv0me6r+0bWxtsWC3izFpxfExZ+MID4NkdJev1CGtME+kZaNmkXCY1LjstEKOP6LZFjS/+Nn0CwEhpTBEG0uQ7iGz8qhbRdPFtJG+xo2hYgsKNyg+8nv/xRtk8RZkcuRUF5loVS1b/f7hyoqeIGAyhASE76GvNUmpr5EpHgf1FY8v1+Jo9vpl1QGGBGGZP/hZ4+OHzKB8O0+/g21iQ9AqJvk/gSbz4dz0dKLn3CeKtxCoauXje9jPNahBCJl1/H7wPSAZhd6hI8r/6pfeqO42w739KFdLlXANpPox3kk/QnH7IQcWBXyc0ivZArVuLPi8A9QhsOSJ70W/mzpHG/9FXUsWpDTj5dRr4eOQkXkZ7f5vxhbMZLxNap9imWFHeBg7z3YjidNLv5t5+eVkJzuQycNmIsXPrq/rlatvr22HnlA9wdD6IYH7fWMUnYgY/k1mHcz1iz4WiEPH6bCy+r//h5dNeP1WEQDDc7qJ0y1znTlZGz4AoZXqY3F9PuV3cGtuBTsjgxxk3QCbP7NMCFyV2mD/wjGdVvs0LQtiY9btQ8dz7qEINR4jUjDKztvrUdFuRHcmrDVHM8IqI8cP1U2snWOi/2PXf7Gor0= + - CEDAR_IMPEX_ADMIN_PORT=9108 + - CEDAR_IMPEX_HTTP_PORT=9008 + - CEDAR_IMPEX_STOP_PORT=9208 + - secure: ym800/x4WbBWpkafu869O3ua6uR5DYHS1JOp4a3Wads6hvAbNn81YyU6gN9hj9xZ3zIVNjUDlGB+f4d40Khu9P/+ZwBit7GHKKMdvfZwnXYTvpHZ0SEV1pykLlqwfjIP94+By5ad4lt3DTRciySP8Vnu3qGrNpsfYAKWr3dw4rC2hagX/YU9OIO2Jt8rfi5qfbXFrDDiExeUqixhbBLICKu3yQAWVamsmXQf4pJsLHekTC65xdwmTMFE88p+qi5Wqs7utqg6tkaTH1rLyepFXVfvQdrqIxGSFU0cyQGvT/7H2OvrBnS/GrV/twSPXM2F4L2oSXeMQAr6eQjZmsTk6dqiUNzIbOWl8zozpOK9NoTxYtYgk9xpkSjLdicY68N3BhfLPSHU/FSAp7AdARjtOtN6k98MfRiHBPe1aMZiJpL6q7ihLExhscxTjqbvFr4bzy9jP7aqoJ57c4p/R9iCiEW4FT+o07+/RDG61NCYLL8CUy/VFUr07MaiTJR+QtI7YwxkCwjEtcgDMqBBBrc8EACIoMh8aN/9ZLkrYwRQNJuGp4xVO/ADun6SXesILnBiYjcEcxu7QOS3d+pgV+lA4xZwVOMdhJTZK4ZFp1wy1As4lzcyqbcXAVsvA+HPe23MYFRjRmJE8jma/QeN4y91v2meeCzKnFcRLpC66Wvfaaw= + - secure: uckHPikTTNCHU8Djf5mW5X/I1jkmXjfLS8qGrPLGeaZC6YUe18to2iJwBt2QNb47ToCBePzG86uP4L0nQFhbof9yi4WV++OsverVbpvKdPYru3HdCexomDFRxxvoOxgcaQjRkVkamyllqfNOMgo9/tpvuD//0y26H5vqCYYbopaR4KnF/f29JS56EbRWXQ7Tc8QbzH+8udm3M/lbusqw+cXqKrXd01eaXlpGB+OegDtt9TG5J7Dx7Nklv4aFWXAXD9n38fsKPG0o8umCmV+YiZNQpBxRCla2AP1/dGpaPUmQBqlIUyFHKvTyN5TnLFcQcw0wl4PdYWHeET/eEoKPPikOGuyZpIzOJyYJMx+nGMVvVcQheU30bpZL8RnNiHP4UGT5uvWgRXPNM7k1omBRsqR8eLAXdxhWiMyu8wKSh5n64BPwD0XYUjQYrxdlxXTyvAED1aQvdqT+JjDvi/y7sbj624tUZTVNMIYa9Bnuk1wSRUpMMik+cPxIHTOKnlOksO5rnccU9G0HfOkSIV6yB4tMpceObPuWKonjB2+sVoHRbED3SsL2FWdO5S5tlVwLmr0IZEgFuTUTA6y8kRvlrdGh9SKtem65rL7oIAa85LbsdXD5nDx8QxRV8X+nZXvOBdtWRzML7ItGtDrdk+LmyaOO6wqoY7iaHQynLh8LMFk= diff --git a/cedar-impex-server-application/src/main/java/org/metadatacenter/impex/resources/ImpexServerResource.java b/cedar-impex-server-application/src/main/java/org/metadatacenter/impex/resources/ImpexServerResource.java index 37623f7..d8c5fac 100644 --- a/cedar-impex-server-application/src/main/java/org/metadatacenter/impex/resources/ImpexServerResource.java +++ b/cedar-impex-server-application/src/main/java/org/metadatacenter/impex/resources/ImpexServerResource.java @@ -4,25 +4,29 @@ import com.fasterxml.jackson.databind.JsonNode; import org.apache.commons.fileupload.FileUploadException; import org.apache.commons.fileupload.servlet.ServletFileUpload; -import org.hibernate.validator.constraints.NotEmpty; import org.metadatacenter.cadsr.form.schema.Form; +import org.metadatacenter.cadsr.ingestor.form.FormParseResult; import org.metadatacenter.cadsr.ingestor.form.FormUtil; +import org.metadatacenter.cadsr.ingestor.util.CedarServerUtil; import org.metadatacenter.cadsr.ingestor.util.CedarServices; import org.metadatacenter.cadsr.ingestor.util.Constants; -import org.metadatacenter.cadsr.ingestor.util.GeneralUtil; import org.metadatacenter.cedar.util.dw.CedarMicroserviceResource; import org.metadatacenter.config.CedarConfig; import org.metadatacenter.exception.CedarException; import org.metadatacenter.impex.exception.UploadInstanceNotFoundException; import org.metadatacenter.impex.imp.cadsr.CadsrImportStatus; import org.metadatacenter.impex.imp.cadsr.CadsrImportStatusManager; +import org.metadatacenter.impex.imp.cadsr.CadsrImportStatusManager.ImportStatus; import org.metadatacenter.impex.upload.FlowData; import org.metadatacenter.impex.upload.FlowUploadUtil; import org.metadatacenter.impex.upload.UploadManager; +import org.metadatacenter.impex.util.ImpexUtil; import org.metadatacenter.rest.context.CedarRequestContext; +import org.metadatacenter.util.http.CedarResponse; import org.metadatacenter.util.json.JsonMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.xml.sax.SAXParseException; import javax.ws.rs.*; import javax.ws.rs.core.MediaType; @@ -30,7 +34,6 @@ import javax.xml.bind.JAXBException; import java.io.FileInputStream; import java.io.IOException; -import java.util.Map; import static org.metadatacenter.rest.assertion.GenericAssertions.LoggedIn; @@ -44,35 +47,44 @@ public ImpexServerResource(CedarConfig cedarConfig) { super(cedarConfig); } + /** + * Used to upload and import caDSR forms into CEDAR. The import is executed asynchronously in an independent thread. + * The client doesn't need to wait for the import task to be completed but it's responsible for using the + * import-cadsr-forms-status endpoint to check the import status. + *

+ * If folder Id is not provided, the imported forms will be saved into the user's home folder + * + * @return + * @throws CedarException + */ @POST @Timed @Path("/import-cadsr-forms") @Consumes(MediaType.MULTIPART_FORM_DATA) - public Response importCadsrForm() throws CedarException { + public Response importCadsrForm(@QueryParam("folderId") String folderId) throws CedarException { CedarRequestContext c = buildRequestContext(); c.must(c.user()).be(LoggedIn); - // Check that this is a file upload request + // Check that it's a file upload request if (ServletFileUpload.isMultipartContent(request)) { try { String userId = c.getCedarUser().getId(); // Extract data from the request FlowData data = FlowUploadUtil.getFlowData(request); - // Every request contains a file chunk that we will save in the appropriate position of a local file - // TODO: read base folder name from constants file + // Every request contains a file chunk that we will save to the appropriate position of a local file String submissionLocalFolderPath = FlowUploadUtil .getUploadLocalFolderPath("impex-upload", userId, data.getUploadId()); - String filePath = FlowUploadUtil - .saveToLocalFile(data, userId, request.getContentLength(), submissionLocalFolderPath); - //logger.info("Saving file chunk to: " + filePath); + FlowUploadUtil.saveToLocalFile(data, userId, request.getContentLength(), submissionLocalFolderPath); // Update the submission upload status UploadManager.getInstance().updateStatus(data, submissionLocalFolderPath); // When the upload is complete, trigger the import process - if (UploadManager.getInstance().isUploadComplete(data.getUploadId())) { + if (UploadManager.getInstance().isUploadComplete(data.getUploadId()) + && !CadsrImportStatusManager.getInstance().exists(data.getUploadId())) { + logger.info("File(s) successfully uploaded to the Impex server: "); logger.info(" - Upload id: " + data.getUploadId()); logger.info(" - Local path: " + submissionLocalFolderPath); @@ -82,40 +94,67 @@ public Response importCadsrForm() throws CedarException { logger.info(" - " + fileName); } - //--start import-- - String cedarFolderId = c.getCedarUser().getHomeFolderId(); - - CadsrImportStatusManager.getInstance().addStatus(data.getUploadId(), cedarFolderId); - - // Import files into CEDAR - for (String formFilePath : UploadManager.getInstance().getUploadFilePaths(data.getUploadId())) { - logger.info("Importing file: " + formFilePath); - Form form = FormUtil.getForm(new FileInputStream(formFilePath)); - Map templateMap = FormUtil.getTemplateMapFromForm(form); - - - // TODO: cedarConfig.getHost instead of using CedarEnvironment - - String apiKey = c.getCedarUser().getFirstActiveApiKey(); - CedarServices.createTemplate(templateMap, cedarFolderId, Constants.CedarEnvironment.LOCAL, apiKey); - System.out.println(GeneralUtil.convertMapToJson(templateMap)); - } - - //--end import- - - // Remove the submission from the status map - UploadManager.getInstance().removeUploadStatus(data.getUploadId()); + // Import files into CEDAR asynchronously + new Thread(() -> { + String fileName = null; + try { + String cedarFolderId = folderId != null ? folderId : c.getCedarUser().getHomeFolderId(); + // Set import status to 'PENDING' for all the files that are part of the upload + CadsrImportStatusManager.getInstance().initImportStatus(data.getUploadId(), cedarFolderId); + + for (String formFilePath : UploadManager.getInstance().getUploadFilePaths(data.getUploadId())) { + + try { + // Set status to IN_PROGRESS + fileName = ImpexUtil.getFileNameFromFilePath(formFilePath); + CadsrImportStatusManager.getInstance().setStatus(data.getUploadId(), fileName, + ImportStatus.IN_PROGRESS); + logger.info("Importing file: " + formFilePath); + // Translate form to CEDAR template + Form form = FormUtil.getForm(new FileInputStream(formFilePath)); + FormParseResult parseResult = FormUtil.getTemplateMapFromForm(form, + data.getUploadId() + "_" + fileName); + //logger.info(JsonMapper.MAPPER.writeValueAsString(parseResult.getTemplateMap())); + CadsrImportStatusManager.getInstance().writeReportMessages(data.uploadId, fileName, + parseResult.getReportMessages()); + // Upload template to CEDAR + Constants.CedarServer cedarServer = CedarServerUtil.toCedarServerFromHostName(cedarConfig.getHost()); + String apiKey = c.getCedarUser().getFirstActiveApiKey(); + CedarServices.createTemplate(parseResult.getTemplateMap(), cedarFolderId, cedarServer, apiKey); + // Set status to COMPLETE + CadsrImportStatusManager.getInstance().setStatus(data.getUploadId(), fileName, ImportStatus.COMPLETE); + } catch (JAXBException e) { + CadsrImportStatusManager.getInstance().writeReportMessage(data.uploadId, fileName, "Parsing error. " + + "Please check that the input file contains a valid XML caDSR form"); + logger.error("Error parsing input file"); + CadsrImportStatusManager.getInstance().setStatus(data.getUploadId(), fileName, ImportStatus.ERROR); + } catch (IOException | RuntimeException e) { + CadsrImportStatusManager.getInstance().writeReportMessage(data.uploadId, fileName, "Error importing form"); + logger.error("Error importing form: " + e.getMessage()); + CadsrImportStatusManager.getInstance().setStatus(data.getUploadId(), fileName, ImportStatus.ERROR); + } + } + // Remove the upload from the status map + UploadManager.getInstance().removeUploadStatus(data.getUploadId()); + } catch (UploadInstanceNotFoundException e) { + logger.error("Upload instance not found: " + e.getMessage()); + } + }).start(); } - - } catch (IOException | FileUploadException /*SubmissionInstanceNotFoundException*/ e) { - logger.error(e.getMessage(), e); - return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); + } catch (FileUploadException e) { + return CedarResponse.internalServerError() + .errorMessage("Error uploading file") + .exception(e).build(); } catch (IllegalAccessException e) { - e.printStackTrace(); + return CedarResponse.internalServerError() + .exception(e).build(); } catch (UploadInstanceNotFoundException e) { - e.printStackTrace(); - } catch (JAXBException e) { - e.printStackTrace(); + return CedarResponse.internalServerError() + .errorMessage("Upload Id not found") + .exception(e).build(); + } catch (IOException e) { + return CedarResponse.internalServerError() + .exception(e).build(); } return Response.ok().build(); } else { @@ -123,25 +162,52 @@ public Response importCadsrForm() throws CedarException { } } + /* In production I may want to this method instead of the method below (it doesn't expose all uploads) */ +// @GET +// @Timed +// @Path("/import-cadsr-forms-status") +// public Response importStatus(@QueryParam("uploadId") @NotEmpty String uploadId) throws CedarException { +// +// CedarRequestContext c = buildRequestContext(); +// c.must(c.user()).be(LoggedIn); +// +// try { +// if (!CadsrImportStatusManager.getInstance().exists(uploadId)) { +// return CedarResponse.notFound().errorMessage("The specified uploadId cannot be found").id(uploadId).build(); +// } else { +// CadsrImportStatus status = CadsrImportStatusManager.getInstance().getStatus(uploadId); +// JsonNode output = JsonMapper.MAPPER.valueToTree(status); +// return Response.ok().entity(output).build(); +// } +// } catch (Exception e) { +// return CedarResponse.internalServerError().exception(e).build(); +// } +// } + @GET @Timed @Path("/import-cadsr-forms-status") - public Response importStatus(@QueryParam("uploadId") @NotEmpty String uploadId) throws CedarException { + public Response importStatus(@QueryParam("uploadId") String uploadId) throws CedarException { CedarRequestContext c = buildRequestContext(); c.must(c.user()).be(LoggedIn); try { - CadsrImportStatus status = CadsrImportStatusManager.getInstance().getStatus(uploadId); - JsonNode output = JsonMapper.MAPPER.valueToTree(status); - return Response.ok().entity(output).build(); - } - catch (Exception e) { // TODO: refine exception - return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); + if (uploadId != null) { + if (!CadsrImportStatusManager.getInstance().exists(uploadId)) { + return CedarResponse.notFound().errorMessage("The specified uploadId cannot be found").id(uploadId).build(); + } else { + CadsrImportStatus status = CadsrImportStatusManager.getInstance().getStatus(uploadId); + JsonNode output = JsonMapper.MAPPER.valueToTree(status); + return Response.ok().entity(output).build(); + } + } else { + JsonNode output = JsonMapper.MAPPER.valueToTree(CadsrImportStatusManager.getInstance().getAllStatuses()); + return Response.ok().entity(output).build(); + } + } catch (Exception e) { + return CedarResponse.internalServerError().exception(e).build(); } - } - - } diff --git a/cedar-impex-server-application/src/main/java/org/metadatacenter/impex/util/Constants.java b/cedar-impex-server-application/src/main/java/org/metadatacenter/impex/util/Constants.java new file mode 100644 index 0000000..3fe717e --- /dev/null +++ b/cedar-impex-server-application/src/main/java/org/metadatacenter/impex/util/Constants.java @@ -0,0 +1,10 @@ +package org.metadatacenter.impex.util; + +import org.metadatacenter.model.BiboStatus; + +public final class Constants { + + public static final String UPLOAD_FOLDER_NAME = "impex-upload"; + + +} diff --git a/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/imp/cadsr/CadsrFileImportStatus.java b/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/imp/cadsr/CadsrFileImportStatus.java new file mode 100644 index 0000000..4f144b1 --- /dev/null +++ b/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/imp/cadsr/CadsrFileImportStatus.java @@ -0,0 +1,51 @@ +package org.metadatacenter.impex.imp.cadsr; + +import java.time.LocalTime; +import java.util.Date; + +public class CadsrFileImportStatus { + + private String fileName; + private CadsrImportStatusManager.ImportStatus importStatus; + private LocalTime statusTime; + private String report; + + public CadsrFileImportStatus(String fileName, CadsrImportStatusManager.ImportStatus importStatus, LocalTime statusTime, String report) { + this.fileName = fileName; + this.importStatus = importStatus; + this.statusTime = statusTime; + this.report = report; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public CadsrImportStatusManager.ImportStatus getImportStatus() { + return importStatus; + } + + public void setImportStatus(CadsrImportStatusManager.ImportStatus importStatus) { + this.importStatus = importStatus; + } + + public LocalTime getStatusTime() { + return statusTime; + } + + public void setStatusTime(LocalTime statusTime) { + this.statusTime = statusTime; + } + + public String getReport() { + return report; + } + + public void setReport(String report) { + this.report = report; + } +} diff --git a/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/imp/cadsr/CadsrFormImportStatus.java b/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/imp/cadsr/CadsrFormImportStatus.java deleted file mode 100644 index 094bfda..0000000 --- a/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/imp/cadsr/CadsrFormImportStatus.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.metadatacenter.impex.imp.cadsr; - -import org.metadatacenter.impex.imp.cadsr.CadsrImportStatusManager.ImportStatus; - -public class CadsrFormImportStatus { - - private ImportStatus status; - private String statusMessage; - - public CadsrFormImportStatus(ImportStatus status) { - this.status = status; - } - - public ImportStatus getStatus() { - return status; - } - - public void setStatus(ImportStatus status) { - this.status = status; - } - - public String getStatusMessage() { - return statusMessage; - } - - public void setStatusMessage(String statusMessage) { - this.statusMessage = statusMessage; - } -} diff --git a/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/imp/cadsr/CadsrImportStatus.java b/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/imp/cadsr/CadsrImportStatus.java index 0b107f6..7d44e9f 100644 --- a/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/imp/cadsr/CadsrImportStatus.java +++ b/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/imp/cadsr/CadsrImportStatus.java @@ -5,13 +5,13 @@ public class CadsrImportStatus { private String uploadId; - private Map formsImportStatus; // the String stores the file name (e.g., form1.xml) + private Map filesImportStatus; // the String stores the file name (e.g., form1.xml) private String destinationCedarFolderId; - public CadsrImportStatus(String uploadId, Map formsImportStatus, + public CadsrImportStatus(String uploadId, Map filesImportStatus, String destinationCedarFolderId) { this.uploadId = uploadId; - this.formsImportStatus = formsImportStatus; + this.filesImportStatus = filesImportStatus; this.destinationCedarFolderId = destinationCedarFolderId; } @@ -19,12 +19,12 @@ public CadsrImportStatus(String uploadId, Map for public void setUploadId(String uploadId) { this.uploadId = uploadId; } - public Map getFormsImportStatus() { - return formsImportStatus; + public Map getFilesImportStatus() { + return filesImportStatus; } - public void setFormsImportStatus(Map formsImportStatus) { - this.formsImportStatus = formsImportStatus; + public void setFilesImportStatus(Map filesImportStatus) { + this.filesImportStatus = filesImportStatus; } public String getDestinationCedarFolderId() { diff --git a/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/imp/cadsr/CadsrImportStatusManager.java b/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/imp/cadsr/CadsrImportStatusManager.java index 643c462..c3720d3 100644 --- a/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/imp/cadsr/CadsrImportStatusManager.java +++ b/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/imp/cadsr/CadsrImportStatusManager.java @@ -3,23 +3,46 @@ import org.metadatacenter.impex.upload.FileUploadStatus; import org.metadatacenter.impex.upload.UploadManager; import org.metadatacenter.impex.upload.UploadStatus; +import org.metadatacenter.impex.util.ImpexUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; public class CadsrImportStatusManager { - enum ImportStatus { + final static Logger logger = LoggerFactory.getLogger(CadsrImportStatusManager.class); + + public enum ImportStatus { PENDING, - SUCCESS, + IN_PROGRESS, + COMPLETE, ERROR } + // Imports older than this threshold will be removed from the map + private final long CLEAN_THRESHOLD_1_MINUTES = 10; // Main threshold, used to clean completed imports + private final long CLEAN_THRESHOLD_2_MINUTES = 60; // Much longer, to avoid keeping errored imports in the map + // Frequency used to check if there are any imports that can be removed from the map + private final long CLEAN_DELAY_MINUTES = 10; + private static CadsrImportStatusManager singleInstance; - private Map importStatus = new HashMap<>(); // The String stores the uploadId + private Map importStatus; // uploadId -> CadsrImportStatus + private ScheduledExecutorService executor; // Single instance private CadsrImportStatusManager() { + importStatus = new HashMap<>(); + initUploadsCleanerExecutor(); } public static synchronized CadsrImportStatusManager getInstance() { @@ -29,21 +52,157 @@ public static synchronized CadsrImportStatusManager getInstance() { return singleInstance; } - // Generate import status from upload status - public synchronized void addStatus(String uploadId, String destinationCedarFolderId) { - UploadStatus uploadStatus = UploadManager.getInstance().getUploadStatus(uploadId); - Map formsImportStatus = new HashMap<>(); + /** + * Uses an executor to clear the older items in the importStatus map periodically + */ + public void initUploadsCleanerExecutor() { + executor = Executors.newSingleThreadScheduledExecutor(); + Runnable cleanTask = () -> { + logger.info("Checking if there are any old imports that can be removed (map size: " + importStatus.size() + ")"); + List uploadIdsToBeRemoved = new ArrayList<>(); + for (CadsrImportStatus importStatus : importStatus.values()) { + int countMeetsConditions = 0; + for (CadsrFileImportStatus fileStatus : importStatus.getFilesImportStatus().values()) { + // Checks if all the files that belong to the particular upload are older than the given threshold. If any + // of them is more recent than the threshold, we don't remove the uploadId from the map + if (((fileStatus.getImportStatus() == ImportStatus.COMPLETE || fileStatus.getImportStatus() == ImportStatus.ERROR) && + fileStatus.getStatusTime().plusMinutes(CLEAN_THRESHOLD_1_MINUTES).isBefore(LocalTime.now())) || + ((fileStatus.getImportStatus() == ImportStatus.COMPLETE || fileStatus.getImportStatus() == ImportStatus.ERROR) && + fileStatus.getStatusTime().plusMinutes(CLEAN_THRESHOLD_2_MINUTES).isBefore(LocalTime.now()))){ + countMeetsConditions++; + } + else { + break; + } + } + // Remove from map if all the files are older than one of the thresholds + if (countMeetsConditions == importStatus.getFilesImportStatus().size()) { + uploadIdsToBeRemoved.add(importStatus.getUploadId()); + } + } + for (String uploadId : uploadIdsToBeRemoved) { + if (importStatus.containsKey(uploadId)) { + importStatus.remove(uploadId); + logger.info("UploadId removed: " + uploadId + ". Updated map size: " + importStatus.size()); + } + } + }; + executor.scheduleWithFixedDelay(cleanTask, 0, CLEAN_DELAY_MINUTES, TimeUnit.MINUTES); + } + + public CadsrImportStatus getStatus(String uploadId) { + return importStatus.get(uploadId); + } + + public Map getAllStatuses() { + return importStatus; + } + /** + * Adds the upload information to the map and sets the import status to PENDING for all the forms + * @param uploadId + */ + public synchronized void initImportStatus(String uploadId, String destinationCedarFolderId) { + // Get the file names from the UploadManager + UploadStatus uploadStatus = UploadManager.getInstance().getUploadStatus(uploadId); + Map filesImportStatus = new HashMap<>(); for (FileUploadStatus fileUploadStatus : uploadStatus.getFilesUploadStatus().values()) { - String fileName = fileUploadStatus.getFileLocalPath().substring(fileUploadStatus.getFileLocalPath().lastIndexOf("/") + 1); - formsImportStatus.put(fileName, new CadsrFormImportStatus(ImportStatus.PENDING)); + String fileName = ImpexUtil.getFileNameFromFilePath(fileUploadStatus.getFileLocalPath()); + filesImportStatus.put(fileName, new CadsrFileImportStatus(fileName, ImportStatus.PENDING, LocalTime.now(), "")); + } + importStatus.put(uploadId, new CadsrImportStatus(uploadId, filesImportStatus, destinationCedarFolderId)); + } + + /** + * Set the import status for a particular file, associated to a given uploadId. This method can only be used to set + * the status to IN_PROGRESS or to COMPLETE. In order to initialize the status to PENDING, use initImportStatus. + * @param uploadId + * @param fileName + * @param status + */ + public synchronized void setStatus(String uploadId, String fileName, ImportStatus status) { + if (!importStatus.containsKey(uploadId)) { + throw new IllegalArgumentException("uploadId not found: " + uploadId); + } + if (!importStatus.get(uploadId).getFilesImportStatus().containsKey(fileName)) { + throw new IllegalArgumentException("fileName not found: " + fileName); + } + + CadsrFileImportStatus fis = importStatus.get(uploadId).getFilesImportStatus().get(fileName); + fis.setImportStatus(status); + + Map fisMap = importStatus.get(uploadId).getFilesImportStatus(); + fisMap.put(fileName, fis); + + CadsrImportStatus is = importStatus.get(uploadId); + is.setFilesImportStatus(fisMap); + + importStatus.replace(uploadId, is); + + } + + private synchronized void removeImportStatus(String uploadId) { + importStatus.remove(uploadId); + } + + public boolean exists(String uploadId) { + return importStatus.containsKey(uploadId); + } + + /** + * Writes multiple lines to the report + * @param uploadId + * @param fileName + * @param messages + */ + public void writeReportMessages(String uploadId, String fileName, List messages) { + for (String message : messages) { + writeReportMessage(uploadId, fileName, message); } + } - importStatus.put(uploadId, new CadsrImportStatus(uploadId, formsImportStatus, destinationCedarFolderId)); + /** + * Writes a new line to the report + * @param uploadId + * @param fileName + * @param message + */ + public void writeReportMessage(String uploadId, String fileName, String message) { + writeReportMessage(uploadId, fileName, message, false); } - public CadsrImportStatus getStatus(String uploadId) { - return importStatus.get(uploadId); + public void writeReportMessage(String uploadId, String fileName, String message, boolean includeDateTime) { + + if (importStatus.containsKey(uploadId)) { + + if (importStatus.get(uploadId).getFilesImportStatus().containsKey(fileName)) { + String dateTime = ""; + if (includeDateTime) { + DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss"); + dateTime = "[" + dtf.format(LocalDateTime.now()) + "] "; + } + String currentReport = importStatus.get(uploadId).getFilesImportStatus().get(fileName).getReport(); + String lineSeparator = currentReport.length() > 0 ? System.getProperty("line.separator") : ""; + String newReport = currentReport + lineSeparator + dateTime + message; + + CadsrFileImportStatus fis = importStatus.get(uploadId).getFilesImportStatus().get(fileName); + fis.setReport(newReport); + + Map fisMap = importStatus.get(uploadId).getFilesImportStatus(); + fisMap.replace(fileName, fis); + + CadsrImportStatus is = importStatus.get(uploadId); + is.setFilesImportStatus(fisMap); + + importStatus.replace(uploadId, is); + } + else { + throw new IllegalArgumentException("FileName not found: " + fileName); + } + } + else { + throw new IllegalArgumentException("UploadId not found: " + uploadId); + } } } diff --git a/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/util/ImpexUtil.java b/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/util/ImpexUtil.java new file mode 100644 index 0000000..cf8e616 --- /dev/null +++ b/cedar-impex-server-core/src/main/java/org/metadatacenter/impex/util/ImpexUtil.java @@ -0,0 +1,9 @@ +package org.metadatacenter.impex.util; + +public class ImpexUtil { + + public static String getFileNameFromFilePath(String filePath) { + return filePath.substring(filePath.lastIndexOf("/") + 1); + } + +}