From 846e47e19cd0235dbea9738887ea0618f44a903f Mon Sep 17 00:00:00 2001
From: Sven F <9976560+sven1103@users.noreply.github.com>
Date: Mon, 3 Jun 2024 10:11:31 +0200
Subject: [PATCH] Introduces recursive copy to target ETL dropboxes (#25)
Since the target ETL dropbox might be on a different file system, a simple move wont work.
This PR introduces the commons-io lib from Apache, that provides a simple FileUtils class with directory copy.
After a successful copy and marker file creation, the task folder is deleted from the evaluation working directory.
---
pom.xml | 7 ++++
.../evaluation/EvaluationRequest.java | 33 ++++++++++++-------
2 files changed, 29 insertions(+), 11 deletions(-)
diff --git a/pom.xml b/pom.xml
index 30d4993..2f87307 100644
--- a/pom.xml
+++ b/pom.xml
@@ -60,6 +60,13 @@
jackson-databind
2.17.0
+
+
+ commons-io
+ commons-io
+ 2.16.1
+
+
diff --git a/src/main/java/life/qbic/data/processing/evaluation/EvaluationRequest.java b/src/main/java/life/qbic/data/processing/evaluation/EvaluationRequest.java
index 6601027..af0995e 100644
--- a/src/main/java/life/qbic/data/processing/evaluation/EvaluationRequest.java
+++ b/src/main/java/life/qbic/data/processing/evaluation/EvaluationRequest.java
@@ -19,6 +19,7 @@
import life.qbic.data.processing.Provenance;
import life.qbic.data.processing.Provenance.ProvenanceException;
import life.qbic.data.processing.config.RoundRobinDraw;
+import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.Logger;
/**
@@ -151,13 +152,25 @@ private void evaluateDirectory(File taskDir) {
LOG.error("Could not update provenance file: {}", taskDir.getAbsolutePath(), e);
moveToSystemIntervention(taskDir, e.getMessage());
}
- moveToTargetDir(taskDir);
+ try {
+ copyToTargetDir(taskDir);
+ } catch (IOException e) {
+ LOG.error("Could not copy to target directory: {}", taskDir.getAbsolutePath(), e);
+ moveToSystemIntervention(taskDir,
+ "Cannot copy task to target directory: %s".formatted(assignedTargetDirectory));
+ }
try {
createMarkerFile(assignedTargetDirectory, taskDir.getName());
} catch (IOException e) {
LOG.error("Could not create marker file in: {}", assignedTargetDirectory, e);
moveToSystemIntervention(taskDir, e.getMessage());
}
+ try {
+ cleanup(taskDir);
+ } catch (IOException e) {
+ LOG.error("Could not clean up task directory: {}", taskDir.getAbsolutePath(), e);
+ moveToSystemIntervention(taskDir, e.getMessage());
+ }
return;
}
var errorMessage = ErrorSummary.createSimple(taskDir.getName(),
@@ -170,6 +183,11 @@ private void evaluateDirectory(File taskDir) {
moveBackToOrigin(taskDir, provenance, errorMessage.toString());
}
+ private void cleanup(File taskDir) throws IOException {
+ LOG.info("Deleting task directory: {}", taskDir.getAbsolutePath());
+ FileUtils.deleteDirectory(taskDir);
+ }
+
private void updateProvenanceFile(File provenanceFile, Provenance provenance) throws IOException {
var mapper = new ObjectMapper();
mapper.writerWithDefaultPrettyPrinter().writeValue(provenanceFile, provenance);
@@ -209,18 +227,11 @@ private void moveBackToOrigin(File taskDir, Provenance provenance, String reason
}
}
- private void moveToTargetDir(File taskDir) {
+ private void copyToTargetDir(File taskDir) throws IOException {
LOG.info(
- "Moving %s to target directory %s".formatted(taskDir.getAbsolutePath(),
+ "Copying %s to target directory %s".formatted(taskDir.getAbsolutePath(),
assignedTargetDirectory));
- try {
- Files.move(taskDir.toPath(), assignedTargetDirectory.resolve(taskDir.getName()));
- } catch (IOException e) {
- LOG.error("Cannot move task to target directory: %s".formatted(assignedTargetDirectory), e);
- moveToSystemIntervention(taskDir,
- "Cannot move task to target directory: %s".formatted(assignedTargetDirectory));
- }
-
+ FileUtils.copyDirectory(taskDir, assignedTargetDirectory.resolve(taskDir.getName()).toFile());
}
private List tasks() {