Introduces recursive copy to target ETL dropboxes (#25)

Since the target ETL dropbox might be on a different file system, a simple move wont work. This PR introduces the commons-io lib from Apache, that provides a simple FileUtils class with directory copy. After a successful copy and marker file creation, the task folder is deleted from the evaluation working directory.
qbicsoftware · Jun 3, 2024 · 846e47e · 846e47e
1 parent 7a5dd2d
commit 846e47e
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 11 deletions.
diff --git a/pom.xml b/pom.xml
@@ -60,6 +60,13 @@
       <artifactId>jackson-databind</artifactId>
       <version>2.17.0</version>
     </dependency>
+
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>2.16.1</version>
+    </dependency>
+
   </dependencies>
   <dependencyManagement>
     <dependencies>

diff --git a/src/main/java/life/qbic/data/processing/evaluation/EvaluationRequest.java b/src/main/java/life/qbic/data/processing/evaluation/EvaluationRequest.java
@@ -19,6 +19,7 @@
 import life.qbic.data.processing.Provenance;
 import life.qbic.data.processing.Provenance.ProvenanceException;
 import life.qbic.data.processing.config.RoundRobinDraw;
+import org.apache.commons.io.FileUtils;
 import org.apache.logging.log4j.Logger;
 
 /**
@@ -151,13 +152,25 @@ private void evaluateDirectory(File taskDir) {
         LOG.error("Could not update provenance file: {}", taskDir.getAbsolutePath(), e);
         moveToSystemIntervention(taskDir, e.getMessage());
       }
-      moveToTargetDir(taskDir);
+      try {
+        copyToTargetDir(taskDir);
+      } catch (IOException e) {
+        LOG.error("Could not copy to target directory: {}", taskDir.getAbsolutePath(), e);
+        moveToSystemIntervention(taskDir,
+            "Cannot copy task to target directory: %s".formatted(assignedTargetDirectory));
+      }
       try {
         createMarkerFile(assignedTargetDirectory, taskDir.getName());
       } catch (IOException e) {
         LOG.error("Could not create marker file in: {}", assignedTargetDirectory, e);
         moveToSystemIntervention(taskDir, e.getMessage());
       }
+      try {
+        cleanup(taskDir);
+      } catch (IOException e) {
+        LOG.error("Could not clean up task directory: {}", taskDir.getAbsolutePath(), e);
+        moveToSystemIntervention(taskDir, e.getMessage());
+      }
       return;
     }
     var errorMessage = ErrorSummary.createSimple(taskDir.getName(),
@@ -170,6 +183,11 @@ private void evaluateDirectory(File taskDir) {
     moveBackToOrigin(taskDir, provenance, errorMessage.toString());
   }
 
+  private void cleanup(File taskDir) throws IOException {
+    LOG.info("Deleting task directory: {}", taskDir.getAbsolutePath());
+    FileUtils.deleteDirectory(taskDir);
+  }
+
   private void updateProvenanceFile(File provenanceFile, Provenance provenance) throws IOException {
     var mapper = new ObjectMapper();
     mapper.writerWithDefaultPrettyPrinter().writeValue(provenanceFile, provenance);
@@ -209,18 +227,11 @@ private void moveBackToOrigin(File taskDir, Provenance provenance, String reason
     }
   }
 
-  private void moveToTargetDir(File taskDir) {
+  private void copyToTargetDir(File taskDir) throws IOException {
     LOG.info(
-        "Moving %s to target directory %s".formatted(taskDir.getAbsolutePath(),
+        "Copying %s to target directory %s".formatted(taskDir.getAbsolutePath(),
             assignedTargetDirectory));
-    try {
-      Files.move(taskDir.toPath(), assignedTargetDirectory.resolve(taskDir.getName()));
-    } catch (IOException e) {
-      LOG.error("Cannot move task to target directory: %s".formatted(assignedTargetDirectory), e);
-      moveToSystemIntervention(taskDir,
-          "Cannot move task to target directory: %s".formatted(assignedTargetDirectory));
-    }
-
+    FileUtils.copyDirectory(taskDir, assignedTargetDirectory.resolve(taskDir.getName()).toFile());
   }
 
   private List<File> tasks() {