Skip to content

Commit

Permalink
Prepare release 0.4.0
Browse files Browse the repository at this point in the history
Prepare release 0.4.0
  • Loading branch information
sven1103 authored May 31, 2024
2 parents 8f7bc62 + 7a5dd2d commit 12abdf0
Show file tree
Hide file tree
Showing 16 changed files with 171 additions and 74 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
package life.qbic.data.processing;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;

/**
* <b>Access and File Evaluation helper class</b>
*
* @since 1.0.0
*/
public class AccessRightsEvaluation {

/**
* Convenience method that checks if the application has write and executable permission for the
* given file.
*
* @param file the path of the file of interest to evaluate
* @throws IOException if one or both of the evaluated conditions are failing
* @since 1.0.0
*/
public static void evaluateWriteAndExecutablePermission(Path file) throws IOException {
evaluateWriteAndExecutablePermission(file.toFile());
}

/**
* Convenience method that checks if the application has write and executable permission for the
* given file.
*
* @param file the file of interest to evaluate
* @throws IOException if one or both of the evaluated conditions are failing
* @since 1.0.0
*/
public static void evaluateWriteAndExecutablePermission(File file) throws IOException {
if (!file.canWrite()) {
throw new IOException("Cannot write to file " + file);
}
if (!file.canExecute()) {
throw new IOException("Cannot execute file " + file);
}
}

/**
* Convenience method that checks if file exists and is a directory.
*
* @param file the path of the file of interest to evaluate
* @throws IOException if neither of the evaluated conditions are failing
* @since 1.0.0
*/
public static void evaluateExistenceAndDirectory(Path file) throws IOException {
evaluateExistenceAndDirectory(file.toFile());
}

/**
* Convenience method that checks if file exists and is a directory.
*
* @param file the path of the file of interest to evaluate
* @throws IOException if neither of the evaluated conditions are failing
* @since 1.0.0
*/
public static void evaluateExistenceAndDirectory(File file) throws IOException {
if (!file.exists()) {
throw new IOException("File does not exist " + file);
}
if (!file.isDirectory()) {
throw new IOException("File is not a directory " + file);
}
}


}
18 changes: 11 additions & 7 deletions src/main/java/life/qbic/data/processing/AppConfig.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package life.qbic.data.processing;

import java.io.IOException;
import java.nio.file.Path;
import java.util.Arrays;
import life.qbic.data.processing.config.EvaluationWorkersConfig;
Expand Down Expand Up @@ -28,7 +29,8 @@ class AppConfig {
@Bean
ScannerConfiguration scannerConfiguration(
@Value("${scanner.directory}") String scannerDirectory,
@Value("${scanner.interval}") int interval, @Value("${scanner.ignore}") String[] ignore) {
@Value("${scanner.interval}") int interval, @Value("${scanner.ignore}") String[] ignore)
throws IOException {
return new ScannerConfiguration(scannerDirectory, interval, ignore);
}

Expand All @@ -44,7 +46,7 @@ RegistrationWorkersConfig registrationWorkersConfig(

@Bean
RegistrationConfiguration registrationConfiguration(
RegistrationWorkersConfig registrationWorkersConfig) {
RegistrationWorkersConfig registrationWorkersConfig) throws IOException {
return new RegistrationConfiguration(registrationWorkersConfig.workingDirectory().toString(),
registrationWorkersConfig.targetDirectory().toString(),
registrationWorkersConfig.metadataFileName());
Expand All @@ -55,12 +57,13 @@ EvaluationWorkersConfig evaluationWorkersConfig(
@Value("${evaluation.threads}") int amountOfWorkers,
@Value("${evaluation.working.dir}") String workingDirectory,
@Value("${evaluation.target.dirs}") String[] targetDirectory) {
return new EvaluationWorkersConfig(amountOfWorkers, workingDirectory, Arrays.stream(targetDirectory).toList());
return new EvaluationWorkersConfig(amountOfWorkers, workingDirectory,
Arrays.stream(targetDirectory).toList());
}

@Bean
EvaluationConfiguration evaluationConfiguration(EvaluationWorkersConfig evaluationWorkersConfig,
GlobalConfig globalConfig) {
GlobalConfig globalConfig) throws IOException {
return new EvaluationConfiguration(evaluationWorkersConfig.workingDirectory().toString(),
evaluationWorkersConfig.targetDirectories(), globalConfig);
}
Expand All @@ -75,7 +78,8 @@ ProcessingWorkersConfig processingWorkersConfig(
}

@Bean
ProcessingConfiguration processingConfiguration(ProcessingWorkersConfig processingWorkersConfig) {
ProcessingConfiguration processingConfiguration(ProcessingWorkersConfig processingWorkersConfig)
throws IOException {
return new ProcessingConfiguration(processingWorkersConfig.workingDirectory(),
processingWorkersConfig.targetDirectory());
}
Expand All @@ -85,7 +89,7 @@ GlobalConfig globalConfig(
@Value("${users.error.directory.name}") String usersErrorDirectoryName,
@Value("${users.registration.directory.name}") String usersRegistrationDirectoryName,
@Value("${qbic.measurement-id.pattern}") String measurementIdPattern) {
return new GlobalConfig(usersErrorDirectoryName, usersRegistrationDirectoryName, measurementIdPattern);
return new GlobalConfig(usersErrorDirectoryName, usersRegistrationDirectoryName,
measurementIdPattern);
}

}
27 changes: 17 additions & 10 deletions src/main/java/life/qbic/data/processing/Application.java
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,18 @@ public static void main(String[] args) {
AppConfig.class);

ScannerConfiguration scannerConfiguration = context.getBean(ScannerConfiguration.class);
RegistrationWorkersConfig registrationWorkersConfig = context.getBean(RegistrationWorkersConfig.class);
RegistrationConfiguration registrationConfiguration = context.getBean(RegistrationConfiguration.class);
ProcessingWorkersConfig processingWorkersConfig = context.getBean(ProcessingWorkersConfig.class);
ProcessingConfiguration processingConfiguration = context.getBean(ProcessingConfiguration.class);
EvaluationWorkersConfig evaluationWorkersConfig = context.getBean(EvaluationWorkersConfig.class);
EvaluationConfiguration evaluationConfiguration = context.getBean(EvaluationConfiguration.class);
RegistrationWorkersConfig registrationWorkersConfig = context.getBean(
RegistrationWorkersConfig.class);
RegistrationConfiguration registrationConfiguration = context.getBean(
RegistrationConfiguration.class);
ProcessingWorkersConfig processingWorkersConfig = context.getBean(
ProcessingWorkersConfig.class);
ProcessingConfiguration processingConfiguration = context.getBean(
ProcessingConfiguration.class);
EvaluationWorkersConfig evaluationWorkersConfig = context.getBean(
EvaluationWorkersConfig.class);
EvaluationConfiguration evaluationConfiguration = context.getBean(
EvaluationConfiguration.class);
GlobalConfig globalConfig = context.getBean(GlobalConfig.class);

var requestQueue = new ConcurrentRegistrationQueue();
Expand All @@ -45,21 +51,22 @@ public static void main(String[] args) {
log.info("Registering {} registration workers...", registrationWorkersConfig.amountOfWorkers());

List<ProcessRegistrationRequest> registrationWorkers = new LinkedList<>();
for (int i=0; i<registrationWorkersConfig.amountOfWorkers(); i++) {
registrationWorkers.add(new ProcessRegistrationRequest(requestQueue, registrationConfiguration, globalConfig));
for (int i = 0; i < registrationWorkersConfig.amountOfWorkers(); i++) {
registrationWorkers.add(
new ProcessRegistrationRequest(requestQueue, registrationConfiguration, globalConfig));
}

log.info("Registering {} processing workers...", processingWorkersConfig.threads());

List<ProcessingRequest> processingWorkers = new LinkedList<>();
for (int i=0; i<processingWorkersConfig.threads(); i++) {
for (int i = 0; i < processingWorkersConfig.threads(); i++) {
processingWorkers.add(new ProcessingRequest(processingConfiguration));
}

log.info("Registering {} evaluation workers...", evaluationWorkersConfig.threads());

List<EvaluationRequest> evaluationWorkers = new LinkedList<>();
for (int i=0; i<evaluationWorkersConfig.threads(); i++) {
for (int i = 0; i < evaluationWorkersConfig.threads(); i++) {
evaluationWorkers.add(new EvaluationRequest(evaluationConfiguration));
}

Expand Down
3 changes: 2 additions & 1 deletion src/main/java/life/qbic/data/processing/GlobalConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ public class GlobalConfig {

private final Pattern qbicMeasurementIdPattern;

public GlobalConfig(String usersErrorDirectoryName, String usersRegistrationDirectoryName, String qbicMeasurementIdPattern) {
public GlobalConfig(String usersErrorDirectoryName, String usersRegistrationDirectoryName,
String qbicMeasurementIdPattern) {
if (usersErrorDirectoryName == null || usersErrorDirectoryName.isBlank()) {
throw new IllegalArgumentException("usersErrorDirectoryName cannot be null or empty");
}
Expand Down
1 change: 0 additions & 1 deletion src/main/java/life/qbic/data/processing/Provenance.java
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ public static Provenance parse(Path json) throws ProvenanceException {
}
ObjectMapper mapper = new ObjectMapper().configure(
DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
;
Provenance provenance;
try {
provenance = mapper.readValue(Files.readString(json), Provenance.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.regex.Pattern;

public class EvaluationWorkersConfig {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ public class RegistrationWorkersConfig {

private final String metadataFileName;

public RegistrationWorkersConfig(int threads, String workingDirectory, String targetDirectory, String metadataFileName) {
public RegistrationWorkersConfig(int threads, String workingDirectory, String targetDirectory,
String metadataFileName) {
if (threads < 1) {
throw new IllegalArgumentException("Number of threads must be greater than 0");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,13 @@ private RoundRobinDraw(Collection<T> items) {
}

/**
* Creates an instance of {@link RoundRobinDraw} based on the type {@link T} of the collection provided
* Creates an instance of {@link RoundRobinDraw} based on the type {@link T} of the collection
* provided
*
* @param items a collection of items the round robin method shall be applied.
* @return an instance of this class
* @throws IllegalArgumentException if an empty collection is provided or the collection is <code>null</code>
* @throws IllegalArgumentException if an empty collection is provided or the collection is
* <code>null</code>
* @since 1.0.0
*/
public static <T> RoundRobinDraw<T> create(Collection<T> items) throws IllegalArgumentException {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
package life.qbic.data.processing.evaluation;

import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.regex.Pattern;
import life.qbic.data.processing.AccessRightsEvaluation;
import life.qbic.data.processing.GlobalConfig;
import life.qbic.data.processing.config.RoundRobinDraw;

Expand All @@ -22,16 +23,15 @@ public class EvaluationConfiguration {
private final RoundRobinDraw<Path> targetDirectoriesRoundRobinDraw;

public EvaluationConfiguration(String workingDirectory, Collection<Path> targetDirectories,
GlobalConfig globalConfig) {
GlobalConfig globalConfig) throws IOException {
this.workingDirectory = Paths.get(workingDirectory);
if (!this.workingDirectory.toFile().exists()) {
throw new IllegalArgumentException("Evaluation worker directory does not exist");
}
AccessRightsEvaluation.evaluateExistenceAndDirectory(this.workingDirectory);
AccessRightsEvaluation.evaluateWriteAndExecutablePermission(this.workingDirectory);
this.targetDirectories = targetDirectories.stream().toList();
this.targetDirectories.stream().filter(path -> !path.toFile().exists()).forEach(path -> {
throw new IllegalArgumentException(
"Evaluation target directory '%s' does not exist".formatted(path));
});
for (Path targetDirectory : this.targetDirectories) {
AccessRightsEvaluation.evaluateExistenceAndDirectory(targetDirectory);
AccessRightsEvaluation.evaluateWriteAndExecutablePermission(targetDirectory);
}
this.targetDirectoriesRoundRobinDraw = RoundRobinDraw.create(targetDirectories);
this.usersErrorDirectory = globalConfig.usersErrorDirectory();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,6 @@
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.MatchResult;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import life.qbic.data.processing.ErrorSummary;
import life.qbic.data.processing.Provenance;
import life.qbic.data.processing.Provenance.ProvenanceException;
Expand Down Expand Up @@ -55,7 +51,7 @@ public class EvaluationRequest extends Thread {
private Path assignedTargetDirectory;

public EvaluationRequest(Path workingDirectory, RoundRobinDraw<Path> targetDirectories,
Path usersErrorDirectory) {
Path usersErrorDirectory) {
this.setName(THREAD_NAME.formatted(nextThreadNumber()));
this.workingDirectory = workingDirectory;
this.targetDirectories = targetDirectories;
Expand Down Expand Up @@ -144,7 +140,9 @@ private void evaluateDirectory(File taskDir) {
return;
}

var measurementIdResult = provenance.qbicMeasurementID == null || provenance.qbicMeasurementID.isBlank() ? Optional.empty() : Optional.of(provenance.qbicMeasurementID);
var measurementIdResult =
provenance.qbicMeasurementID == null || provenance.qbicMeasurementID.isBlank()
? Optional.empty() : Optional.of(provenance.qbicMeasurementID);
if (measurementIdResult.isPresent()) {
provenance.addToHistory(taskDir.getAbsolutePath());
try {
Expand All @@ -157,7 +155,7 @@ private void evaluateDirectory(File taskDir) {
try {
createMarkerFile(assignedTargetDirectory, taskDir.getName());
} catch (IOException e) {
LOG.error("Could not create marker file: {}", taskDir.getAbsolutePath(), e);
LOG.error("Could not create marker file in: {}", assignedTargetDirectory, e);
moveToSystemIntervention(taskDir, e.getMessage());
}
return;
Expand Down Expand Up @@ -205,20 +203,22 @@ private void moveBackToOrigin(File taskDir, Provenance provenance, String reason
Paths.get(provenance.userWorkDirectoryPath).resolve(usersErrorDirectory)
.resolve(taskDir.getName()));
} catch (IOException e) {
LOG.error("Cannot move task to user intervention: %s".formatted(Paths.get(provenance.userWorkDirectoryPath).resolve(usersErrorDirectory)), e);
LOG.error("Cannot move task to user intervention: %s".formatted(
Paths.get(provenance.userWorkDirectoryPath).resolve(usersErrorDirectory)), e);
moveToSystemIntervention(taskDir, e.getMessage());
}
}

private void moveToTargetDir(File taskDir) {
LOG.info(
"Moving %s to target directory %s".formatted(taskDir.getAbsolutePath(), assignedTargetDirectory));
"Moving %s to target directory %s".formatted(taskDir.getAbsolutePath(),
assignedTargetDirectory));
try {
Files.move(taskDir.toPath(), assignedTargetDirectory.resolve(taskDir.getName()));
} catch (IOException e) {
LOG.error("Cannot move task to target directory: %s".formatted(targetDirectories), e);
LOG.error("Cannot move task to target directory: %s".formatted(assignedTargetDirectory), e);
moveToSystemIntervention(taskDir,
"Cannot move task to target directory: %s".formatted(targetDirectories));
"Cannot move task to target directory: %s".formatted(assignedTargetDirectory));
}

}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package life.qbic.data.processing.processing;

import java.io.IOException;
import java.nio.file.Path;
import life.qbic.data.processing.AccessRightsEvaluation;

/**
* <b>Processing Configuration</b>
Expand All @@ -17,15 +19,13 @@ public class ProcessingConfiguration {

private final Path targetDirectory;

public ProcessingConfiguration(Path workingDirectory, Path targetDirectory) {
public ProcessingConfiguration(Path workingDirectory, Path targetDirectory) throws IOException {
this.workingDirectory = workingDirectory;
if (!workingDirectory.toFile().exists()) {
throw new IllegalArgumentException("Working directory does not exist: " + workingDirectory);
}
AccessRightsEvaluation.evaluateExistenceAndDirectory(this.workingDirectory);
AccessRightsEvaluation.evaluateWriteAndExecutablePermission(this.workingDirectory);
this.targetDirectory = targetDirectory;
if (!targetDirectory.toFile().exists()) {
throw new IllegalArgumentException("Target directory does not exist: " + targetDirectory);
}
AccessRightsEvaluation.evaluateExistenceAndDirectory(this.targetDirectory);
AccessRightsEvaluation.evaluateWriteAndExecutablePermission(this.workingDirectory);
}

public Path getWorkingDirectory() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,9 @@ public void run() {
* @since
*/
private void processFile(File taskDir) {
var taskDirContent = Arrays.stream(Objects.requireNonNull(taskDir.listFiles(), "Task dir must not be null: " + taskDir)).toList();
var taskDirContent = Arrays.stream(
Objects.requireNonNull(taskDir.listFiles(), "Task dir must not be null: " + taskDir))
.toList();

if (checkForEmpty(taskDir, taskDirContent)) {
return;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public class ProcessRegistrationRequest extends Thread {
private final String metadataFileName;
private final Path userErrorDirectory;
private final Pattern measurementIdPattern;
private AtomicBoolean active = new AtomicBoolean(false);
private final AtomicBoolean active = new AtomicBoolean(false);

public ProcessRegistrationRequest(@NonNull ConcurrentRegistrationQueue registrationQueue,
@NonNull RegistrationConfiguration configuration, @NonNull GlobalConfig globalConfig) {
Expand Down
Loading

0 comments on commit 12abdf0

Please sign in to comment.