Merge remote-tracking branch 'origin/main' into feature/directly-load…

…-acm-files

tests/integration-tests/tests-base/pom.xml

@@ -17,6 +17,12 @@
       <groupId>com.tngtech.archunit</groupId>
       <artifactId>archunit-junit5</artifactId>
     </dependency>
+    <dependency>
+      <groupId>io.github.ardoco</groupId>
+      <artifactId>metrics</artifactId>
+      <version>0.1.1-SNAPSHOT</version>
+      <scope>compile</scope>
+    </dependency>
     <dependency>
       <groupId>io.github.ardoco.core</groupId>
       <artifactId>common</artifactId>

tests/integration-tests/tests-base/src/main/java/edu/kit/kastel/mcse/ardoco/core/tests/TestUtil.java

@@ -3,17 +3,14 @@
 
 import java.util.Locale;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import org.eclipse.collections.api.collection.ImmutableCollection;
-import org.eclipse.collections.api.factory.Lists;
-import org.eclipse.collections.api.list.ImmutableList;
 import org.slf4j.Logger;
 
 import edu.kit.kastel.mcse.ardoco.core.api.output.ArDoCoResult;
 import edu.kit.kastel.mcse.ardoco.core.tests.eval.results.EvaluationResults;
 import edu.kit.kastel.mcse.ardoco.core.tests.eval.results.ExpectedResults;
-import edu.kit.kastel.mcse.ardoco.core.tests.eval.results.ResultMatrix;
+import edu.kit.kastel.mcse.ardoco.metrics.ClassificationMetricsCalculator;
 
 /**
  * This utility class provides methods for running the tests, especially regarding the evaluations.
@@ -37,20 +34,18 @@ public static <T> EvaluationResults<T> compareTLR(ArDoCoResult arDoCoResult, Imm
         Set<T> distinctTraceLinks = new java.util.LinkedHashSet<>(results.castToCollection());
         Set<T> distinctGoldStandard = new java.util.LinkedHashSet<>(goldStandard.castToCollection());
 
-        // True Positives are the trace links that are contained on both lists
-        Set<T> truePositives = distinctTraceLinks.stream().filter(distinctGoldStandard::contains).collect(Collectors.toSet());
-        ImmutableList<T> truePositivesList = Lists.immutable.ofAll(truePositives);
+        int sentences = arDoCoResult.getText().getSentences().size();
+        int modelElements = 0;
+        for (var model : arDoCoResult.getModelIds()) {
+            modelElements += arDoCoResult.getModelState(model).getInstances().size();
+        }
 
-        // False Positives are the trace links that are only contained in the result set
-        Set<T> falsePositives = distinctTraceLinks.stream().filter(tl -> !distinctGoldStandard.contains(tl)).collect(Collectors.toSet());
-        ImmutableList<T> falsePositivesList = Lists.immutable.ofAll(falsePositives);
+        int confusionMatrixSum = sentences * modelElements;
 
-        // False Negatives are the trace links that are only contained in the gold standard
-        Set<T> falseNegatives = distinctGoldStandard.stream().filter(tl -> !distinctTraceLinks.contains(tl)).collect(Collectors.toSet());
-        ImmutableList<T> falseNegativesList = Lists.immutable.ofAll(falseNegatives);
+        var calculator = ClassificationMetricsCalculator.getInstance();
 
-        int trueNegatives = TestUtil.calculateTrueNegativesForTLR(arDoCoResult, truePositives.size(), falsePositives.size(), falseNegatives.size());
-        return EvaluationResults.createEvaluationResults(new ResultMatrix<>(truePositivesList, trueNegatives, falsePositivesList, falseNegativesList));
+        var classification = calculator.calculateMetrics(distinctTraceLinks, distinctGoldStandard, confusionMatrixSum);
+        return new EvaluationResults<>(classification);
     }
 
     /**
@@ -67,57 +62,11 @@ public static <T> EvaluationResults<T> compareInconsistencies(ArDoCoResult arDoC
         Set<T> distinctTraceLinks = new java.util.LinkedHashSet<>(results.castToCollection());
         Set<T> distinctGoldStandard = new java.util.LinkedHashSet<>(goldStandard.castToCollection());
 
-        // True Positives are the trace links that are contained on both lists
-        Set<T> truePositives = distinctTraceLinks.stream().filter(distinctGoldStandard::contains).collect(Collectors.toSet());
-        ImmutableList<T> truePositivesList = Lists.immutable.ofAll(truePositives);
-
-        // False Positives are the trace links that are only contained in the result set
-        Set<T> falsePositives = distinctTraceLinks.stream().filter(tl -> !distinctGoldStandard.contains(tl)).collect(Collectors.toSet());
-        ImmutableList<T> falsePositivesList = Lists.immutable.ofAll(falsePositives);
-
-        // False Negatives are the trace links that are only contained in the gold standard
-        Set<T> falseNegatives = distinctGoldStandard.stream().filter(tl -> !distinctTraceLinks.contains(tl)).collect(Collectors.toSet());
-        ImmutableList<T> falseNegativesList = Lists.immutable.ofAll(falseNegatives);
-
-        int trueNegatives = TestUtil.calculateTrueNegativesForInconsistencies(arDoCoResult, truePositives.size(), falsePositives.size(), falseNegatives.size());
-        return EvaluationResults.createEvaluationResults(new ResultMatrix<>(truePositivesList, trueNegatives, falsePositivesList, falseNegativesList));
-    }
-
-    /**
-     * Calculates the number of true negatives based on the given {@link ArDoCoResult} and the calculated {@link EvaluationResults evaluation results}. Uses the
-     * total sum of all entries in the confusion matrix and then substracts the true positives, false positives, and false negatives.
-     *
-     * @param arDoCoResult   the output of ArDoCo
-     * @param truePositives  nr of true positives
-     * @param falsePositives nr of false positives
-     * @param falseNegatives nr of false negatives
-     * @return the number of true negatives
-     */
-    public static int calculateTrueNegativesForTLR(ArDoCoResult arDoCoResult, int truePositives, int falsePositives, int falseNegatives) {
-        int sentences = arDoCoResult.getText().getSentences().size();
-        int modelElements = 0;
-        for (var model : arDoCoResult.getModelIds()) {
-            modelElements += arDoCoResult.getModelState(model).getInstances().size();
-        }
-
-        int confusionMatrixSum = sentences * modelElements;
-        return confusionMatrixSum - (truePositives + falsePositives + falseNegatives);
-    }
-
-    /**
-     * Calculates the number of true negatives based on the given {@link ArDoCoResult} and the calculated {@link EvaluationResults evaluation results}. Uses the
-     * total sum of all sentences in the {@link ArDoCoResult} and then substracts the true positives, false positives, and false negatives.
-     *
-     * @param arDoCoResult   the output of ArDoCo
-     * @param truePositives  nr of true positives
-     * @param falsePositives nr of false positives
-     * @param falseNegatives nr of false negatives
-     * @return the number of true negatives
-     */
-    public static int calculateTrueNegativesForInconsistencies(ArDoCoResult arDoCoResult, int truePositives, int falsePositives, int falseNegatives) {
-        int numberOfSentences = arDoCoResult.getText().getSentences().size();
-        return numberOfSentences - (truePositives + falsePositives + falseNegatives);
+        int confusionMatrixSum = arDoCoResult.getText().getSentences().size();
 
+        var calculator = ClassificationMetricsCalculator.getInstance();
+        var classification = calculator.calculateMetrics(distinctTraceLinks, distinctGoldStandard, confusionMatrixSum);
+        return new EvaluationResults<>(classification);
     }
 
     /**
@@ -168,19 +117,6 @@ public static void logExplicitResults(Logger logger, String name, EvaluationResu
         logger.info(logString);
     }
 
-    /**
-     * Log the provided {@link EvaluationResults} using the provided logger and name. Additionally, provided the expected results.
-     *
-     * @param logger          Logger to use
-     * @param name            Name to show in the output
-     * @param results         the results
-     * @param expectedResults the expected results
-     */
-    public static void logResultsWithExpected(Logger logger, String name, EvaluationResults<?> results, ExpectedResults expectedResults) {
-        var infoString = String.format(Locale.ENGLISH, "%n%s:%n%s", name, results.getResultStringWithExpected(expectedResults));
-        logger.info(infoString);
-    }
-
     public static void logExtendedResultsWithExpected(Logger logger, Object testClass, String name, EvaluationResults<?> results,
             ExpectedResults expectedResults) {
         var infoString = String.format(Locale.ENGLISH, """

tests/integration-tests/tests-base/src/main/java/edu/kit/kastel/mcse/ardoco/core/tests/architecture/DeterministicArDoCoTest.java

@@ -50,7 +50,7 @@ public class DeterministicArDoCoTest {
 
     @ArchTest
     public static final ArchRule forbidUnorderedSetsAndMaps = noClasses().that()
-            .resideOutsideOfPackages("..tests..")
+            .resideOutsideOfPackages("..tests..", "..metrics..")
             .and(areNotDirectlyAnnotatedWith(Deterministic.class))
             .should()
             .accessClassesThat(areForbiddenClasses())

tests/integration-tests/tests-base/src/main/java/edu/kit/kastel/mcse/ardoco/core/tests/eval/CodeProject.java

@@ -5,6 +5,7 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.List;
@@ -159,8 +160,7 @@ public String getCodeModelDirectory() {
             loadCodeModelFromResourcesIfNeeded();
             return getTemporaryCodeLocation().getAbsolutePath();
         } catch (IOException e) {
-            logger.error(e.getMessage(), e);
-            return null;
+            throw new UncheckedIOException(e);
         }
     }