diff --git a/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/BoundedPriorityQueue.java b/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/BoundedPriorityQueue.java deleted file mode 100644 index 8e49308ffd..0000000000 --- a/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/BoundedPriorityQueue.java +++ /dev/null @@ -1,56 +0,0 @@ -package org.mobilitydata.gtfsvalidator.outputcomparator.io; - -import java.util.Comparator; -import java.util.PriorityQueue; - -/** - * A bounded priority queue that keeps the N smallest elements. If the queue is full and a new - * element is offered, the largest element is removed. The smallest element is computed using a - * comparator or its natural order. - * - * @param - */ -public class BoundedPriorityQueue extends PriorityQueue { - private final int maxCapacity; - - public BoundedPriorityQueue(int maxCapacity) { - super(); - if (maxCapacity <= 0) { - throw new IllegalArgumentException("Max capacity must be greater than zero"); - } - this.maxCapacity = maxCapacity; - } - - public BoundedPriorityQueue(int maxCapacity, int initialCapacity, Comparator comparator) { - super(initialCapacity, comparator); - if (maxCapacity <= 0) { - throw new IllegalArgumentException("Max capacity must be greater than zero"); - } - this.maxCapacity = maxCapacity; - } - - @Override - public boolean offer(E e) { - if (size() >= maxCapacity) { - E head = peek(); - if (head != null && compare(e, head) > 0) { - poll(); - } else { - return false; - } - } - return super.offer(e); - } - - @SuppressWarnings("unchecked") - private int compare(E a, E b) { - if (comparator() != null) { - return comparator().compare(a, b); - } - return ((Comparable) a).compareTo(b); - } - - public int getMaxCapacity() { - return maxCapacity; - } -} diff --git a/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/UsedMemoryIncreasedComparator.java b/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/UsedMemoryIncreasedComparator.java deleted file mode 100644 index 9b37b6bd8e..0000000000 --- a/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/UsedMemoryIncreasedComparator.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.mobilitydata.gtfsvalidator.outputcomparator.io; - -import java.util.Comparator; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -/** - * Comparator to compare two {@link DatasetMemoryUsage} objects based on the difference between the - * used memory of the two objects. The difference is calculated by comparing the used memory of the - * two objects for each key present in both objects. If a key is present in one object but not in - * the other, the key it is ignored. This comparator is used to sort DatasetMemoryUsage by the - * minimum difference between the used memory of the two. This means the order is by the dataset - * validation that increased the memory. - */ -public class UsedMemoryIncreasedComparator implements Comparator { - - @Override - public int compare(DatasetMemoryUsage o1, DatasetMemoryUsage o2) { - if (o1 == o2) { - return 0; - } - if (o1 == null || o2 == null) { - return o1 == null ? -1 : 1; - } - if (o1.getReferenceMemoryUsage() == null - && o1.getLatestMemoryUsage() == null - && o2.getReferenceMemoryUsage() == null - && o2.getLatestMemoryUsage() == null) { - return 0; - } - if (o1.getReferenceMemoryUsage() == null || o2.getReferenceMemoryUsage() == null) { - return o1.getReferenceMemoryUsage() == null ? -1 : 1; - } - if (o1.getLatestMemoryUsage() == null || o2.getLatestMemoryUsage() == null) { - return o1.getLatestMemoryUsage() == null ? -1 : 1; - } - long o1MinDiff = - getMinimumDifferenceByKey(o1.getReferenceUsedMemoryByKey(), o1.getLatestUsedMemoryByKey()); - long o2MinDiff = - getMinimumDifferenceByKey(o2.getReferenceUsedMemoryByKey(), o2.getLatestUsedMemoryByKey()); - return Long.compare(o1MinDiff, o2MinDiff); - } - - private long getMinimumDifferenceByKey( - Map referenceMemoryUsage, Map latestMemoryUsage) { - Set keys = new HashSet<>(); - keys.addAll(latestMemoryUsage.keySet()); - keys.addAll(referenceMemoryUsage.keySet()); - return keys.stream() - .filter(key -> latestMemoryUsage.containsKey(key) && referenceMemoryUsage.containsKey(key)) - .filter(key -> latestMemoryUsage.get(key) - referenceMemoryUsage.get(key) != 0) - .mapToLong(key -> referenceMemoryUsage.get(key) - latestMemoryUsage.get(key)) - .min() - .orElse(Long.MAX_VALUE); - } -} diff --git a/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollector.java b/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollector.java index db7468d00f..f4712651a6 100644 --- a/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollector.java +++ b/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollector.java @@ -1,34 +1,26 @@ package org.mobilitydata.gtfsvalidator.outputcomparator.io; import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import java.util.stream.Collectors; import org.mobilitydata.gtfsvalidator.model.ValidationReport; import org.mobilitydata.gtfsvalidator.outputcomparator.model.report.ValidationPerformance; import org.mobilitydata.gtfsvalidator.performance.MemoryUsage; public class ValidationPerformanceCollector { - public static final int MEMORY_USAGE_COMPARE_MAX = 25; + public static final String MEMORY_PIVOT_KEY = + "org.mobilitydata.gtfsvalidator.table.GtfsFeedLoader.loadAndValidate"; private final Map referenceTimes; private final Map latestTimes; - private final BoundedPriorityQueue datasetsDecreasedMemoryUsage; - private final BoundedPriorityQueue datasetsIncreasedMemoryUsage; private final List datasetsMemoryUsageNoReference; + private final List datasetsMemoryUsageWithReference; public ValidationPerformanceCollector() { this.referenceTimes = new HashMap<>(); this.latestTimes = new HashMap<>(); - this.datasetsDecreasedMemoryUsage = - new BoundedPriorityQueue<>( - MEMORY_USAGE_COMPARE_MAX, - MEMORY_USAGE_COMPARE_MAX, - (new UsedMemoryIncreasedComparator().reversed())); - this.datasetsIncreasedMemoryUsage = - new BoundedPriorityQueue<>( - MEMORY_USAGE_COMPARE_MAX, - MEMORY_USAGE_COMPARE_MAX, - new UsedMemoryIncreasedComparator()); this.datasetsMemoryUsageNoReference = new ArrayList<>(); + this.datasetsMemoryUsageWithReference = new ArrayList<>(); } public void addReferenceTime(String sourceId, Double time) { @@ -39,11 +31,11 @@ public void addLatestTime(String sourceId, Double time) { latestTimes.put(sourceId, time); } - private Double computeAverage(List times) { + private Double computeAverage(Collection times) { return times.stream().mapToDouble(Double::doubleValue).average().orElse(Double.NaN); } - private Double computeMedian(List times) { + private Double computeMedian(Collection times) { if (times.isEmpty()) { return Double.NaN; } @@ -59,31 +51,37 @@ private Double computeMedian(List times) { return median; } - private Double computeStandardDeviation(List times) { + private Double computeStandardDeviation(Collection times) { double mean = computeAverage(times); return Math.sqrt( times.stream().mapToDouble(time -> Math.pow(time - mean, 2)).average().orElse(Double.NaN)); } - private Double computeMax(List times) { + private Double computeMax(Collection times) { return times.stream().mapToDouble(Double::doubleValue).max().orElse(Double.NaN); } - private Double computeMin(List times) { + private Double computeMin(Collection times) { return times.stream().mapToDouble(Double::doubleValue).min().orElse(Double.NaN); } - private String formatMetrics(String metric, String datasetId, Double reference, Double latest) { + private String formatMetrics( + String metric, + String datasetId, + Double reference, + Double latest, + Function render) { String diff; if (reference.isNaN() || latest.isNaN()) { diff = "N/A"; } else { double difference = latest - reference; String arrow = difference > 0 ? "⬆️+" : "⬇️"; - diff = String.format("%s%.2f", arrow, difference); + diff = String.format("%s%s", arrow, render.apply(difference)); } return String.format( - "| %s | %s | %.2f | %.2f | %s |\n", metric, datasetId, reference, latest, diff); + "| %s | %s | %s | %s | %s |\n", + metric, datasetId, render.apply(reference), render.apply(latest), diff); } private static String getMemoryDiff(Long reference, Long latest) { @@ -134,70 +132,8 @@ public String generateLogString() { allLatestTimes.add(latestTimes); } - if (!allReferenceTimes.isEmpty() && !allLatestTimes.isEmpty()) { - Double avgReference = computeAverage(allReferenceTimes); - Double avgLatest = computeAverage(allLatestTimes); - Double medianReference = computeMedian(allReferenceTimes); - Double medianLatest = computeMedian(allLatestTimes); - Double stdDevReference = computeStandardDeviation(allReferenceTimes); - Double stdDevLatest = computeStandardDeviation(allLatestTimes); - - b.append(formatMetrics("Average", "--", avgReference, avgLatest)) - .append(formatMetrics("Median", "--", medianReference, medianLatest)) - .append(formatMetrics("Standard Deviation", "--", stdDevReference, stdDevLatest)); - } - - if (!allReferenceTimes.isEmpty()) { - Double minReference = computeMin(allReferenceTimes); - String minReferenceId = - referenceTimes.entrySet().stream() - .filter(entry -> Objects.equals(entry.getValue(), minReference)) - .map(Map.Entry::getKey) - .findFirst() - .orElse("N/A"); - - Double maxReference = computeMax(allReferenceTimes); - String maxReferenceId = - referenceTimes.entrySet().stream() - .filter(entry -> Objects.equals(entry.getValue(), maxReference)) - .map(Map.Entry::getKey) - .findFirst() - .orElse("N/A"); - - Double minLatest = latestTimes.getOrDefault(minReferenceId, Double.NaN); - Double maxLatest = latestTimes.getOrDefault(maxReferenceId, Double.NaN); - - b.append( - formatMetrics( - "Minimum in References Reports", minReferenceId, minReference, minLatest)) - .append( - formatMetrics( - "Maximum in Reference Reports", maxReferenceId, maxReference, maxLatest)); - } - - if (!allLatestTimes.isEmpty()) { - Double minLatest = computeMin(allLatestTimes); - String minLatestId = - latestTimes.entrySet().stream() - .filter(entry -> Objects.equals(entry.getValue(), minLatest)) - .map(Map.Entry::getKey) - .findFirst() - .orElse("N/A"); - - Double maxLatest = computeMax(allLatestTimes); - String maxLatestId = - latestTimes.entrySet().stream() - .filter(entry -> Objects.equals(entry.getValue(), maxLatest)) - .map(Map.Entry::getKey) - .findFirst() - .orElse("N/A"); - - Double minReference = referenceTimes.getOrDefault(minLatestId, Double.NaN); - Double maxReference = referenceTimes.getOrDefault(maxLatestId, Double.NaN); - - b.append(formatMetrics("Minimum in Latest Reports", minLatestId, minReference, minLatest)) - .append(formatMetrics("Maximum in Latest Reports", maxLatestId, maxReference, maxLatest)); - } + generatePerformanceMetricsLog( + referenceTimes, latestTimes, b, value -> String.format("%.2f", value)); // Add warning message for feeds that are missing validation times either in reference or latest if (!warnings.isEmpty()) { @@ -211,102 +147,121 @@ public String generateLogString() { b.append("\n\n"); - if (datasetsIncreasedMemoryUsage.size() > 0 - || datasetsDecreasedMemoryUsage.size() > 0 - || datasetsMemoryUsageNoReference.size() > 0) { + if (datasetsMemoryUsageWithReference.size() > 0) { + Map referenceMemoryUsageById = + datasetsMemoryUsageWithReference.stream() + .filter( + datasetMemoryUsage -> + datasetMemoryUsage.getReferenceUsedMemoryByKey().get(MEMORY_PIVOT_KEY) + != null) + .collect( + Collectors.toMap( + DatasetMemoryUsage::getDatasetId, + datasetMemoryUsage -> + datasetMemoryUsage + .getReferenceUsedMemoryByKey() + .get(MEMORY_PIVOT_KEY) + .doubleValue())); + Map latestMemoryUsageById = + datasetsMemoryUsageWithReference.stream() + .filter( + datasetMemoryUsage -> + datasetMemoryUsage.getLatestUsedMemoryByKey().get(MEMORY_PIVOT_KEY) != null) + .collect( + Collectors.toMap( + DatasetMemoryUsage::getDatasetId, + datasetMemoryUsage -> + datasetMemoryUsage + .getLatestUsedMemoryByKey() + .get(MEMORY_PIVOT_KEY) + .doubleValue())); + b.append("
\n"); - b.append("📜 Memory Consumption\n"); - if (datasetsIncreasedMemoryUsage.size() > 0) { - List increasedMemoryUsages = - getDatasetMemoryUsages(datasetsIncreasedMemoryUsage); - addMemoryUsageReport(increasedMemoryUsages, "memory has increased", b, true); - } - if (datasetsDecreasedMemoryUsage.size() > 0) { - List decreasedMemoryUsages = - getDatasetMemoryUsages(datasetsDecreasedMemoryUsage); - addMemoryUsageReport(decreasedMemoryUsages, "memory has decreased", b, true); - } - if (datasetsMemoryUsageNoReference.size() > 0) { - // Sorting from the highest to the lowest memory usage - datasetsMemoryUsageNoReference.sort((new LatestReportUsedMemoryComparator()).reversed()); - addMemoryUsageReport( - datasetsMemoryUsageNoReference.subList( - 0, Math.min(datasetsMemoryUsageNoReference.size(), MEMORY_USAGE_COMPARE_MAX)), - "no reference available", - b, - false); - } + b.append("📜 Memory Consumption\n\n"); + + b.append( + "| Metric | Dataset ID | Reference (s) | Latest (s) | Difference (s) |\n") + .append( + "|-----------------------------|-------------------|----------------|----------------|----------------|\n"); + + generatePerformanceMetricsLog( + referenceMemoryUsageById, + latestMemoryUsageById, + b, + ValidationPerformanceCollector::convertToHumanReadableMemory); b.append("
\n"); } return b.toString(); } - private List getDatasetMemoryUsages( - BoundedPriorityQueue datasetsMemoryUsage) { - List increasedMemoryUsages = new ArrayList<>(datasetsMemoryUsage); - increasedMemoryUsages.sort(datasetsMemoryUsage.comparator()); - return increasedMemoryUsages; - } - - private void addMemoryUsageReport( - List memoryUsages, - String order, + private void generatePerformanceMetricsLog( + Map references, + Map latests, StringBuilder b, - boolean includeDifference) { - b.append(String.format("

List of %s datasets(%s).

", MEMORY_USAGE_COMPARE_MAX, order)) - .append("\n\n") - .append( - "| Dataset ID | Snapshot Key(Used Memory) | Reference | Latest |"); - if (includeDifference) { - b.append(" Difference |"); + Function render) { + PerformanceMetrics performanceMetrics = computeMetrics(references, latests); + if (!references.isEmpty() && !latests.isEmpty()) { + b.append( + formatMetrics( + "Average", + "--", + performanceMetrics.avgReference, + performanceMetrics.avgLatest, + render)) + .append( + formatMetrics( + "Median", + "--", + performanceMetrics.medianReference, + performanceMetrics.medianLatest, + render)) + .append( + formatMetrics( + "Standard Deviation", + "--", + performanceMetrics.stdDevReference, + performanceMetrics.stdDevLatest, + render)); } - b.append("\n"); - b.append( - "|-----------------------------|-------------------|----------------|----------------|"); - if (includeDifference) { - b.append("----------------|"); + + if (!references.isEmpty()) { + Double minLatest = latests.getOrDefault(performanceMetrics.minReferenceId, Double.NaN); + Double maxLatest = latests.getOrDefault(performanceMetrics.maxReferenceId, Double.NaN); + b.append( + formatMetrics( + "Minimum in References Reports", + performanceMetrics.minReferenceId, + performanceMetrics.minReference, + minLatest, + render)) + .append( + formatMetrics( + "Maximum in Reference Reports", + performanceMetrics.maxReferenceId, + performanceMetrics.maxReference, + maxLatest, + render)); } - b.append("\n"); - memoryUsages.stream() - .forEachOrdered( - datasetMemoryUsage -> { - generateMemoryLogByKey(datasetMemoryUsage, b, includeDifference); - }); - } - private static void generateMemoryLogByKey( - DatasetMemoryUsage datasetMemoryUsage, StringBuilder b, boolean includeDifference) { - AtomicBoolean isFirst = new AtomicBoolean(true); - Set keys = new HashSet<>(); - keys.addAll(datasetMemoryUsage.getReferenceUsedMemoryByKey().keySet()); - keys.addAll(datasetMemoryUsage.getLatestUsedMemoryByKey().keySet()); - keys.stream() - .forEach( - key -> { - var reference = datasetMemoryUsage.getReferenceUsedMemoryByKey().get(key); - var latest = datasetMemoryUsage.getLatestUsedMemoryByKey().get(key); - if (isFirst.get()) { - b.append(String.format("| %s | | | |", datasetMemoryUsage.getDatasetId())); - if (includeDifference) { - b.append(" |"); - } - b.append("\n"); - isFirst.set(false); - } - String usedMemoryDiff = getMemoryDiff(reference, latest); - b.append( - String.format( - "| | %s | %s | %s |", - key, - reference != null - ? MemoryUsage.convertToHumanReadableMemory(reference) - : "N/A", - latest != null ? MemoryUsage.convertToHumanReadableMemory(latest) : "N/A")); - if (includeDifference) { - b.append(String.format(" %s |", usedMemoryDiff)); - } - b.append("\n"); - }); + if (!latests.isEmpty()) { + Double minReference = references.getOrDefault(performanceMetrics.minLatestId, Double.NaN); + Double maxReference = references.getOrDefault(performanceMetrics.maxLatestId, Double.NaN); + + b.append( + formatMetrics( + "Minimum in Latest Reports", + performanceMetrics.minLatestId, + minReference, + performanceMetrics.minLatest, + render)) + .append( + formatMetrics( + "Maximum in Latest Reports", + performanceMetrics.maxLatestId, + maxReference, + performanceMetrics.maxLatest, + render)); + } } public void compareValidationReports( @@ -332,8 +287,7 @@ private void compareValidationReportMemoryUsage( && referenceReport.getMemoryUsageRecords().size() > 0 && latestReport.getMemoryUsageRecords() != null && latestReport.getMemoryUsageRecords().size() > 0) { - datasetsIncreasedMemoryUsage.offer(datasetMemoryUsage); - datasetsDecreasedMemoryUsage.offer(datasetMemoryUsage); + datasetsMemoryUsageWithReference.add(datasetMemoryUsage); } else { datasetsMemoryUsageNoReference.add(datasetMemoryUsage); } @@ -352,4 +306,78 @@ public List toReport() { } return affectedSources; } + + private PerformanceMetrics computeMetrics( + Map referencesById, Map latestsById) { + Collection allReferences = referencesById.values(); + Collection allLatest = latestsById.values(); + PerformanceMetrics performanceMetrics = new PerformanceMetrics(); + if (!allReferences.isEmpty() && !allLatest.isEmpty()) { + performanceMetrics.avgReference = computeAverage(allReferences); + performanceMetrics.avgLatest = computeAverage(allLatest); + performanceMetrics.medianReference = computeMedian(allReferences); + performanceMetrics.medianLatest = computeMedian(allLatest); + performanceMetrics.stdDevReference = computeStandardDeviation(allReferences); + performanceMetrics.stdDevLatest = computeStandardDeviation(allLatest); + } + + if (!allReferences.isEmpty()) { + performanceMetrics.minReference = computeMin(allReferences); + performanceMetrics.minReferenceId = + referencesById.entrySet().stream() + .filter(entry -> Objects.equals(entry.getValue(), performanceMetrics.minReference)) + .map(Map.Entry::getKey) + .findFirst() + .orElse("N/A"); + + performanceMetrics.maxReference = computeMax(allReferences); + performanceMetrics.maxReferenceId = + referencesById.entrySet().stream() + .filter(entry -> Objects.equals(entry.getValue(), performanceMetrics.maxReference)) + .map(Map.Entry::getKey) + .findFirst() + .orElse("N/A"); + } + + if (!allLatest.isEmpty()) { + performanceMetrics.minLatest = computeMin(allLatest); + performanceMetrics.minLatestId = + latestsById.entrySet().stream() + .filter(entry -> Objects.equals(entry.getValue(), performanceMetrics.minLatest)) + .map(Map.Entry::getKey) + .findFirst() + .orElse("N/A"); + + performanceMetrics.maxLatest = computeMax(allLatest); + performanceMetrics.maxLatestId = + latestsById.entrySet().stream() + .filter(entry -> Objects.equals(entry.getValue(), performanceMetrics.maxLatest)) + .map(Map.Entry::getKey) + .findFirst() + .orElse("N/A"); + } + return performanceMetrics; + } + + private static String convertToHumanReadableMemory(Double bytes) { + // Ignoring the decimals in bytes + return MemoryUsage.convertToHumanReadableMemory(bytes.longValue()); + } +} + +class PerformanceMetrics { + Double minReference; + String minReferenceId; + Double maxReference; + Double minLatest; + Double maxLatest; + String minLatestId; + String maxLatestId; + String maxReferenceId; + Double avgReference; + Double avgLatest; + Double medianReference; + Double medianLatest; + Double stdDevReference; + Double stdDevLatest; } diff --git a/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/cli/ValidationReportComparatorTest.java b/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/cli/ValidationReportComparatorTest.java index 6cf2ac3a7d..18e8d66e56 100644 --- a/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/cli/ValidationReportComparatorTest.java +++ b/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/cli/ValidationReportComparatorTest.java @@ -141,13 +141,7 @@ public void addedErrorNotice_summaryString() throws Exception { + "\n" + "| Time Metric | Dataset ID | Reference (s) | Latest (s) | Difference (s) |\n" + "|-----------------------------|-------------------|----------------|----------------|----------------|\n" - + "\n\n" - + "
\n" - + "📜 Memory Consumption\n" - + "

List of 25 datasets(no reference available).

\n\n" - + "| Dataset ID | Snapshot Key(Used Memory) | Reference | Latest |\n" - + "|-----------------------------|-------------------|----------------|----------------|\n" - + "
\n\n"); + + "\n\n\n"); } @Test diff --git a/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/MemoryUsageUsedMemoryComparatorTest.java b/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/MemoryUsageUsedMemoryComparatorTest.java deleted file mode 100644 index b09e55387d..0000000000 --- a/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/MemoryUsageUsedMemoryComparatorTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.mobilitydata.gtfsvalidator.outputcomparator.io; - -import static org.junit.Assert.assertEquals; - -import java.util.*; -import org.junit.Before; -import org.junit.Test; -import org.mobilitydata.gtfsvalidator.performance.MemoryUsage; - -public class MemoryUsageUsedMemoryComparatorTest { - - private UsedMemoryIncreasedComparator comparator; - - @Before - public void setUp() { - comparator = new UsedMemoryIncreasedComparator(); - } - - @Test - public void testCompare_equalMemoryUsage() { - List referenceMemoryUsage = getMemoryUsage(100L); - List latestMemoryUsage = getMemoryUsage(100L); - DatasetMemoryUsage o1 = - new DatasetMemoryUsage("dataset1", referenceMemoryUsage, latestMemoryUsage); - DatasetMemoryUsage o2 = - new DatasetMemoryUsage("dataset1", referenceMemoryUsage, latestMemoryUsage); - assertEquals(0, comparator.compare(o1, o2)); - } - - @Test - public void testCompare_firstHasMoreMemoryDifference() { - List referenceMemoryUsage = getMemoryUsage(100L); - List latestMemoryUsage = getMemoryUsage(50L); - DatasetMemoryUsage o1 = - new DatasetMemoryUsage("dataset1", referenceMemoryUsage, latestMemoryUsage); - DatasetMemoryUsage o2 = - new DatasetMemoryUsage("dataset1", referenceMemoryUsage, getMemoryUsage(100L)); - assertEquals(-1, comparator.compare(o1, o2)); - } - - @Test - public void testCompare_firstHasLessMemoryDifference() { - List referenceMemoryUsage = getMemoryUsage(100L); - List latestMemoryUsage = getMemoryUsage(50L); - DatasetMemoryUsage o1 = - new DatasetMemoryUsage("dataset1", referenceMemoryUsage, latestMemoryUsage); - DatasetMemoryUsage o2 = - new DatasetMemoryUsage("dataset1", referenceMemoryUsage, getMemoryUsage(10L)); - assertEquals(1, comparator.compare(o1, o2)); - } - - private static List getMemoryUsage(long freeMemory) { - MemoryUsage[] referenceMemoryUsage = - new MemoryUsage[] { - new MemoryUsage("key1", 100L, freeMemory, 100L, 100L), - new MemoryUsage("key2", 100L, freeMemory, 100L, 100L), - }; - return Arrays.asList(referenceMemoryUsage); - } -} diff --git a/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollectorTest.java b/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollectorTest.java index 82c204b20c..12a13e31b8 100644 --- a/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollectorTest.java +++ b/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollectorTest.java @@ -32,7 +32,12 @@ public void generateLogString_test() { Collections.EMPTY_SET, null, Arrays.asList( - new MemoryUsage("key1", baseMemory, baseMemory + baseMemory * 10, 200, 50L), + new MemoryUsage( + ValidationPerformanceCollector.MEMORY_PIVOT_KEY, + baseMemory, + baseMemory + baseMemory * 10, + 200, + 50L), new MemoryUsage("key2", baseMemory, baseMemory, 200, 50L))), new ValidationReport(Collections.EMPTY_SET, 16.0, Collections.EMPTY_LIST)); // Memory usage increased as there is less free memory @@ -42,13 +47,23 @@ public void generateLogString_test() { Collections.EMPTY_SET, null, Arrays.asList( - new MemoryUsage("key1", baseMemory, baseMemory, 200, 50L), + new MemoryUsage( + ValidationPerformanceCollector.MEMORY_PIVOT_KEY, + baseMemory, + baseMemory, + 200, + 50L), new MemoryUsage("key2", baseMemory, baseMemory, 200, 50L))), new ValidationReport( Collections.EMPTY_SET, null, Arrays.asList( - new MemoryUsage("key1", baseMemory, baseMemory - baseMemory / 2, 200, null), + new MemoryUsage( + ValidationPerformanceCollector.MEMORY_PIVOT_KEY, + baseMemory, + baseMemory - baseMemory / 2, + 200, + null), new MemoryUsage("key2", baseMemory, baseMemory - baseMemory / 2, 200, null)))); // // Memory usage decreased as there is more free memory @@ -77,49 +92,28 @@ public void generateLogString_test() { + "\n" + "| Time Metric | Dataset ID | Reference (s) | Latest (s) | Difference (s) |\n" + "|-----------------------------|-------------------|----------------|----------------|----------------|\n" - + "| Average | -- | 17.00 | 20.00 | ⬆\uFE0F+3.00 |\n" - + "| Median | -- | 17.00 | 20.00 | ⬆\uFE0F+3.00 |\n" - + "| Standard Deviation | -- | 3.00 | 2.00 | ⬇\uFE0F-1.00 |\n" + + "| Average | -- | 17.00 | 18.67 | ⬆\uFE0F+1.67 |\n" + + "| Median | -- | 17.00 | 18.00 | ⬆\uFE0F+1.00 |\n" + + "| Standard Deviation | -- | 3.00 | 2.49 | ⬇\uFE0F-0.51 |\n" + "| Minimum in References Reports | feed-id-a | 14.00 | 18.00 | ⬆\uFE0F+4.00 |\n" + "| Maximum in Reference Reports | feed-id-b | 20.00 | 22.00 | ⬆️+2.00 |\n" - + "| Minimum in Latest Reports | feed-id-a | 14.00 | 18.00 | ⬆\uFE0F+4.00 |\n" + + "| Minimum in Latest Reports | feed-id-m1 | NaN | 16.00 | N/A |\n" + "| Maximum in Latest Reports | feed-id-b | 20.00 | 22.00 | ⬆️+2.00 |\n" + "#### ⚠️ Warnings\n\n" + "The following dataset IDs are missing validation times either in reference or latest:\n" + "feed-id-m1\n\n" + "\n\n" + "
\n" - + "📜 Memory Consumption\n" - + "

List of " - + ValidationPerformanceCollector.MEMORY_USAGE_COMPARE_MAX - + " datasets(memory has increased).

\n\n" - + "| Dataset ID | Snapshot Key(Used Memory) | Reference | Latest | Difference |\n" + + "📜 Memory Consumption\n\n" + + "| Metric | Dataset ID | Reference (s) | Latest (s) | Difference (s) |\n" + "|-----------------------------|-------------------|----------------|----------------|----------------|\n" - + "| feed-id-m2 | | | | |\n" - + "| | key1 | 0 bytes | 488.28 KiB | ⬆\uFE0F+488.28 KiB |\n" - + "| | key2 | 0 bytes | 488.28 KiB | ⬆\uFE0F+488.28 KiB |\n" - + "| feed-id-m3 | | | | |\n" - + "| | key3 | -100 bytes | -976.56 KiB | ⬇\uFE0F-976.46 KiB |\n" - + "| | key4 | -100 bytes | -976.56 KiB | ⬇\uFE0F-976.46 KiB |\n" - + "

List of " - + ValidationPerformanceCollector.MEMORY_USAGE_COMPARE_MAX - + " datasets(memory has decreased).

\n\n" - + "| Dataset ID | Snapshot Key(Used Memory) | Reference | Latest | Difference |\n" - + "|-----------------------------|-------------------|----------------|----------------|----------------|\n" - + "| feed-id-m3 | | | | |\n" - + "| | key3 | -100 bytes | -976.56 KiB | ⬇️-976.46 KiB |\n" - + "| | key4 | -100 bytes | -976.56 KiB | ⬇️-976.46 KiB |\n" - + "| feed-id-m2 | | | | |\n" - + "| | key1 | 0 bytes | 488.28 KiB | ⬆️+488.28 KiB |\n" - + "| | key2 | 0 bytes | 488.28 KiB | ⬆️+488.28 KiB |\n" - + "

List of " - + ValidationPerformanceCollector.MEMORY_USAGE_COMPARE_MAX - + " datasets(no reference available).

\n\n" - + "| Dataset ID | Snapshot Key(Used Memory) | Reference | Latest |\n" - + "|-----------------------------|-------------------|----------------|----------------|\n" - + "| feed-id-m1 | | | |\n" - + "| | key1 | -9.54 MiB | N/A |\n" - + "| | key2 | 0 bytes | N/A |\n" + + "| Average | -- | 0 bytes | 488.28 KiB | ⬆️+488.28 KiB |\n" + + "| Median | -- | 0 bytes | 488.28 KiB | ⬆️+488.28 KiB |\n" + + "| Standard Deviation | -- | 0 bytes | 0 bytes | ⬇️0 bytes |\n" + + "| Minimum in References Reports | feed-id-m2 | 0 bytes | 488.28 KiB | ⬆\uFE0F+488.28 KiB |\n" + + "| Maximum in Reference Reports | feed-id-m2 | 0 bytes | 488.28 KiB | ⬆\uFE0F+488.28 KiB |\n" + + "| Minimum in Latest Reports | feed-id-m2 | 0 bytes | 488.28 KiB | ⬆\uFE0F+488.28 KiB |\n" + + "| Maximum in Latest Reports | feed-id-m2 | 0 bytes | 488.28 KiB | ⬆\uFE0F+488.28 KiB |\n" + "
\n"; // Assert that the generated log string matches the expected log string assertThat(logString).isEqualTo(expectedLogString);