simplifly memory usage report

output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/DatasetMemoryUsage.java

@@ -1,23 +1,37 @@
 package org.mobilitydata.gtfsvalidator.outputcomparator.io;
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 import org.mobilitydata.gtfsvalidator.performance.MemoryUsage;
 
 public class DatasetMemoryUsage {
 
   private String datasetId;
-  private String key;
-  private MemoryUsage referenceMemoryUsage;
-  private MemoryUsage latestMemoryUsage;
+  private List<MemoryUsage> referenceMemoryUsage;
+  private List<MemoryUsage> latestMemoryUsage;
+  private Map<String, Long> referenceUsedMemoryByKey = Collections.unmodifiableMap(new HashMap<>());
+  private Map<String, Long> latestUsedMemoryByKey = Collections.unmodifiableMap(new HashMap<>());
 
   public DatasetMemoryUsage(
-      String datasetId, MemoryUsage referenceMemoryUsage, MemoryUsage latestMemoryUsage) {
+      String datasetId,
+      List<MemoryUsage> referenceMemoryUsage,
+      List<MemoryUsage> latestMemoryUsage) {
     this.datasetId = datasetId;
-    this.key = referenceMemoryUsage != null ? referenceMemoryUsage.getKey() : null;
-    if (key == null) {
-      this.key = latestMemoryUsage.getKey() != null ? latestMemoryUsage.getKey() : null;
-    }
     this.referenceMemoryUsage = referenceMemoryUsage;
     this.latestMemoryUsage = latestMemoryUsage;
+    if (referenceMemoryUsage != null) {
+      this.referenceUsedMemoryByKey =
+          referenceMemoryUsage.stream()
+              .collect(Collectors.toUnmodifiableMap(MemoryUsage::getKey, MemoryUsage::usedMemory));
+    }
+    if (latestMemoryUsage != null) {
+      this.latestUsedMemoryByKey =
+          latestMemoryUsage.stream()
+              .collect(Collectors.toUnmodifiableMap(MemoryUsage::getKey, MemoryUsage::usedMemory));
+    }
   }
 
   public String getDatasetId() {
@@ -28,23 +42,27 @@ public void setDatasetId(String datasetId) {
     this.datasetId = datasetId;
   }
 
-  public MemoryUsage getReferenceMemoryUsage() {
+  public List<MemoryUsage> getReferenceMemoryUsage() {
     return referenceMemoryUsage;
   }
 
-  public void setReferenceMemoryUsage(MemoryUsage referenceMemoryUsage) {
+  public void setReferenceMemoryUsage(List<MemoryUsage> referenceMemoryUsage) {
     this.referenceMemoryUsage = referenceMemoryUsage;
   }
 
-  public MemoryUsage getLatestMemoryUsage() {
+  public List<MemoryUsage> getLatestMemoryUsage() {
     return latestMemoryUsage;
   }
 
-  public void setLatestMemoryUsage(MemoryUsage latestMemoryUsage) {
+  public void setLatestMemoryUsage(List<MemoryUsage> latestMemoryUsage) {
     this.latestMemoryUsage = latestMemoryUsage;
   }
 
-  public String getKey() {
-    return key;
+  public Map<String, Long> getReferenceUsedMemoryByKey() {
+    return referenceUsedMemoryByKey;
+  }
+
+  public Map<String, Long> getLatestUsedMemoryByKey() {
+    return latestUsedMemoryByKey;
   }
 }

output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/UsedMemoryIncreasedComparator.java

@@ -0,0 +1,57 @@
+package org.mobilitydata.gtfsvalidator.outputcomparator.io;
+
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Comparator to compare two {@link DatasetMemoryUsage} objects based on the difference between the
+ * used memory of the two objects. The difference is calculated by comparing the used memory of the
+ * two objects for each key present in both objects. If a key is present in one object but not in
+ * the other, the key it is ignored. This comparator is used to sort DatasetMemoryUsage by the
+ * minimum difference between the used memory of the two. This means the order is by the dataset
+ * validation that increased the memory.
+ */
+public class UsedMemoryIncreasedComparator implements Comparator<DatasetMemoryUsage> {
+
+  @Override
+  public int compare(DatasetMemoryUsage o1, DatasetMemoryUsage o2) {
+    if (o1 == o2) {
+      return 0;
+    }
+    if (o1 == null || o2 == null) {
+      return o1 == null ? -1 : 1;
+    }
+    if (o1.getReferenceMemoryUsage() == null && o2.getLatestMemoryUsage() == null) {
+      return 0;
+    }
+    if (o1.getReferenceMemoryUsage() == null || o2.getLatestMemoryUsage() == null) {
+      return o1.getReferenceMemoryUsage() == null ? -1 : 1;
+    }
+
+    long o1MinDiff =
+        getMinimumDifferenceByKey(o1.getReferenceUsedMemoryByKey(), o1.getLatestUsedMemoryByKey());
+    long o2MinDiff =
+        getMinimumDifferenceByKey(o2.getReferenceUsedMemoryByKey(), o2.getLatestUsedMemoryByKey());
+    return Long.compare(o1MinDiff, o2MinDiff);
+  }
+
+  private long getMinimumDifferenceByKey(
+      Map<String, Long> referenceMemoryUsage, Map<String, Long> latestMemoryUsage) {
+    Set<String> keys = new HashSet<>();
+    keys.addAll(latestMemoryUsage.keySet());
+    keys.addAll(referenceMemoryUsage.keySet());
+    return keys.stream()
+        .filter(key -> latestMemoryUsage.containsKey(key) && referenceMemoryUsage.containsKey(key))
+        .filter(key -> latestMemoryUsage.get(key) - referenceMemoryUsage.get(key) != 0)
+        .mapToLong(key -> referenceMemoryUsage.get(key) - latestMemoryUsage.get(key))
+        .min()
+        .orElse(Long.MAX_VALUE);
+  }
+
+  @Override
+  public Comparator<DatasetMemoryUsage> reversed() {
+    return Comparator.super.reversed();
+  }
+}

output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollector.java

@@ -1,26 +1,32 @@
 package org.mobilitydata.gtfsvalidator.outputcomparator.io;
 
 import java.util.*;
-import java.util.stream.Collectors;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.mobilitydata.gtfsvalidator.model.ValidationReport;
 import org.mobilitydata.gtfsvalidator.outputcomparator.model.report.ValidationPerformance;
 import org.mobilitydata.gtfsvalidator.performance.MemoryUsage;
 
 public class ValidationPerformanceCollector {
 
-  private static final int MEMORY_USAGE_COMPARE_MAX = 20;
+  public static final int MEMORY_USAGE_COMPARE_MAX = 25;
   private final Map<String, Double> referenceTimes;
   private final Map<String, Double> latestTimes;
-  private final Map<String, BoundedPriorityQueue<DatasetMemoryUsage>>
-      largestFirstMemoryUsageBySourceId;
-  private final Map<String, BoundedPriorityQueue<DatasetMemoryUsage>>
-      smallestFirstMemoryMapBySourceId;
+  private final BoundedPriorityQueue<DatasetMemoryUsage> datasetsDecreasedMemoryUsage;
+  private final BoundedPriorityQueue<DatasetMemoryUsage> datasetsIncreasedMemoryUsage;
 
   public ValidationPerformanceCollector() {
     this.referenceTimes = new HashMap<>();
     this.latestTimes = new HashMap<>();
-    this.largestFirstMemoryUsageBySourceId = new HashMap<>();
-    this.smallestFirstMemoryMapBySourceId = new HashMap<>();
+    this.datasetsDecreasedMemoryUsage =
+        new BoundedPriorityQueue<>(
+            MEMORY_USAGE_COMPARE_MAX,
+            MEMORY_USAGE_COMPARE_MAX,
+            (new UsedMemoryIncreasedComparator().reversed()));
+    this.datasetsIncreasedMemoryUsage =
+        new BoundedPriorityQueue<>(
+            MEMORY_USAGE_COMPARE_MAX,
+            MEMORY_USAGE_COMPARE_MAX,
+            new UsedMemoryIncreasedComparator());
   }
 
   public void addReferenceTime(String sourceId, Double time) {
@@ -201,68 +207,60 @@ public String generateLogString() {
           .append("\n\n");
     }
 
-    if (smallestFirstMemoryMapBySourceId.size() > 0
-        || largestFirstMemoryUsageBySourceId.size() > 0) {
-      b.append("<summary><strong>📜 Memory Consumption</strong></summary>\n");
-      addMemoryUsageReport(smallestFirstMemoryMapBySourceId, "decreased", b);
-      addMemoryUsageReport(largestFirstMemoryUsageBySourceId, "increased", b);
-    }
-
     b.append("</details>\n\n");
 
+    if (datasetsIncreasedMemoryUsage.size() > 0 || datasetsDecreasedMemoryUsage.size() > 0) {
+      b.append("<details>\n");
+      b.append("<summary><strong>📜 Memory Consumption</strong></summary>\n");
+      addMemoryUsageReport(datasetsIncreasedMemoryUsage, "increased", b);
+      addMemoryUsageReport(datasetsDecreasedMemoryUsage, "decreased", b);
+      b.append("</details>\n");
+    }
     return b.toString();
   }
 
   private void addMemoryUsageReport(
-      Map<String, BoundedPriorityQueue<DatasetMemoryUsage>> queueMap,
-      String order,
-      StringBuilder b) {
+      BoundedPriorityQueue<DatasetMemoryUsage> queue, String order, StringBuilder b) {
     b.append(
             String.format(
-                "<p>List of %s datasets where memory has %s .</p>\n",
-                MEMORY_USAGE_COMPARE_MAX, order))
+                "<p>List of %s datasets where memory has %s.</p>", MEMORY_USAGE_COMPARE_MAX, order))
         .append("\n")
         .append(
-            "| Key(Used Memory)                      | Dataset ID        | Reference (s)  | Latest (s)     | Difference (s) |\n")
+            "| Dataset ID                  | Snapshot Key(Used Memory)  | Reference (s)  | Latest (s)     | Difference (s) |\n")
         .append(
             "|-----------------------------|-------------------|----------------|----------------|----------------|\n");
-
-    queueMap.keySet().stream()
+    List<DatasetMemoryUsage> sortedList = new ArrayList<>(queue);
+    sortedList.sort(queue.comparator());
+    sortedList.stream()
         .forEachOrdered(
-            sourceId -> {
-              var pq = queueMap.get(sourceId);
-              List<DatasetMemoryUsage> datasetMemoryUsages =
-                  Arrays.asList(pq.toArray(new DatasetMemoryUsage[pq.size()]));
-              Collections.sort(datasetMemoryUsages, pq.comparator());
-              generateMemoryLogByKey(datasetMemoryUsages, b);
+            datasetMemoryUsage -> {
+              generateMemoryLogByKey(datasetMemoryUsage, b);
             });
   }
 
   private static void generateMemoryLogByKey(
-      List<DatasetMemoryUsage> memoryIncreases, StringBuilder b) {
-    memoryIncreases.stream()
-        .forEachOrdered(
-            item -> {
-              String usedMemoryDiff =
-                  getMemoryDiff(
-                      item.getReferenceMemoryUsage() != null
-                          ? item.getReferenceMemoryUsage().usedMemory()
-                          : null,
-                      item.getLatestMemoryUsage() != null
-                          ? item.getLatestMemoryUsage().usedMemory()
-                          : null);
-              b.append(
-                  String.format(
-                      "| %s | %s | %s | %s | %s |\n",
-                      item.getKey(),
-                      item.getDatasetId(),
-                      item.getReferenceMemoryUsage() != null
-                          ? item.getReferenceMemoryUsage().usedMemory()
-                          : "-",
-                      item.getLatestMemoryUsage() != null
-                          ? item.getLatestMemoryUsage().usedMemory()
-                          : "-",
-                      usedMemoryDiff));
+      DatasetMemoryUsage datasetMemoryUsage, StringBuilder b) {
+    AtomicBoolean isFirst = new AtomicBoolean(true);
+    Set<String> keys = new HashSet<>();
+    keys.addAll(datasetMemoryUsage.getReferenceUsedMemoryByKey().keySet());
+    keys.addAll(datasetMemoryUsage.getLatestUsedMemoryByKey().keySet());
+    keys.stream()
+        .forEach(
+            key -> {
+              var reference = datasetMemoryUsage.getReferenceUsedMemoryByKey().get(key);
+              var latest = datasetMemoryUsage.getLatestUsedMemoryByKey().get(key);
+              if (reference != null && latest != null) {
+                String usedMemoryDiff = getMemoryDiff(reference, latest);
+                if (isFirst.get()) {
+                  b.append(
+                      String.format("| %s |  |  |  |  |\n", datasetMemoryUsage.getDatasetId()));
+                  isFirst.set(false);
+                }
+                b.append(
+                    String.format(
+                        "| | %s | %s | %s | %s |\n", key, reference, latest, usedMemoryDiff));
+                isFirst.set(false);
+              }
             });
   }
 
@@ -280,54 +278,20 @@ public void compareValidationReports(
 
   private void compareValidationReportMemoryUsage(
       String sourceId, ValidationReport referenceReport, ValidationReport latestReport) {
-    Set<String> keys =
-        referenceReport.getMemoryUsageRecords() != null
-            ? referenceReport.getMemoryUsageRecords().stream()
-                .map(MemoryUsage::getKey)
-                .collect(Collectors.toSet())
-            : new HashSet<>();
-    if (latestReport.getMemoryUsageRecords() != null) {
-      keys.addAll(
-          latestReport.getMemoryUsageRecords().stream()
-              .map(MemoryUsage::getKey)
-              .collect(Collectors.toSet()));
+    DatasetMemoryUsage datasetMemoryUsage =
+        new DatasetMemoryUsage(
+            sourceId,
+            referenceReport.getMemoryUsageRecords(),
+            latestReport.getMemoryUsageRecords());
+    if (referenceReport.getMemoryUsageRecords() != null
+        && referenceReport.getMemoryUsageRecords().size() > 0
+        && latestReport.getMemoryUsageRecords() != null
+        && latestReport.getMemoryUsageRecords().size() > 0) {
+      datasetsIncreasedMemoryUsage.offer(datasetMemoryUsage);
+      datasetsDecreasedMemoryUsage.offer(datasetMemoryUsage);
+    } else {
+      //      add to not found keys
     }
-    Map<String, MemoryUsage> referenceMap =
-        referenceReport.getMemoryUsageRecords() != null
-            ? referenceReport.getMemoryUsageRecords().stream()
-                .collect(Collectors.toMap(MemoryUsage::getKey, memoryUsage -> memoryUsage))
-            : new HashMap<>();
-    Map<String, MemoryUsage> latestMap =
-        latestReport.getMemoryUsageRecords() != null
-            ? latestReport.getMemoryUsageRecords().stream()
-                .collect(Collectors.toMap(MemoryUsage::getKey, memoryUsage -> memoryUsage))
-            : new HashMap<>();
-    keys.stream()
-        .forEachOrdered(
-            key -> {
-              var datasetMemoryUsage =
-                  new DatasetMemoryUsage(sourceId, referenceMap.get(key), latestMap.get(key));
-              BoundedPriorityQueue<DatasetMemoryUsage> decreasingQueue =
-                  largestFirstMemoryUsageBySourceId.get(sourceId);
-              BoundedPriorityQueue<DatasetMemoryUsage> increasingQueue =
-                  smallestFirstMemoryMapBySourceId.get(sourceId);
-              if (decreasingQueue == null) {
-                decreasingQueue =
-                    new BoundedPriorityQueue<>(
-                        MEMORY_USAGE_COMPARE_MAX,
-                        2,
-                        (new MemoryUsageUsedMemoryComparator()).reversed());
-                largestFirstMemoryUsageBySourceId.put(sourceId, decreasingQueue);
-                increasingQueue =
-                    new BoundedPriorityQueue<>(
-                        MEMORY_USAGE_COMPARE_MAX, 2, new MemoryUsageUsedMemoryComparator());
-                smallestFirstMemoryMapBySourceId.put(sourceId, increasingQueue);
-              }
-              if (referenceMap.containsKey(key) || latestMap.containsKey(key)) {
-                increasingQueue.offer(datasetMemoryUsage);
-                decreasingQueue.offer(datasetMemoryUsage);
-              }
-            });
   }
 
   public List<ValidationPerformance> toReport() {