Initial commit

.gitignore

@@ -0,0 +1,62 @@
+*.log
+/out/
+/work/
+.idea/
+*/build/
+xcuserdata/
+*.iws
+.DS_Store
+/bamboo/TEST*.xml
+/bamboo/junit*.properties
+/pilots/ionic/work/
+*.suo
+*.csproj.user
+gridgain*.jar
+*.o
+*.lo
+*.obj
+.deps
+atlassian-ide-plugin.xml
+*.iml
+target
+target/*
+/libs
+pom-installed.xml
+bin/include/visorui
+bin/include/visorcmd
+bin/include/visor-common
+*.gar
+/incubator-ignite
+git-patch-prop-local.sh
+
+#Visual Studio files
+*.[Oo]bj
+*.user
+*.aps
+*.pch
+*.vspscc
+*.vssscc
+*_i.c
+*_p.c
+*.ncb
+*.suo
+*.tlb
+*.tlh
+*.bak
+*.[Cc]ache
+*.ilk
+*.log
+*.lib
+*.sbr
+*.sdf
+*.opensdf
+*.db
+*.opendb
+.vs
+ipch/
+[Oo]bj/
+[Bb]in
+[Dd]ebug*/
+[Rr]elease*/
+packages
+*.nupkg

README.md

@@ -0,0 +1,27 @@
+# Simple heap analyzer for java heap
+This is simple heap analyzer working from bottom to top (from biggest classes to their holders) based on https://github.com/aragozin/jvm-tools
+
+## Intro
+1) https://github.com/aragozin/jvm-tools and mvn clean install for installing dependency
+2) run HeapAnalyzer from u IDE or make some jar
+## Description
+This tool parse hava heaps (compressed too) and do:
+1) Scan all heap to build class histogram and find <N> biggest (all instance counting) classes
+2) Scan all heap to make Map<BiggestClass, Map<instanceId, size>
+3) Scan all heap to make Map<{HolderClass,BiggestClass}, Map<holderInstanceId, sumSize>> where sumSize = size of all holded BiggestClass instances and sum of all sizes of holding they HolderInstances.
+
+For example:
+1) from histo we get that biggest classes is long[] and char[]
+2) build map for long[] with all instances-sizes and map for char[] with all instances-sizes
+3) iterate throught all objects in heap and if they point to any instance in map from (2) - collect it as, for example String,char[]->Map<StringInstanceId, sumOfStringObjectAndCharArray>
+
+4) Repeat step 3 for <M> times to get root holders (not really GC root, but after 2-3 steps we go from primitives to business objects and then we can debug it in runtime or better understand heap dump). After each iteration we get longer path from big object to GC root.
+TODO: add path filters to filter some primitive classes or paths
+
+## Parameters
+run HeapAnalyzer with command line arguments:
+1) path to heap dump, mandatory, no default value.
+2) BIGGEST_CLASS_TO_ANALYSE - number of biggest class to analyze, optional, default 2.
+3) HOLDING_TREE_HEIGHT - number of step 4 iteration, optional, default 2,
+4) HOLDING_TREE_WIDTH - after each step only biggest HOLDING_TREE_WIDTH path (with biggest total size) will be processed in next iteration, optional, default 20.
+

pom.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>java</groupId>
+    <artifactId>heap-analyzer</artifactId>
+    <version>1.0-SNAPSHOT</version>
+    <dependencies>
+        <dependency>
+            <groupId>org.gridkit.jvmtool</groupId>
+            <artifactId>hprof-heap</artifactId>
+            <version>0.9-SNAPSHOT</version>
+        </dependency>
+        <dependency>
+            <groupId>org.eclipse.collections</groupId>
+            <artifactId>eclipse-collections</artifactId>
+            <version>8.1.0</version>
+        </dependency>
+    </dependencies>
+
+</project>

src/main/java/HeapAnalyzer/ClassRecord.java

@@ -0,0 +1,48 @@
+package HeapAnalyzer;
+
+import org.netbeans.lib.profiler.heap.Instance;
+import org.netbeans.lib.profiler.heap.JavaClass;
+
+public class ClassRecord {
+    public JavaClass jClass;
+    public int instanceCount;
+    public long totalSize;
+
+    public ClassRecord(JavaClass jClass) {
+        this.jClass = jClass;
+    }
+
+    public void addInstance(Instance i) {
+        ++instanceCount;
+        totalSize += i.getSize();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        ClassRecord that = (ClassRecord) o;
+
+        if (instanceCount != that.instanceCount) return false;
+        if (totalSize != that.totalSize) return false;
+        return jClass != null ? jClass.equals(that.jClass) : that.jClass == null;
+    }
+
+    @Override
+    public int hashCode() {
+        int result = jClass != null ? jClass.hashCode() : 0;
+        result = 31 * result + instanceCount;
+        result = 31 * result + (int) (totalSize ^ (totalSize >>> 32));
+        return result;
+    }
+
+    @Override
+    public String toString() {
+        return "ClassRecord{" +
+            "jClass=" + jClass.getName() +
+            ", instanceCount=" + instanceCount +
+            ", totalSize=" + totalSize +
+            '}';
+    }
+}

src/main/java/HeapAnalyzer/HeapAnalyzer.java

@@ -0,0 +1,223 @@
+package HeapAnalyzer;
+
+import HeapAnalyzer.util.TextTable;
+import org.eclipse.collections.api.iterator.IntIterator;
+import org.eclipse.collections.impl.map.mutable.primitive.LongIntHashMap;
+
+import org.netbeans.lib.profiler.heap.FieldValue;
+import org.netbeans.lib.profiler.heap.Heap;
+import org.netbeans.lib.profiler.heap.HeapFactory;
+import org.netbeans.lib.profiler.heap.Instance;
+import org.netbeans.lib.profiler.heap.JavaClass;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class HeapAnalyzer {
+    /**
+     * Top classes count shitch use heap.
+     */
+    public static int TOP_CLASS = 2;
+
+    /**
+     * Levels of analys deepth.
+     */
+    public static int HOLDING_TREE_HEIGHT = 2;
+
+    /**
+     * With of analys step
+     */
+    public static int HOLDING_TREE_WIDTH = 20;
+
+
+    public static void log(String text) {
+        System.out.println(new Date() + " " + text);
+    }
+
+    public static Heap load(String filename) throws IOException {
+        File sFile = new File(filename);
+        if (!sFile.exists())
+            throw new IllegalArgumentException("File " + filename + " doesn't exist!");
+        if (!sFile.isFile())
+            throw new IllegalArgumentException("File " + filename + " not a regular file!");
+        log("Loading " + filename + "...");
+        return HeapFactory.createFastHeap(sFile);
+    }
+
+    public static List<ClassRecord> collectBiggestClasses(Heap heap) {
+        HistoAnalyzer histo = new HistoAnalyzer(true);
+
+        for (Instance i : heap.getAllInstances())
+            histo.feed(i);
+
+        List<ClassRecord> ht = new ArrayList<ClassRecord>(histo.getHisto());
+        Collections.sort(ht, HistoAnalyzer.BY_SIZE_DESC);
+
+        return ht;
+
+    }
+
+    public static Map<PathKey, LongIntHashMap> collectBiggestObjects(Heap heap, List<ClassRecord> biggestClasses) {
+        Map<JavaClass, LongIntHashMap> biggestClassObjects = new HashMap();
+        for (ClassRecord classRecord : biggestClasses) {
+            biggestClassObjects.put(classRecord.jClass, new LongIntHashMap((int) classRecord.instanceCount));
+        }
+        JavaClass tClass;
+        LongIntHashMap tMap;
+        for (Instance i : heap.getAllInstances()) {
+            tClass = i.getJavaClass();
+            tMap = biggestClassObjects.get(tClass);
+            if (tMap != null) {
+                assert i.getSize() < Integer.MAX_VALUE : "Object greater than Integer.MAX_VALUE, what u plat to optimize?";
+                tMap.put(i.getInstanceId(), (int)i.getSize());
+            }
+        }
+
+        Map<PathKey, LongIntHashMap> result = new HashMap();
+        for(ClassRecord cr : biggestClasses)
+            result.put(new PathKey(cr), biggestClassObjects.get(cr.jClass));
+
+        return result;
+    }
+
+    public static void addResult(Instance i, boolean addSelfSize, int holdedSize, PathKey holdedPR, Map<PathKey, LongIntHashMap> result) {
+
+        PathKey newPathRecord = new PathKey(i.getJavaClass(), holdedPR);
+        LongIntHashMap pathMap = result.get(newPathRecord);
+        if (pathMap == null)
+            result.put(newPathRecord, pathMap = new LongIntHashMap());
+        long instanceId = i.getInstanceId();
+        int holded = pathMap.get(instanceId);
+        if (addSelfSize)
+            holded += i.getSize();
+        holded += holdedSize;
+        pathMap.put(instanceId, holded);
+
+    }
+
+    public static Map<PathKey, LongIntHashMap> doStep(Heap head, Map<PathKey, LongIntHashMap> prevLvl) {
+
+        Map<PathKey, LongIntHashMap> result = new HashMap();
+
+        // Prepare arrays for fast iterations
+        PathKey paths[] = new PathKey[prevLvl.size()];
+        LongIntHashMap maps[] = new LongIntHashMap[prevLvl.size()];
+        int i = 0;
+        for (Map.Entry<PathKey, LongIntHashMap> entry : prevLvl.entrySet()) {
+            paths[i] = entry.getKey();
+            maps[i++] = entry.getValue();
+        }
+
+        long tHoldedSize;
+        long tFieldRef;
+        boolean countInstanceSelfSize;
+        for (Instance instance : head.getAllInstances()) {
+            countInstanceSelfSize = false;
+
+            List<FieldValue> values = instance.getFieldValues();
+            if (values != null) {
+                for (FieldValue fv : values) {
+                    if (fv.getField().getType().getName() == "object") {
+                        String value = fv.getValue();
+                        //System.out.println(value);
+                        if (value != null) {
+                            tFieldRef = Long.valueOf(value);
+                            for (i = 0;i < maps.length;i++) {
+                                tHoldedSize = maps[i].get(tFieldRef);
+                                if (tHoldedSize > 0L) {
+                                    addResult (instance, !countInstanceSelfSize, (int)tHoldedSize, paths[i], result);
+                                    countInstanceSelfSize = true;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+
+        return result;
+    }
+
+    public static void printResult(ResultRow[] result) {
+        System.out.println(new Date() + " Next step:");
+        for (ResultRow rr : result)
+            System.out.println(rr);
+    }
+
+    public static ResultRow[] collectResult (Map<PathKey, LongIntHashMap> lvl) {
+
+        PathKey tPK;
+        LongIntHashMap tEntryMap;
+        long tSizeSum;
+        ResultRow[] result = new ResultRow[lvl.size()];
+        int i = 0;
+        for (Map.Entry<PathKey, LongIntHashMap> entry : lvl.entrySet()) {
+            tSizeSum = 0;
+
+            LongIntHashMap instancesMap = entry.getValue();
+            IntIterator iterator = instancesMap.intIterator();
+            while (iterator.hasNext()) {
+                tSizeSum += iterator.next();
+            }
+            result[i++] = new ResultRow(entry.getKey().path, instancesMap.size(), tSizeSum);
+        }
+        Arrays.sort(result, ResultRow.BY_SIZE_DESC);
+        return result;
+    }
+
+    public static void printBiggestClasses(List<ClassRecord> biggestClasses) {
+        TextTable tt = new TextTable();
+        int n = 0;
+        for(ClassRecord cr: biggestClasses.subList(0, 500)) {
+            tt.addRow("" + (++n), " " + cr.totalSize, " " + cr.instanceCount, " " + cr.jClass.getName());
+        }
+        System.out.println(tt.formatTextTableUnbordered(1000));
+    }
+
+    public static void main(String[] args) throws IOException {
+
+        if (args.length < 1 || args.length > 4) {
+            System.out.println("Start with parameters: <path/to/dump> [<BIGGEST_CLASS_TO_ANALYSE> <HOLDING_TREE_HEIGHT> <HOLDING_TREE_WIDTH>]");
+            System.exit(1);
+        }
+        String dumppath = args[0];
+        if (args.length > 1)
+            TOP_CLASS = Integer.parseInt(args[1]);
+        if (args.length > 2)
+            HOLDING_TREE_HEIGHT = Integer.parseInt(args[2]);
+        if (args.length > 3)
+            HOLDING_TREE_WIDTH = Integer.parseInt(args[3]);
+
+        dumppath = "/mnt/ssd/work/gg/FD/FD-5122_memory_consumption/heap18175_s1.hprof";
+
+        Heap heap = load(dumppath);
+        log( "Heap loaded. Searching for biggest classes...");
+
+        List<ClassRecord> biggestClasses = collectBiggestClasses(heap);
+        printBiggestClasses(biggestClasses);
+        log("Biggest classes collected. Mapping...");
+
+        biggestClasses = biggestClasses.subList(0, Math.min(biggestClasses.size(), TOP_CLASS));
+        Map<PathKey, LongIntHashMap> biggestClassObjects = collectBiggestObjects(heap, biggestClasses);
+        log("Biggest classes mapped, search for holding objects...");
+        collectResult(biggestClassObjects);
+
+        Map<PathKey, LongIntHashMap> nextLvl = biggestClassObjects;
+        for (int i = 0; i < HOLDING_TREE_HEIGHT; i++) {
+            nextLvl = doStep(heap, nextLvl);
+            ResultRow[] result = collectResult(nextLvl);
+            printResult(result);
+            result = Arrays.copyOf(result, HOLDING_TREE_WIDTH);
+        }
+        log("Done");
+
+    }
+}