Filter out unused dependencies in Go (#119)

README.md

@@ -175,7 +175,7 @@ The exclude pattern can be configured in the [Extension Settings](#extension-set
 
 ### Go Projects
 
-Behind the scenes, the JFrog VS Code Extension scans all the project dependencies, both direct and indirect (transitive), even if they are not declared in the project's go.mod. It builds the Go dependencies tree by running `go mod graph`. Therefore, please make sure to have Go CLI in your system PATH.
+Behind the scenes, the JFrog VS Code Extension scans all the project dependencies, both direct and indirect (transitive), even if they are not declared in the project's go.mod. It builds the Go dependencies tree by running `go mod graph` and intersecting the results with `go list -f '{{with .Module}}{{.Path}} {{.Version}}{{end}}' all` command. Therefore, please make sure to have Go CLI in your system PATH.
 
 ### Maven Projects
 

src/main/treeDataProviders/dependenciesTree/dependenciesRoot/goTree.ts

@@ -1,7 +1,6 @@
-import * as exec from 'child_process';
+import { ComponentDetails } from 'jfrog-client-js';
 import * as Collections from 'typescript-collections';
 import * as vscode from 'vscode';
-import { ComponentDetails } from 'jfrog-client-js';
 import { GeneralInfo } from '../../../types/generalInfo';
 import { GoUtils } from '../../../utils/goUtils';
 import { ScanUtils } from '../../../utils/scanUtils';
@@ -11,9 +10,6 @@ import { RootNode } from './rootTree';
 
 export class GoTreeNode extends RootNode {
     private static readonly COMPONENT_PREFIX: string = 'go://';
-
-    private _dependenciesMap: Map<string, string[]> = new Map();
-
     constructor(
         workspaceFolder: string,
         private _componentsToScan: Collections.Set<ComponentDetails>,
@@ -24,14 +20,13 @@ export class GoTreeNode extends RootNode {
     }
 
     public async refreshDependencies(quickScan: boolean) {
+        let goModGraph: PackageDependencyPair[] = [];
         let goList: string[] = [];
         let rootPackageName: string = '';
         try {
-            goList = ScanUtils.executeCmd('go mod graph', this.workspaceFolder)
-                .toString()
-                .split(/\s+/);
-            goList.pop(); // Remove the last new line
-            rootPackageName = this.getModuleName();
+            rootPackageName = this.getRootPackageName();
+            goModGraph = this.runGoModGraph();
+            goList = this.runGoList();
         } catch (error) {
             this._treesManager.logManager.logError(error, !quickScan);
             this.label = this.workspaceFolder + ' [Not installed]';
@@ -40,52 +35,104 @@ export class GoTreeNode extends RootNode {
         }
         this.generalInfo = new GeneralInfo(rootPackageName, '', ['None'], this.workspaceFolder, GoUtils.PKG_TYPE);
         this.label = rootPackageName;
-        if (goList.length === 0) {
+        if (goModGraph.length === 0) {
             return;
         }
-        this.buildDependenciesMapAndDirectDeps(goList);
-        this.children.forEach(child => this.populateDependenciesTree(child, quickScan));
+        let dependenciesMap: Map<string, string[]> = this.buildDependenciesMapAndDirectDeps(goModGraph, goList);
+        this.children.forEach(child => this.populateDependenciesTree(dependenciesMap, child, quickScan));
+    }
+
+    /**
+     * run "go list -m" to get the name of the root module.
+     * @returns the root package name
+     */
+    private getRootPackageName(): string {
+        return ScanUtils.executeCmd('go list -m', this.workspaceFolder)
+            .toString()
+            .trim();
     }
 
-    private buildDependenciesMapAndDirectDeps(goList: string[]) {
-        let i: number = 0;
+    /**
+     * Run "go mod graph" in order to create the dependency tree later on.
+     * @returns a list of package to dependency pairs
+     */
+    private runGoModGraph(): PackageDependencyPair[] {
+        let goModGraphOutput: string[] = ScanUtils.executeCmd('go mod graph', this.workspaceFolder)
+            .toString()
+            .split(/\s+/);
+        goModGraphOutput.pop(); // Remove the last new line
+
+        // For a given index i, if i is even (i%2==0) goModGraphOutput[i] is the package that depends on goModGraphOutput[i+1]: goModGraphOutput[i] -> goModGraphOutput[i+1].
+        // The first lines of the even indices contain no versions. Those are the direct dependencies.
+        let results: PackageDependencyPair[] = [];
+        for (let i: number = 0; i < goModGraphOutput.length; i += 2) {
+            results.push(new PackageDependencyPair(goModGraphOutput[i], goModGraphOutput[i + 1]));
+        }
+        return results;
+    }
+
+    /**
+     * Run "go list" to retrieve a list of dependencies which are actually in use in the project.
+     * @returns "go list" results.
+     */
+    private runGoList(): string[] {
+        return ScanUtils.executeCmd(`go list -f "{{with .Module}}{{.Path}} {{.Version}}{{end}}" all`, this.workspaceFolder)
+            .toString()
+            .split(/\n/);
+    }
+
+    private buildDependenciesMapAndDirectDeps(goModGraph: PackageDependencyPair[], goList: string[]): Map<string, string[]> {
+        let goModGraphIndex: number = 0;
 
         // Populate direct dependencies
         let directDependenciesGeneralInfos: GeneralInfo[] = [];
-        for (; i < goList.length && !goList[i].includes('@'); i += 2) {
-            let nameVersionTuple: string[] = this.getNameVersionTuple(goList[i + 1]);
+        for (; goModGraphIndex < goModGraph.length && !goModGraph[goModGraphIndex].package.includes('@'); goModGraphIndex++) {
+            let nameVersionTuple: string[] = this.getNameVersionTuple(goModGraph[goModGraphIndex].dependency);
             directDependenciesGeneralInfos.push(new GeneralInfo(nameVersionTuple[0], nameVersionTuple[1], ['None'], '', GoUtils.PKG_TYPE));
         }
 
+        // Create a set of packages that are actually in use in the project
+        let goListPackages: Set<string> = new Set<string>();
+        goList.forEach((dependency: string) => {
+            goListPackages.add(dependency.replace(' ', '@'));
+        });
+
         // Build dependencies map
-        for (; i < goList.length; i += 2) {
-            let dependency: string[] = this._dependenciesMap.get(goList[i]) || [];
-            dependency.push(goList[i + 1]);
-            this._dependenciesMap.set(goList[i], dependency);
+        let dependenciesMap: Map<string, string[]> = new Map();
+        for (; goModGraphIndex < goModGraph.length; goModGraphIndex++) {
+            let dependency: string[] = dependenciesMap.get(goModGraph[goModGraphIndex].package) || [];
+            if (!goListPackages.has(goModGraph[goModGraphIndex].dependency)) {
+                // If the dependency is included in "go mod graph", but isn't included in "go mod -m all", it means that it's not in use by the project.
+                // It can therefore be ignored.
+                continue;
+            }
+            dependency.push(goModGraph[goModGraphIndex].dependency);
+            dependenciesMap.set(goModGraph[goModGraphIndex].package, dependency);
         }
 
         // Add direct dependencies to tree
         directDependenciesGeneralInfos.forEach(generalInfo => {
-            this.addChild(new DependenciesTreeNode(generalInfo, this.getTreeCollapsibleState(generalInfo)));
+            this.addChild(new DependenciesTreeNode(generalInfo, this.getTreeCollapsibleState(dependenciesMap, generalInfo)));
         });
+        return dependenciesMap;
     }
 
-    private populateDependenciesTree(dependenciesTreeNode: DependenciesTreeNode, quickScan: boolean) {
+    private populateDependenciesTree(dependenciesMap: Map<string, string[]>, dependenciesTreeNode: DependenciesTreeNode, quickScan: boolean) {
         if (this.hasLoop(dependenciesTreeNode)) {
             return;
         }
         this.addComponentToScan(dependenciesTreeNode, quickScan);
         let childDependencies: string[] =
-            this._dependenciesMap.get(dependenciesTreeNode.generalInfo.artifactId + '@v' + dependenciesTreeNode.generalInfo.version) || [];
+            dependenciesMap.get(dependenciesTreeNode.generalInfo.artifactId + '@v' + dependenciesTreeNode.generalInfo.version) || [];
         childDependencies.forEach(childDependency => {
             let nameVersionTuple: string[] = this.getNameVersionTuple(childDependency);
             let generalInfo: GeneralInfo = new GeneralInfo(nameVersionTuple[0], nameVersionTuple[1], ['None'], '', GoUtils.PKG_TYPE);
             let grandchild: DependenciesTreeNode = new DependenciesTreeNode(
                 generalInfo,
-                this.getTreeCollapsibleState(generalInfo),
+                this.getTreeCollapsibleState(dependenciesMap, generalInfo),
                 dependenciesTreeNode
             );
-            this.populateDependenciesTree(grandchild, quickScan);
+            this.populateDependenciesTree(dependenciesMap, grandchild, quickScan);
         });
     }
 
@@ -101,13 +148,6 @@ export class GoTreeNode extends RootNode {
         return false;
     }
 
-    private getModuleName(): string {
-        return exec
-            .execSync('go list -m', { cwd: this.workspaceFolder })
-            .toString()
-            .trim();
-    }
-
     private addComponentToScan(dependenciesTreeNode: DependenciesTreeNode, quickScan: boolean) {
         let componentId: string = dependenciesTreeNode.generalInfo.artifactId + ':' + dependenciesTreeNode.generalInfo.version;
         if (!quickScan || !this._treesManager.scanCacheManager.isValid(componentId)) {
@@ -120,9 +160,21 @@ export class GoTreeNode extends RootNode {
         return [split[0], split[1]];
     }
 
-    private getTreeCollapsibleState(generalInfo: GeneralInfo): vscode.TreeItemCollapsibleState {
-        return this._dependenciesMap.has(generalInfo.artifactId + '@v' + generalInfo.version)
+    private getTreeCollapsibleState(dependenciesMap: Map<string, string[]>, generalInfo: GeneralInfo): vscode.TreeItemCollapsibleState {
+        return dependenciesMap.has(generalInfo.artifactId + '@v' + generalInfo.version)
             ? vscode.TreeItemCollapsibleState.Collapsed
             : vscode.TreeItemCollapsibleState.None;
     }
 }
+
+class PackageDependencyPair {
+    constructor(private _package: string, private _dependency: string) {}
+
+    public get package() {
+        return this._package;
+    }
+
+    public get dependency() {
+        return this._dependency;
+    }
+}

src/test/resources/go/dependency/go.mod

@@ -3,6 +3,6 @@ module github.com/shield/black-widow
 go 1.13
 
 require (
-	github.com/jfrog/gofrog v1.0.5
-	github.com/opencontainers/runc v1.0.0-rc2
+	github.com/jfrog/jfrog-cli-core v1.9.0
+	github.com/jfrog/jfrog-client-go v0.26.1 // indirect
 )