🐛 improve decompile performance

Signed-off-by: Pranav Gaikwad <[email protected]>

cmd/analyzer/main.go

@@ -20,6 +20,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/swaggest/openapi-go/openapi3"
+	"go.opentelemetry.io/otel/attribute"
 	"gopkg.in/yaml.v2"
 )
 
@@ -112,8 +113,8 @@ func AnalysisCmd() *cobra.Command {
 
 			defer tracing.Shutdown(ctx, log, tp)
 
-			ctx, span := tracing.StartNewSpan(ctx, "main")
-			defer span.End()
+			ctx, mainSpan := tracing.StartNewSpan(ctx, "main")
+			defer mainSpan.End()
 
 			// Get the configs
 			configs, err := provider.GetConfig(settingsFile)
@@ -194,14 +195,19 @@ func AnalysisCmd() *cobra.Command {
 			}
 			// Now that we have all the providers, we need to start them.
 			for name, provider := range needProviders {
-				err := provider.ProviderInit(ctx)
+				initCtx, initSpan := tracing.StartNewSpan(ctx, "init",
+					attribute.Key("provider").String(name))
+				err := provider.ProviderInit(initCtx)
 				if err != nil {
 					errLog.Error(err, "unable to init the providers", "provider", name)
 					os.Exit(1)
 				}
+				initSpan.End()
 			}
 
-			rulesets := eng.RunRules(ctx, ruleSets, selectors...)
+			rulesCtx, rulesSpan := tracing.StartNewSpan(ctx, "run-rules")
+			rulesets := eng.RunRules(rulesCtx, ruleSets, selectors...)
+			rulesSpan.End()
 			eng.Stop()
 
 			for _, provider := range needProviders {

provider/internal/java/provider.go

@@ -20,6 +20,7 @@ import (
 	"github.com/konveyor/analyzer-lsp/lsp/protocol"
 	"github.com/konveyor/analyzer-lsp/output/v1/konveyor"
 	"github.com/konveyor/analyzer-lsp/provider"
+	"github.com/konveyor/analyzer-lsp/tracing"
 	"github.com/swaggest/openapi-go/openapi3"
 	"go.lsp.dev/uri"
 )
@@ -234,7 +235,8 @@ func (p *javaProvider) Init(ctx context.Context, log logr.Logger, config provide
 	extension := strings.ToLower(path.Ext(config.Location))
 	switch extension {
 	case JavaArchive, WebArchive, EnterpriseArchive:
-		depLocation, sourceLocation, err := decompileJava(ctx, log, config.Location)
+		depLocation, sourceLocation, err := decompileJava(ctx, log,
+			config.Location, getMavenLocalRepoPath(mavenSettingsFile))
 		if err != nil {
 			cancelFunc()
 			return nil, err
@@ -404,6 +406,10 @@ func (j *javaProvider) GetLocation(ctx context.Context, dep konveyor.Dep) (engin
 // resolveSourcesJars for a given source code location, runs maven to find
 // deps that don't have sources attached and decompiles them
 func resolveSourcesJars(ctx context.Context, log logr.Logger, location, mavenSettings string) error {
+	// TODO (pgaikwad): inherit ctx from parent when we this becoms external provider
+	ctx, span := tracing.StartNewSpan(ctx, "resolve-sources")
+	defer span.End()
+
 	decompileJobs := []decompileJob{}
 
 	log.V(5).Info("resolving dependency sources")

provider/internal/java/util.go

@@ -23,6 +23,7 @@ import (
 	"github.com/konveyor/analyzer-lsp/engine/labels"
 	"github.com/konveyor/analyzer-lsp/provider"
 	"github.com/konveyor/analyzer-lsp/tracing"
+	"go.opentelemetry.io/otel/attribute"
 )
 
 const javaProjectPom = `<?xml version="1.0" encoding="UTF-8"?>
@@ -93,6 +94,7 @@ type decompileJob struct {
 	inputPath  string
 	outputPath string
 	artifact   javaArtifact
+	m2RepoPath string
 }
 
 // decompile decompiles files submitted via a list of decompileJob concurrently
@@ -107,11 +109,14 @@ func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, wor
 	for i := 0; i < workerCount; i++ {
 		logger := log.WithName(fmt.Sprintf("decompileWorker-%d", i))
 		wg.Add(1)
-		go func(log logr.Logger) {
+		go func(log logr.Logger, workerId int) {
 			defer log.V(6).Info("shutting down decompile worker")
 			defer wg.Done()
 			log.V(6).Info("init decompile worker")
 			for job := range jobChan {
+				// TODO (pgaikwad): when we move to external provider, inherit context from parent
+				jobCtx, span := tracing.StartNewSpan(ctx, "decomp-job",
+					attribute.Key("worker").Int(workerId))
 				// apply decompile filter
 				if !filter.shouldDecompile(job.artifact) {
 					continue
@@ -126,8 +131,9 @@ func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, wor
 						"failed to create directories for decompiled file", "path", outputPathDir)
 					continue
 				}
+				// -mpm (max processing method) is required to keep decomp time low
 				cmd := exec.CommandContext(
-					ctx, "java", "-jar", "/bin/fernflower.jar", job.inputPath, outputPathDir)
+					jobCtx, "java", "-jar", "/bin/fernflower.jar", "-mpm=30", job.inputPath, outputPathDir)
 				err := cmd.Run()
 				if err != nil {
 					log.V(5).Error(err, "failed to decompile file", "file", job.inputPath, job.outputPath)
@@ -137,13 +143,15 @@ func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, wor
 				// if we just decompiled a java archive, we need to
 				// explode it further and copy files to project
 				if job.artifact.packaging == JavaArchive && projectPath != "" {
-					_, _, _, err = explode(ctx, log, job.outputPath, projectPath)
+					_, _, _, err = explode(jobCtx, log, job.outputPath, projectPath, job.m2RepoPath)
 					if err != nil {
 						log.V(5).Error(err, "failed to explode decompiled jar", "path", job.inputPath)
 					}
 				}
+				span.End()
+				jobCtx.Done()
 			}
-		}(logger)
+		}(logger, i)
 	}
 
 	seenJobs := map[string]bool{}
@@ -165,15 +173,15 @@ func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, wor
 // decompileJava unpacks archive at archivePath, decompiles all .class files in it
 // creates new java project and puts the java files in the tree of the project
 // returns path to exploded archive, path to java project, and an error when encountered
-func decompileJava(ctx context.Context, log logr.Logger, archivePath string) (explodedPath, projectPath string, err error) {
+func decompileJava(ctx context.Context, log logr.Logger, archivePath string, m2RepoPath string) (explodedPath, projectPath string, err error) {
 	ctx, span := tracing.StartNewSpan(ctx, "decompile")
 	defer span.End()
 
 	projectPath = filepath.Join(filepath.Dir(archivePath), "java-project")
 
 	decompFilter := alwaysDecompileFilter(true)
 
-	explodedPath, decompJobs, deps, err := explode(ctx, log, archivePath, projectPath)
+	explodedPath, decompJobs, deps, err := explode(ctx, log, archivePath, projectPath, m2RepoPath)
 	if err != nil {
 		log.Error(err, "failed to decompile archive", "path", archivePath)
 		return "", "", err
@@ -212,7 +220,7 @@ func deduplicateJavaArtifacts(artifacts []javaArtifact) []javaArtifact {
 // explode explodes the given JAR, WAR or EAR archive, generates javaArtifact struct for given archive
 // and identifies all .class found recursively. returns output path, a list of decompileJob for .class files
 // it also returns a list of any javaArtifact we could interpret from jars
-func explode(ctx context.Context, log logr.Logger, archivePath, projectPath string) (string, []decompileJob, []javaArtifact, error) {
+func explode(ctx context.Context, log logr.Logger, archivePath, projectPath string, m2Repo string) (string, []decompileJob, []javaArtifact, error) {
 	var dependencies []javaArtifact
 	fileInfo, err := os.Stat(archivePath)
 	if err != nil {
@@ -313,7 +321,7 @@ func explode(ctx context.Context, log logr.Logger, archivePath, projectPath stri
 		// decompile web archives
 		case strings.HasSuffix(f.Name, WebArchive):
 			// TODO(djzager): Should we add these deps to the pom?
-			_, nestedJobs, deps, err := explode(ctx, log, filePath, projectPath)
+			_, nestedJobs, deps, err := explode(ctx, log, filePath, projectPath, m2Repo)
 			if err != nil {
 				log.Error(err, "failed to decompile file", "file", filePath)
 			}
@@ -344,6 +352,20 @@ func explode(ctx context.Context, log logr.Logger, archivePath, projectPath stri
 			if (dep != javaArtifact{}) {
 				if dep.foundOnline {
 					dependencies = append(dependencies, dep)
+					// copy this into m2 repo to avoid downloading again
+					destPath := m2Repo
+					for _, subPath := range strings.Split(dep.GroupId, ".") {
+						destPath = filepath.Join(destPath, subPath)
+					}
+					for _, subPath := range strings.Split(dep.ArtifactId, ".") {
+						destPath = filepath.Join(destPath, subPath)
+					}
+					destPath = filepath.Join(destPath, dep.Version, filepath.Base(filePath))
+					if err := moveFile(filePath, destPath); err != nil {
+						log.V(8).Error(err, "failed moving jar to m2 local repo")
+					} else {
+						log.V(8).Info("moved jar file", "src", filePath, "dest", destPath)
+					}
 				} else {
 					// when it isn't found online, decompile it
 					outputPath := filepath.Join(
@@ -387,6 +409,9 @@ func createJavaProject(ctx context.Context, dir string, dependencies []javaArtif
 }
 
 func moveFile(srcPath string, destPath string) error {
+	if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
+		return err
+	}
 	inputFile, err := os.Open(srcPath)
 	if err != nil {
 		return err