diff --git a/cmd/analyzer/main.go b/cmd/analyzer/main.go index f96fcfe9..1a0bdb04 100644 --- a/cmd/analyzer/main.go +++ b/cmd/analyzer/main.go @@ -20,6 +20,7 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/swaggest/openapi-go/openapi3" + "go.opentelemetry.io/otel/attribute" "gopkg.in/yaml.v2" ) @@ -112,8 +113,8 @@ func AnalysisCmd() *cobra.Command { defer tracing.Shutdown(ctx, log, tp) - ctx, span := tracing.StartNewSpan(ctx, "main") - defer span.End() + ctx, mainSpan := tracing.StartNewSpan(ctx, "main") + defer mainSpan.End() // Get the configs configs, err := provider.GetConfig(settingsFile) @@ -194,14 +195,19 @@ func AnalysisCmd() *cobra.Command { } // Now that we have all the providers, we need to start them. for name, provider := range needProviders { - err := provider.ProviderInit(ctx) + initCtx, initSpan := tracing.StartNewSpan(ctx, "init", + attribute.Key("provider").String(name)) + err := provider.ProviderInit(initCtx) if err != nil { errLog.Error(err, "unable to init the providers", "provider", name) os.Exit(1) } + initSpan.End() } - rulesets := eng.RunRules(ctx, ruleSets, selectors...) + rulesCtx, rulesSpan := tracing.StartNewSpan(ctx, "run-rules") + rulesets := eng.RunRules(rulesCtx, ruleSets, selectors...) + rulesSpan.End() eng.Stop() for _, provider := range needProviders { diff --git a/provider/internal/java/provider.go b/provider/internal/java/provider.go index 2b2535eb..4176c434 100644 --- a/provider/internal/java/provider.go +++ b/provider/internal/java/provider.go @@ -20,6 +20,7 @@ import ( "github.com/konveyor/analyzer-lsp/lsp/protocol" "github.com/konveyor/analyzer-lsp/output/v1/konveyor" "github.com/konveyor/analyzer-lsp/provider" + "github.com/konveyor/analyzer-lsp/tracing" "github.com/swaggest/openapi-go/openapi3" "go.lsp.dev/uri" ) @@ -234,7 +235,8 @@ func (p *javaProvider) Init(ctx context.Context, log logr.Logger, config provide extension := strings.ToLower(path.Ext(config.Location)) switch extension { case JavaArchive, WebArchive, EnterpriseArchive: - depLocation, sourceLocation, err := decompileJava(ctx, log, config.Location) + depLocation, sourceLocation, err := decompileJava(ctx, log, + config.Location, getMavenLocalRepoPath(mavenSettingsFile)) if err != nil { cancelFunc() return nil, err @@ -404,6 +406,10 @@ func (j *javaProvider) GetLocation(ctx context.Context, dep konveyor.Dep) (engin // resolveSourcesJars for a given source code location, runs maven to find // deps that don't have sources attached and decompiles them func resolveSourcesJars(ctx context.Context, log logr.Logger, location, mavenSettings string) error { + // TODO (pgaikwad): inherit ctx from parent when we this becoms external provider + ctx, span := tracing.StartNewSpan(ctx, "resolve-sources") + defer span.End() + decompileJobs := []decompileJob{} log.V(5).Info("resolving dependency sources") diff --git a/provider/internal/java/util.go b/provider/internal/java/util.go index f5c1468c..9aebffb2 100644 --- a/provider/internal/java/util.go +++ b/provider/internal/java/util.go @@ -23,6 +23,7 @@ import ( "github.com/konveyor/analyzer-lsp/engine/labels" "github.com/konveyor/analyzer-lsp/provider" "github.com/konveyor/analyzer-lsp/tracing" + "go.opentelemetry.io/otel/attribute" ) const javaProjectPom = ` @@ -93,6 +94,7 @@ type decompileJob struct { inputPath string outputPath string artifact javaArtifact + m2RepoPath string } // decompile decompiles files submitted via a list of decompileJob concurrently @@ -107,11 +109,14 @@ func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, wor for i := 0; i < workerCount; i++ { logger := log.WithName(fmt.Sprintf("decompileWorker-%d", i)) wg.Add(1) - go func(log logr.Logger) { + go func(log logr.Logger, workerId int) { defer log.V(6).Info("shutting down decompile worker") defer wg.Done() log.V(6).Info("init decompile worker") for job := range jobChan { + // TODO (pgaikwad): when we move to external provider, inherit context from parent + jobCtx, span := tracing.StartNewSpan(ctx, "decomp-job", + attribute.Key("worker").Int(workerId)) // apply decompile filter if !filter.shouldDecompile(job.artifact) { continue @@ -126,8 +131,9 @@ func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, wor "failed to create directories for decompiled file", "path", outputPathDir) continue } + // -mpm (max processing method) is required to keep decomp time low cmd := exec.CommandContext( - ctx, "java", "-jar", "/bin/fernflower.jar", job.inputPath, outputPathDir) + jobCtx, "java", "-jar", "/bin/fernflower.jar", "-mpm=30", job.inputPath, outputPathDir) err := cmd.Run() if err != nil { log.V(5).Error(err, "failed to decompile file", "file", job.inputPath, job.outputPath) @@ -137,13 +143,15 @@ func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, wor // if we just decompiled a java archive, we need to // explode it further and copy files to project if job.artifact.packaging == JavaArchive && projectPath != "" { - _, _, _, err = explode(ctx, log, job.outputPath, projectPath) + _, _, _, err = explode(jobCtx, log, job.outputPath, projectPath, job.m2RepoPath) if err != nil { log.V(5).Error(err, "failed to explode decompiled jar", "path", job.inputPath) } } + span.End() + jobCtx.Done() } - }(logger) + }(logger, i) } seenJobs := map[string]bool{} @@ -165,7 +173,7 @@ func decompile(ctx context.Context, log logr.Logger, filter decompileFilter, wor // decompileJava unpacks archive at archivePath, decompiles all .class files in it // creates new java project and puts the java files in the tree of the project // returns path to exploded archive, path to java project, and an error when encountered -func decompileJava(ctx context.Context, log logr.Logger, archivePath string) (explodedPath, projectPath string, err error) { +func decompileJava(ctx context.Context, log logr.Logger, archivePath string, m2RepoPath string) (explodedPath, projectPath string, err error) { ctx, span := tracing.StartNewSpan(ctx, "decompile") defer span.End() @@ -173,7 +181,7 @@ func decompileJava(ctx context.Context, log logr.Logger, archivePath string) (ex decompFilter := alwaysDecompileFilter(true) - explodedPath, decompJobs, deps, err := explode(ctx, log, archivePath, projectPath) + explodedPath, decompJobs, deps, err := explode(ctx, log, archivePath, projectPath, m2RepoPath) if err != nil { log.Error(err, "failed to decompile archive", "path", archivePath) return "", "", err @@ -212,7 +220,7 @@ func deduplicateJavaArtifacts(artifacts []javaArtifact) []javaArtifact { // explode explodes the given JAR, WAR or EAR archive, generates javaArtifact struct for given archive // and identifies all .class found recursively. returns output path, a list of decompileJob for .class files // it also returns a list of any javaArtifact we could interpret from jars -func explode(ctx context.Context, log logr.Logger, archivePath, projectPath string) (string, []decompileJob, []javaArtifact, error) { +func explode(ctx context.Context, log logr.Logger, archivePath, projectPath string, m2Repo string) (string, []decompileJob, []javaArtifact, error) { var dependencies []javaArtifact fileInfo, err := os.Stat(archivePath) if err != nil { @@ -313,7 +321,7 @@ func explode(ctx context.Context, log logr.Logger, archivePath, projectPath stri // decompile web archives case strings.HasSuffix(f.Name, WebArchive): // TODO(djzager): Should we add these deps to the pom? - _, nestedJobs, deps, err := explode(ctx, log, filePath, projectPath) + _, nestedJobs, deps, err := explode(ctx, log, filePath, projectPath, m2Repo) if err != nil { log.Error(err, "failed to decompile file", "file", filePath) } @@ -344,6 +352,20 @@ func explode(ctx context.Context, log logr.Logger, archivePath, projectPath stri if (dep != javaArtifact{}) { if dep.foundOnline { dependencies = append(dependencies, dep) + // copy this into m2 repo to avoid downloading again + destPath := m2Repo + for _, subPath := range strings.Split(dep.GroupId, ".") { + destPath = filepath.Join(destPath, subPath) + } + for _, subPath := range strings.Split(dep.ArtifactId, ".") { + destPath = filepath.Join(destPath, subPath) + } + destPath = filepath.Join(destPath, dep.Version, filepath.Base(filePath)) + if err := moveFile(filePath, destPath); err != nil { + log.V(8).Error(err, "failed moving jar to m2 local repo") + } else { + log.V(8).Info("moved jar file", "src", filePath, "dest", destPath) + } } else { // when it isn't found online, decompile it outputPath := filepath.Join( @@ -387,6 +409,9 @@ func createJavaProject(ctx context.Context, dir string, dependencies []javaArtif } func moveFile(srcPath string, destPath string) error { + if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil { + return err + } inputFile, err := os.Open(srcPath) if err != nil { return err