From 1811e14fc5e87fc66ac208252f6642d5665fa14c Mon Sep 17 00:00:00 2001 From: Pranav Gaikwad Date: Thu, 9 Nov 2023 07:24:20 -0500 Subject: [PATCH] :sparkles: get location for deps Signed-off-by: Pranav Gaikwad --- provider/internal/java/dependency.go | 57 ++++++++ provider/internal/java/provider_test.go | 167 ++++++++++++++++++++++++ provider/provider.go | 28 +++- 3 files changed, 249 insertions(+), 3 deletions(-) diff --git a/provider/internal/java/dependency.go b/provider/internal/java/dependency.go index d387bdcb..7bf1c5dc 100644 --- a/provider/internal/java/dependency.go +++ b/provider/internal/java/dependency.go @@ -12,8 +12,10 @@ import ( "path/filepath" "reflect" "regexp" + "strconv" "strings" + "github.com/konveyor/analyzer-lsp/engine" "github.com/konveyor/analyzer-lsp/engine/labels" "github.com/konveyor/analyzer-lsp/output/v1/konveyor" "github.com/konveyor/analyzer-lsp/provider" @@ -28,6 +30,15 @@ const ( providerSpecificConfigExcludePackagesKey = "excludePackages" ) +// keys used in dep.Extras for extra information about a dep +const ( + artifactIdKey = "artifactId" + groupIdKey = "groupId" + pomPathKey = "pomPath" +) + +var _ provider.DependencyLocationResolver = &javaServiceClient{} + // TODO implement this for real func (p *javaServiceClient) findPom() string { var depPath string @@ -517,3 +528,49 @@ func loadDepLabelItems(r io.Reader, depToLabels map[string]*depLabelItem, label } return nil } + +// getLineNumberForDep given a dep and pom file, attempts to find line number +func (j *javaServiceClient) GetLocation(ctx context.Context, dep konveyor.Dep) (engine.Location, error) { + if dep.Extras == nil { + return engine.Location{}, fmt.Errorf("unable to get location for dep %s", dep.Name) + } + extrasKeys := []string{artifactIdKey, groupIdKey, pomPathKey} + for _, key := range extrasKeys { + if _, ok := dep.Extras[key]; !ok { + return engine.Location{}, fmt.Errorf("unable to get location for dep %s, missing extras", dep.Name) + } + } + groupId := dep.Extras[groupIdKey] + artifactId := dep.Extras[artifactIdKey] + path := dep.Extras[pomPathKey] + cmd := exec.CommandContext(ctx, "pcre2grep", []string{ + "-M", + "-n", + "--line-offsets", + fmt.Sprintf("%s(\\n|.)*?%s.*", groupId, artifactId), + }...) + output, err := cmd.CombinedOutput() + if err != nil { + return engine.Location{}, fmt.Errorf("failed to run pcre2grep to get line number (%w)", err) + } + outputLines := strings.Split(string(output), "\n") + if len(outputLines) < 1 { + return engine.Location{}, fmt.Errorf("no match found for %s.%s in %s", groupId, artifactId, path) + } + parts := strings.Split(outputLines[0], ":") + if len(parts) < 1 { + return engine.Location{}, fmt.Errorf("unparseable pcre2grep output - %s", outputLines[0]) + } + lineNumber, err := strconv.Atoi(parts[0]) + if err != nil { + return engine.Location{}, fmt.Errorf("unparseable pcre2grep output - %s", outputLines[0]) + } + return engine.Location{ + StartPosition: engine.Position{ + Line: lineNumber, + }, + EndPosition: engine.Position{ + Line: lineNumber, + }, + }, nil +} diff --git a/provider/internal/java/provider_test.go b/provider/internal/java/provider_test.go index e99d5a68..940e41af 100644 --- a/provider/internal/java/provider_test.go +++ b/provider/internal/java/provider_test.go @@ -41,6 +41,173 @@ func Test_parseUnresolvedSources(t *testing.T) { }, }, }, + { + name: "", + wantErr: false, + wantList: []javaArtifact{}, + mvnOutput: ` +[INFO] Scanning for projects... +[INFO] +[INFO] -----------------< io.konveyor.demo:customers-tomcat >------------------ +[INFO] Building Order Management 0.0.1-SNAPSHOT +[INFO] from pom.xml +[INFO] --------------------------------[ war ]--------------------------------- +[INFO] +[INFO] --- dependency:3.6.0:sources (default-cli) @ customers-tomcat --- +[INFO] Can't extract module name from jackson-core-2.12.3-sources.jar: Provider class com.fasterxml.jackson.core.JsonFactory not in module +[INFO] Can't extract module name from jackson-databind-2.12.3-sources.jar: Provider class com.fasterxml.jackson.databind.ObjectMapper not in module +[INFO] Can't extract module name from spring-data-jpa-2.5.1-sources.jar: Provider class org.springframework.data.jpa.repository.cdi.JpaRepositoryExtension not in module +[INFO] Can't extract module name from spring-core-5.3.7-sources.jar: Provider class org.springframework.core.ReactiveAdapterRegistry$SpringCoreBlockHoundIntegration not in module +[INFO] Can't extract module name from spring-jcl-5.3.7-sources.jar: Provider class org.apache.commons.logging.LogFactoryService not in module +[INFO] Can't extract module name from spring-web-5.3.7-sources.jar: Provider class org.springframework.web.SpringServletContainerInitializer not in module +[INFO] Can't extract module name from log4j-to-slf4j-2.14.1-sources.jar: Provider class org.apache.logging.slf4j.SLF4JProvider not in module +[INFO] Can't extract module name from log4j-api-2.14.1-sources.jar: Provider class org.apache.logging.log4j.util.EnvironmentPropertySource not in module +[INFO] Can't extract module name from jackson-datatype-jsr310-2.12.3-sources.jar: Provider class com.fasterxml.jackson.datatype.jsr310.JavaTimeModule not in module +[INFO] Can't extract module name from hibernate-core-5.4.32.Final-sources.jar: Provider class org.hibernate.jpa.HibernatePersistenceProvider not in module +[INFO] Can't extract module name from jaxb-runtime-2.3.1-sources.jar: Provider class com.sun.xml.bind.v2.ContextFactory not in module +[INFO] Can't extract module name from byte-buddy-1.10.22-sources.jar: byte.buddy: Invalid module name: 'byte' is not a Java identifier +[INFO] Can't extract module name from jboss-transaction-api_1.2_spec-1.1.1.Final-sources.jar: jboss.transaction.api.1.2.spec: Invalid module name: '1' is not a Java identifier +[INFO] Can't extract module name from hibernate-validator-6.2.0.Final-sources.jar: Provider class org.hibernate.validator.HibernateValidator not in module +[INFO] Can't extract module name from postgresql-42.2.23-sources.jar: Provider class org.postgresql.Driver not in module +[INFO] +[INFO] The following files have been resolved: +[INFO] org.apache.tomcat:tomcat-servlet-api:jar:sources:9.0.46 -- module tomcat.servlet.api (auto) +[INFO] com.fasterxml.jackson.core:jackson-core:jar:sources:2.12.3 +[INFO] com.fasterxml.jackson.core:jackson-databind:jar:sources:2.12.3 +[INFO] com.fasterxml.jackson.core:jackson-annotations:jar:sources:2.12.3 -- module jackson.annotations (auto) +[INFO] org.springframework.data:spring-data-jpa:jar:sources:2.5.1 +[INFO] org.springframework.data:spring-data-commons:jar:sources:2.5.1 -- module spring.data.commons (auto) +[INFO] org.springframework:spring-orm:jar:sources:5.3.7 -- module spring.orm (auto) +[INFO] org.springframework:spring-context:jar:sources:5.3.7 -- module spring.context (auto) +[INFO] org.springframework:spring-aop:jar:sources:5.3.7 -- module spring.aop (auto) +[INFO] org.springframework:spring-tx:jar:sources:5.3.7 -- module spring.tx (auto) +[INFO] org.springframework:spring-beans:jar:sources:5.3.7 -- module spring.beans (auto) +[INFO] org.springframework:spring-core:jar:sources:5.3.7 +[INFO] org.springframework:spring-jcl:jar:sources:5.3.7 +[INFO] org.aspectj:aspectjrt:jar:sources:1.9.6 -- module aspectjrt (auto) +[INFO] org.slf4j:slf4j-api:jar:sources:1.7.26 -- module slf4j.api (auto) +[INFO] org.springframework:spring-jdbc:jar:sources:5.3.7 -- module spring.jdbc (auto) +[INFO] org.springframework:spring-webmvc:jar:sources:5.3.7 -- module spring.webmvc (auto) +[INFO] org.springframework:spring-expression:jar:sources:5.3.7 -- module spring.expression (auto) +[INFO] org.springframework:spring-web:jar:sources:5.3.7 +[INFO] org.springframework.boot:spring-boot-starter-actuator:jar:sources:2.5.0 -- module spring.boot.starter.actuator [auto] +[INFO] org.springframework.boot:spring-boot-starter:jar:sources:2.5.0 -- module spring.boot.starter [auto] +[INFO] org.springframework.boot:spring-boot:jar:sources:2.5.0 -- module spring.boot [auto] +[INFO] org.springframework.boot:spring-boot-autoconfigure:jar:sources:2.5.0 -- module spring.boot.autoconfigure [auto] +[INFO] org.springframework.boot:spring-boot-starter-logging:jar:sources:2.5.0 -- module spring.boot.starter.logging [auto] +[INFO] org.apache.logging.log4j:log4j-to-slf4j:jar:sources:2.14.1 +[INFO] org.apache.logging.log4j:log4j-api:jar:sources:2.14.1 +[INFO] org.slf4j:jul-to-slf4j:jar:sources:1.7.30 -- module jul.to.slf4j (auto) +[INFO] jakarta.annotation:jakarta.annotation-api:jar:sources:1.3.5 -- module jakarta.annotation.api (auto) +[INFO] org.yaml:snakeyaml:jar:sources:1.28 -- module snakeyaml (auto) +[INFO] org.springframework.boot:spring-boot-actuator-autoconfigure:jar:sources:2.5.0 -- module spring.boot.actuator.autoconfigure [auto] +[INFO] org.springframework.boot:spring-boot-actuator:jar:sources:2.5.0 -- module spring.boot.actuator [auto] +[INFO] com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar:sources:2.12.3 +[INFO] io.micrometer:micrometer-core:jar:sources:1.7.0 -- module micrometer.core (auto) +[INFO] org.hdrhistogram:HdrHistogram:jar:sources:2.1.12 -- module HdrHistogram (auto) +[INFO] org.latencyutils:LatencyUtils:jar:sources:2.0.3 -- module LatencyUtils (auto) +[INFO] org.apache.tomcat:tomcat-jdbc:jar:sources:9.0.46 -- module tomcat.jdbc (auto) +[INFO] org.apache.tomcat:tomcat-juli:jar:sources:9.0.46 -- module tomcat.juli (auto) +[INFO] org.hibernate:hibernate-entitymanager:jar:sources:5.4.32.Final -- module hibernate.entitymanager (auto) +[INFO] org.jboss.logging:jboss-logging:jar:sources:3.4.1.Final -- module jboss.logging (auto) +[INFO] org.hibernate:hibernate-core:jar:sources:5.4.32.Final +[INFO] org.javassist:javassist:jar:sources:3.27.0-GA -- module javassist (auto) +[INFO] org.jboss:jandex:jar:sources:2.2.3.Final -- module jandex (auto) +[INFO] javax.activation:javax.activation-api:jar:sources:1.2.0 -- module javax.activation.api (auto) +[INFO] javax.xml.bind:jaxb-api:jar:sources:2.3.1 -- module jaxb.api (auto) +[INFO] org.glassfish.jaxb:jaxb-runtime:jar:sources:2.3.1 +[INFO] org.glassfish.jaxb:txw2:jar:sources:2.3.1 -- module txw2 (auto) +[INFO] com.sun.istack:istack-commons-runtime:jar:sources:3.0.7 -- module istack.commons.runtime (auto) +[INFO] org.jvnet.staxex:stax-ex:jar:sources:1.8 -- module stax.ex (auto) +[INFO] com.sun.xml.fastinfoset:FastInfoset:jar:sources:1.2.15 -- module FastInfoset (auto) +[INFO] org.dom4j:dom4j:jar:sources:2.1.3 -- module dom4j (auto) +[INFO] org.hibernate.common:hibernate-commons-annotations:jar:sources:5.1.2.Final -- module hibernate.commons.annotations (auto) +[INFO] javax.persistence:javax.persistence-api:jar:sources:2.2 -- module javax.persistence.api (auto) +[INFO] net.bytebuddy:byte-buddy:jar:sources:1.10.22 +[INFO] org.jboss.spec.javax.transaction:jboss-transaction-api_1.2_spec:jar:sources:1.1.1.Final +[INFO] org.hibernate.validator:hibernate-validator:jar:sources:6.2.0.Final +[INFO] jakarta.validation:jakarta.validation-api:jar:sources:2.0.2 -- module jakarta.validation.api (auto) +[INFO] com.fasterxml:classmate:jar:sources:1.5.1 -- module classmate (auto) +[INFO] ch.qos.logback:logback-classic:jar:sources:1.1.7 -- module logback.classic (auto) +[INFO] ch.qos.logback:logback-core:jar:sources:1.1.7 -- module logback.core (auto) +[INFO] com.oracle.database.jdbc:ojdbc11:jar:sources:21.1.0.0 -- module ojdbc11 (auto) +[INFO] org.postgresql:postgresql:jar:sources:42.2.23 +[INFO] org.checkerframework:checker-qual:jar:sources:3.5.0 -- module checker.qual (auto) +[INFO] +[INFO] The following files have NOT been resolved: +[INFO] org.apache.tomcat:tomcat-servlet-api:jar:9.0.46:provided -- module java.servlet +[INFO] com.fasterxml.jackson.core:jackson-core:jar:2.12.3:compile -- module com.fasterxml.jackson.core +[INFO] com.fasterxml.jackson.core:jackson-databind:jar:2.12.3:compile -- module com.fasterxml.jackson.databind +[INFO] com.fasterxml.jackson.core:jackson-annotations:jar:2.12.3:compile -- module com.fasterxml.jackson.annotation +[INFO] org.springframework.data:spring-data-jpa:jar:2.5.1:compile -- module spring.data.jpa [auto] +[INFO] org.springframework.data:spring-data-commons:jar:2.5.1:compile -- module spring.data.commons [auto] +[INFO] org.springframework:spring-orm:jar:5.3.7:compile -- module spring.orm [auto] +[INFO] org.springframework:spring-context:jar:5.3.7:compile -- module spring.context [auto] +[INFO] org.springframework:spring-aop:jar:5.3.7:compile -- module spring.aop [auto] +[INFO] org.springframework:spring-tx:jar:5.3.7:compile -- module spring.tx [auto] +[INFO] org.springframework:spring-beans:jar:5.3.7:compile -- module spring.beans [auto] +[INFO] org.springframework:spring-core:jar:5.3.7:compile -- module spring.core [auto] +[INFO] org.springframework:spring-jcl:jar:5.3.7:compile -- module spring.jcl [auto] +[INFO] org.aspectj:aspectjrt:jar:1.9.6:compile -- module org.aspectj.runtime [auto] +[INFO] org.slf4j:slf4j-api:jar:1.7.26:compile -- module slf4j.api (auto) +[INFO] org.springframework:spring-jdbc:jar:5.3.7:compile -- module spring.jdbc [auto] +[INFO] org.springframework:spring-webmvc:jar:5.3.7:compile -- module spring.webmvc [auto] +[INFO] org.springframework:spring-expression:jar:5.3.7:compile -- module spring.expression [auto] +[INFO] org.springframework:spring-web:jar:5.3.7:compile -- module spring.web [auto] +[INFO] org.springframework.boot:spring-boot-starter-actuator:jar:2.5.0:compile -- module spring.boot.starter.actuator [auto] +[INFO] org.springframework.boot:spring-boot-starter:jar:2.5.0:compile -- module spring.boot.starter [auto] +[INFO] org.springframework.boot:spring-boot:jar:2.5.0:compile -- module spring.boot [auto] +[INFO] org.springframework.boot:spring-boot-autoconfigure:jar:2.5.0:compile -- module spring.boot.autoconfigure [auto] +[INFO] org.springframework.boot:spring-boot-shttps://github.com/konveyor/analyzer-lsp/pull/403tarter-logging:jar:2.5.0:compile -- module spring.boot.starter.logging [auto] +[INFO] org.apache.logging.log4j:log4j-to-slf4j:jar:2.14.1:compile -- module org.apache.logging.slf4j [auto] +[INFO] org.apache.logging.log4j:log4j-api:jar:2.14.1:compile -- module org.apache.logging.log4j +[INFO] org.slf4j:jul-to-slf4j:jar:1.7.30:compile -- module jul.to.slf4j (auto) +[INFO] jakarta.annotation:jakarta.annotation-api:jar:1.3.5:compile -- module java.annotation [auto] +[INFO] org.yaml:snakeyaml:jar:1.28:compile -- module org.yaml.snakeyaml [auto] +[INFO] org.springframework.boot:spring-boot-actuator-autoconfigure:jar:2.5.0:compile -- module spring.boot.actuator.autoconfigure [auto] +[INFO] org.springframework.boot:spring-boot-actuator:jar:2.5.0:compile -- module spring.boot.actuator [auto] +[INFO] com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar:2.12.3:runtime -- module com.fasterxml.jackson.datatype.jsr310 +[INFO] io.micrometer:micrometer-core:jar:1.7.0:compile -- module micrometer.core [auto] +[INFO] org.hdrhistogram:HdrHistogram:jar:2.1.12:compile -- module HdrHistogram (auto) +[INFO] org.latencyutils:LatencyUtils:jar:2.0.3:runtime -- module LatencyUtils (auto) +[INFO] org.apache.tomcat:tomcat-jdbc:jar:9.0.46:runtime -- module tomcat.jdbc (auto) +[INFO] org.apache.tomcat:tomcat-juli:jar:9.0.46:runtime -- module org.apache.tomcat.juli +[INFO] org.hibernate:hibernate-entitymanager:jar:5.4.32.Final:compile -- module hibernate.entitymanager (auto) +[INFO] org.jboss.logging:jboss-logging:jar:3.4.1.Final:compile -- module org.jboss.logging [auto] +[INFO] org.hibernate:hibernate-core:jar:5.4.32.Final:compile -- module org.hibernate.orm.core [auto] +[INFO] org.javassist:javassist:jar:3.27.0-GA:compile -- module javassist (auto) +[INFO] antlr:antlr:jar:2.7.7:compile -- module antlr (auto) +[INFO] org.jboss:jandex:jar:2.2.3.Final:compile -- module org.jboss.jandex [auto] +[INFO] javax.activation:javax.activation-api:jar:1.2.0:compile -- module java.activation [auto] +[INFO] javax.xml.bind:jaxb-api:jar:2.3.1:compile -- module java.xml.bind +[INFO] org.glassfish.jaxb:jaxb-runtime:jar:2.3.1:compile -- module com.sun.xml.bind +[INFO] org.glassfish.jaxb:txw2:jar:2.3.1:compile -- module com.sun.xml.txw2 +[INFO] com.sun.istack:istack-commons-runtime:jar:3.0.7:compile -- module com.sun.istack.runtime +[INFO] org.jvnet.staxex:stax-ex:jar:1.8:compile -- module org.jvnet.staxex +[INFO] com.sun.xml.fastinfoset:FastInfoset:jar:1.2.15:compile -- module com.sun.xml.fastinfoset +[INFO] org.dom4j:dom4j:jar:2.1.3:compile -- module dom4j (auto) +[INFO] org.hibernate.common:hibernate-commons-annotations:jar:5.1.2.Final:compile -- module org.hibernate.commons.annotations [auto] +[INFO] javax.persistence:javax.persistence-api:jar:2.2:compile -- module java.persistence [auto] +[INFO] net.bytebuddy:byte-buddy:jar:1.10.22:compile -- module net.bytebuddy +[INFO] org.jboss.spec.javax.transaction:jboss-transaction-api_1.2_spec:jar:1.1.1.Final:compile -- module java.transaction [auto] +[INFO] org.hibernate.validator:hibernate-validator:jar:6.2.0.Final:compile -- module org.hibernate.validator [auto] +[INFO] jakarta.validation:jakarta.validation-api:jar:2.0.2:compile -- module java.validation [auto] +[INFO] com.fasterxml:classmate:jar:1.5.1:compile -- module com.fasterxml.classmate +[INFO] ch.qos.logback:logback-classic:jar:1.1.7:compile -- module logback.classic (auto) +[INFO] ch.qos.logback:logback-core:jar:1.1.7:compile -- module logback.core (auto) +[INFO] com.oracle.database.jdbc:ojdbc11:jar:21.1.0.0:compile -- module ojdbc11 (auto) +[INFO] org.postgresql:postgresql:jar:42.2.23:compile -- module org.postgresql.jdbc [auto] +[INFO] org.checkerframework:checker-qual:jar:3.5.0:runtime -- module org.checkerframework.checker.qual [auto] +[INFO] io.konveyor.demo:config-utils:jar:1.0.0:compile -- module config.utils (auto) +[INFO] +[INFO] ------------------------------------------------------------------------ +[INFO] BUILD SUCCESS +[INFO] ------------------------------------------------------------------------ +[INFO] Total time: 5.109 s +[INFO] Finished at: 2023-11-06T14:28:04Z +[INFO] ------------------------------------------------------------------------ + +`}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/provider/provider.go b/provider/provider.go index 1e2f5b83..d0b18971 100644 --- a/provider/provider.go +++ b/provider/provider.go @@ -6,6 +6,7 @@ import ( "os" "regexp" "strings" + "time" "github.com/cbroglie/mustache" "github.com/getkin/kin-openapi/openapi3" @@ -336,6 +337,10 @@ type ServiceClient interface { GetDependenciesDAG(ctx context.Context) (map[uri.URI][]DepDAGItem, error) } +type DependencyLocationResolver interface { + GetLocation(ctx context.Context, dep konveyor.Dep) (engine.Location, error) +} + type Dep = konveyor.Dep type DepDAGItem = konveyor.DepDAGItem type Startable interface { @@ -561,17 +566,34 @@ func (dc DependencyCondition) Evaluate(ctx context.Context, log logr.Logger, con return resp, nil } + var depLocationResolver DependencyLocationResolver + if val, ok := dc.Client.(DependencyLocationResolver); ok { + depLocationResolver = val + } + for _, matchedDep := range matchedDeps { if matchedDep.dep.Version == "" || (dc.Lowerbound == "" && dc.Upperbound == "") { - resp.Matched = true - resp.Incidents = append(resp.Incidents, engine.IncidentContext{ + incident := engine.IncidentContext{ FileURI: matchedDep.uri, Variables: map[string]interface{}{ "name": matchedDep.dep.Name, "version": matchedDep.dep.Version, "type": matchedDep.dep.Type, }, - }) + } + if depLocationResolver != nil { + // this is a best-effort step and we don't want to block if resolver misbehaves + timeoutContext, cancelFunc := context.WithTimeout(ctx, time.Second*3) + location, err := depLocationResolver.GetLocation(timeoutContext, *matchedDep.dep) + if err == nil { + incident.LineNumber = &location.StartPosition.Line + } else { + log.V(7).Error(err, "failed to get location for dependency", "dep", matchedDep.dep.Name) + } + cancelFunc() + } + resp.Matched = true + resp.Incidents = append(resp.Incidents, incident) // For now, lets leave this TODO to figure out what we should be setting in the context resp.TemplateContext = map[string]interface{}{ "name": matchedDep.dep.Name,