✨ Generic Provider

Dockerfile

@@ -21,7 +21,8 @@ FROM quay.io/konveyor/jdtls-server-base
 
 COPY --from=builder /analyzer-lsp/konveyor-analyzer /usr/bin/konveyor-analyzer
 COPY --from=builder /analyzer-lsp/konveyor-analyzer-dep /usr/bin/konveyor-analyzer-dep
-COPY --from=builder /analyzer-lsp/external-providers/golang-external-provider/golang-external-provider /usr/bin/golang-external-provider
+COPY --from=builder /analyzer-lsp/external-providers/generic-external-provider/generic-external-provider /usr/bin/generic-external-provider
+COPY --from=builder /analyzer-lsp/external-providers/golang-dependency-provider/golang-dependency-provider /usr/bin/golang-dependency-provider
 
 COPY provider_container_settings.json /analyzer-lsp/provider_settings.json
 

Makefile

@@ -1,12 +1,15 @@
 DOCKER_IMAGE = test
 
-build: analyzer deps external-golang
+build: analyzer deps external-generic golang-dependency-provider
 
 analyzer:
 	go build -o konveyor-analyzer ./cmd/analyzer/main.go
 
-external-golang:
-	( cd external-providers/golang-external-provider && go build -o golang-external-provider main.go)
+external-generic:
+	( cd external-providers/generic-external-provider && go build -o generic-external-provider main.go)
+
+golang-dependency-provider:
+	go build -o ./external-providers/golang-dependency-provider/golang-dependency-provider ./external-providers/golang-dependency-provider/main.go
 
 deps:
 	go build -o konveyor-analyzer-dep ./cmd/dep/main.go

docs/experimental/python-provider.md

@@ -0,0 +1,77 @@
+# Python Provider using Generic Provider
+
+We are using the jedi-language-server (https://github.com/pappasam/jedi-language-server) to make a python provider using the generic provider.
+
+jedi-language-server can be installed using
+
+```
+pip install jedi-langauge-server
+```
+
+It will be installed in `/home/<user_name>/.local/bin/jedi-language-server`
+
+It will run without any arguments, but for more information it can be run with `--log-file LOG_FILE --verbose`
+
+The configuration used was:
+
+```json
+    {
+        "name": "python",
+        "binaryPath": "/path/to/generic/provider/binary",
+        "initConfig": [{
+            "location": "examples/python",
+            "analysisMode": "full",
+            "providerSpecificConfig": {
+                "name": "python",
+                "lspServerPath": "/path/to/jedi/language/server",
+            }
+        }]
+    },
+```
+
+The rule used to test it out was:
+
+```yaml
+    - message: python sample rule
+    ruleID: python-sample-rule-001
+    when:
+        python.referenced: 
+            pattern: "create_custom_resource_definition"
+```
+
+The example used for testing was:
+```python
+    #!/usr/bin/env python
+
+    import kubernetes
+
+    def main():
+        print(kubernetes.client.ApiextensionsV1beta1Api.create_custom_resource_definition)
+
+    if __name__ == '__main__':
+        main()
+```
+
+## Findings
+
+The jedi-language-server was able to get initialized and communicate with the analyzer-lsp.
+
+However, it returned `null` as a response to the rule. 
+
+After further testing, it was found that the jedi-language-server isn't able to find references to imported functions. 
+
+jedi-language-server returned a response when the rule was
+
+```yaml
+    - message: python sample rule
+    ruleID: python-sample-rule-001
+    when:
+        python.referenced: 
+            pattern: "main"
+```
+
+## Results
+
+We are going to move onto a different language server to test out the Generic Provider. 
+
+We are also going to investigate the behaviour of jedi-language-server to see whether not recognizing imported functions is intended. And also investigate GoPls to see whether recognizing imported functions is intended. 

docs/providers.md

@@ -29,24 +29,40 @@ If an explicit `proxyConfig` is not specified for a provider, system-wide proxy
 ```Note For Java: full analysis mode will search all the dependency and source, source-only will only search the source code. for a Jar/Ear/War, this is the code that is compiled in that archive and nothing else.
 ```
 
-#### Go provider
+#### Generic provider
 
-Here's an example config for an external `go` provider that is initialized using a binary and works on gRPC:
+Generic provider can be used to create an external provider for any language that is compliant with LSP 3.17 specifications.
+
+Here's an example config for a external `go` provider that is initialized using the generic provider binary.
 
 ```json
 {
     "name": "go",
-    "binaryPath": "/path/to/go/grpc/provider/binary",
+    "binaryPath": "/path/to/generic/provider/binary",
     "initConfig": [
         {
             "location": "/path/to/application/source/code",
-            "lspServerPath": "/path/to/language/server/binary",
             "analysisMode": "full",
+            "providerSpecificConfig": {
+                "name": "go",
+                "lspServerPath": "/path/to/language/server/binary",
+                "lspArgs": ["arg1", "arg2", "arg3"],
+                "dependencyProviderPath": "/path/to/dependency/provider/binary"
+            }
         }
     ]
 }
 ```
 
+The `generic provider` takes the following options in `providerSpecificConfig`:
+
+* `name`: Name of the provider to be displayed in the logs.
+
+* `lspArgs`: Arguments to be passed to run the langauge server. Optional field.
+
+* `dependencyProviderPath`: Path to a binary that prints the dependencies of the application as a `map[uri.URI][]provider.Dep{}`. The Dep struct can be imported from 
+`"github.com/konveyor/analyzer-lsp/provider"`.
+
 #### Java provider
 
 Here's an example config for `java` provider that is currently in-tree and does not use gRPC:

external-providers/golang-external-provider/go.mod → external-providers/generic-external-provider/go.mod

@@ -1,13 +1,13 @@
-module github.com/konveyor/golang-external-provider
+module github.com/konveyor/generic-external-provider
 
 go 1.19
 
 require (
-	github.com/bombsimon/logrusr/v3 v3.0.0
-	github.com/getkin/kin-openapi v0.116.0
+	github.com/bombsimon/logrusr/v3 v3.1.0
+	github.com/getkin/kin-openapi v0.118.0
 	github.com/go-logr/logr v1.2.4
-	github.com/konveyor/analyzer-lsp v0.0.0-20230503143412-a13c5b7be8cb
-	github.com/sirupsen/logrus v1.9.0
+	github.com/konveyor/analyzer-lsp v0.0.0-20230717225202-ba6d8da016c1
+	github.com/sirupsen/logrus v1.9.3
 	go.lsp.dev/uri v0.3.0
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -39,4 +39,4 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-replace github.com/konveyor/analyzer-lsp => ../../
+replace github.com/konveyor/analyzer-lsp => ../../

external-providers/golang-external-provider/go.sum → external-providers/generic-external-provider/go.sum

@@ -2,15 +2,15 @@ github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E
 github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac=
 github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi0jPI=
 github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
-github.com/bombsimon/logrusr/v3 v3.0.0 h1:tcAoLfuAhKP9npBxWzSdpsvKPQt1XV02nSf2lZA82TQ=
-github.com/bombsimon/logrusr/v3 v3.0.0/go.mod h1:PksPPgSFEL2I52pla2glgCyyd2OqOHAnFF5E+g8Ixco=
+github.com/bombsimon/logrusr/v3 v3.1.0 h1:zORbLM943D+hDMGgyjMhSAz/iDz86ZV72qaak/CA0zQ=
+github.com/bombsimon/logrusr/v3 v3.1.0/go.mod h1:PksPPgSFEL2I52pla2glgCyyd2OqOHAnFF5E+g8Ixco=
 github.com/cbroglie/mustache v1.4.0 h1:Azg0dVhxTml5me+7PsZ7WPrQq1Gkf3WApcHMjMprYoU=
 github.com/cbroglie/mustache v1.4.0/go.mod h1:SS1FTIghy0sjse4DUVGV1k/40B1qE1XkD9DtDsHo9iM=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/getkin/kin-openapi v0.116.0 h1:o986hwgMzR972JzOG5j6+WTwWqllZLs1EJKMKCivs2E=
-github.com/getkin/kin-openapi v0.116.0/go.mod h1:l5e9PaFUo9fyLJCPGQeXI2ML8c3P8BHOEV2VaAVf/pc=
+github.com/getkin/kin-openapi v0.118.0 h1:z43njxPmJ7TaPpMSCQb7PN0dEYno4tyBPQcrFdHoLuM=
+github.com/getkin/kin-openapi v0.118.0/go.mod h1:l5e9PaFUo9fyLJCPGQeXI2ML8c3P8BHOEV2VaAVf/pc=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -35,6 +35,8 @@ github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc=
 github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/konveyor/analyzer-lsp v0.0.0-20230717225202-ba6d8da016c1 h1:ayGO4il6x3cb/CakkTZVUmAjteBzTiOvrVTlarlfUd4=
+github.com/konveyor/analyzer-lsp v0.0.0-20230717225202-ba6d8da016c1/go.mod h1:+k6UreVv8ztI29/RyQN8/71AAmB0aWwQoWwZd3yR8sc=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -52,8 +54,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
-github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
-github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=

external-providers/golang-external-provider/main.go → external-providers/generic-external-provider/main.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/bombsimon/logrusr/v3"
 	"github.com/konveyor/analyzer-lsp/provider"
-	"github.com/konveyor/golang-external-provider/pkg/golang"
+	"github.com/konveyor/generic-external-provider/pkg/generic"
 	"github.com/sirupsen/logrus"
 )
 
@@ -26,7 +26,7 @@ func main() {
 
 	log := logrusr.New(logrusLog)
 
-	client := golang.NewGolangProvider()
+	client := generic.NewGenericProvider()
 
 	if port == nil || *port == 0 {
 		panic(fmt.Errorf("must pass in the port for the external provider"))

external-providers/generic-external-provider/pkg/generic/dependency.go

@@ -0,0 +1,38 @@
+package generic
+
+import (
+	"encoding/json"
+	"fmt"
+	"os/exec"
+
+	"github.com/konveyor/analyzer-lsp/provider"
+	"go.lsp.dev/uri"
+)
+
+func (g *genericServiceClient) GetDependencies() (map[uri.URI][]*provider.Dep, error) {
+	cmdStr, isString := g.config.ProviderSpecificConfig["dependencyProviderPath"].(string)
+	if !isString {
+		return nil, fmt.Errorf("dependency provider path is not a string")
+	}
+	// Expects dependency provider to output provider.Dep structs to stdout
+	cmd := exec.Command(cmdStr)
+	cmd.Dir = g.config.Location
+	dataR, err := cmd.Output()
+	if err != nil {
+		return nil, err
+	}
+	data := string(dataR)
+	if len(data) == 0 {
+		return nil, nil
+	}
+	m := map[uri.URI][]*provider.Dep{}
+	err = json.Unmarshal([]byte(data), &m)
+	if err != nil {
+		return nil, err
+	}
+	return m, err
+}
+
+func (p *genericServiceClient) GetDependenciesDAG() (map[uri.URI][]provider.DepDAGItem, error) {
+	return nil, nil
+}

external-providers/golang-external-provider/pkg/golang/dependency_test.go → external-providers/generic-external-provider/pkg/generic/dependency_test.go

@@ -1,4 +1,4 @@
-package golang
+package generic
 
 import (
 	"fmt"

external-providers/golang-external-provider/pkg/golang/provider.go → external-providers/generic-external-provider/pkg/generic/provider.go

@@ -1,4 +1,4 @@
-package golang
+package generic
 
 import (
 	"context"
@@ -14,17 +14,17 @@ import (
 
 // TODO(shawn-hurley): Pipe the logger through
 // Determine how and where external providers will add the logs to make the logs viewable in a single location.
-type golangProvider struct {
+type genericProvider struct {
 	ctx context.Context
 }
 
-var _ provider.BaseClient = &golangProvider{}
+var _ provider.BaseClient = &genericProvider{}
 
-func NewGolangProvider() *golangProvider {
-	return &golangProvider{}
+func NewGenericProvider() *genericProvider {
+	return &genericProvider{}
 }
 
-func (p *golangProvider) Capabilities() []provider.Capability {
+func (p *genericProvider) Capabilities() []provider.Capability {
 	return []provider.Capability{
 		{
 			Name:            "referenced",
@@ -37,18 +37,24 @@ func (p *golangProvider) Capabilities() []provider.Capability {
 	}
 }
 
-type golangCondition struct {
-	Referenced string `yaml:"referenced"`
+type genericCondition struct {
+	Referenced referenceCondition `yaml:"referenced"`
 }
 
-func (p *golangProvider) Init(ctx context.Context, log logr.Logger, c provider.InitConfig) (provider.ServiceClient, error) {
+type referenceCondition struct {
+	Pattern string `yaml:"pattern"`
+}
+
+func (p *genericProvider) Init(ctx context.Context, log logr.Logger, c provider.InitConfig) (provider.ServiceClient, error) {
 	if c.AnalysisMode != provider.FullAnalysisMode {
 		return nil, fmt.Errorf("only full analysis is supported")
 	}
 
-	// handle proxy settings
-	for k, v := range c.Proxy.ToEnvVars() {
-		os.Setenv(k, v)
+	if c.Proxy != nil {
+		// handle proxy settings
+		for k, v := range c.Proxy.ToEnvVars() {
+			os.Setenv(k, v)
+		}
 	}
 
 	lspServerPath, ok := c.ProviderSpecificConfig[provider.LspServerPathConfigKey].(string)
@@ -57,8 +63,22 @@ func (p *golangProvider) Init(ctx context.Context, log logr.Logger, c provider.I
 	}
 
 	ctx, cancelFunc := context.WithCancel(ctx)
-	log = log.WithValues("provider", "golang")
-	cmd := exec.CommandContext(ctx, lspServerPath)
+	log = log.WithValues("provider", c.ProviderSpecificConfig["name"])
+	var args []string
+	if lspArgs, ok := c.ProviderSpecificConfig["lspArgs"]; ok {
+		rawArgs, isArray := lspArgs.([]interface{})
+		if !isArray {
+			return nil, fmt.Errorf("lspArgs is not an array")
+		}
+		for _, rawArg := range rawArgs {
+			if arg, ok := rawArg.(string); ok {
+				args = append(args, arg)
+			} else {
+				return nil, fmt.Errorf("item of lspArgs is not a string")
+			}
+		}
+	}
+	cmd := exec.CommandContext(ctx, lspServerPath, args...)
 	stdin, err := cmd.StdinPipe()
 	if err != nil {
 		return nil, err
@@ -88,7 +108,7 @@ func (p *golangProvider) Init(ctx context.Context, log logr.Logger, c provider.I
 		}
 	}()
 
-	svcClient := golangServiceClient{
+	svcClient := genericServiceClient{
 		rpc:        rpc,
 		ctx:        ctx,
 		cancelFunc: cancelFunc,