✨ add fallbacks for when workspace/symbol isn't giving us what we need (

#304)

- ~~This is currently failing with Go for... some reason. It can't load
packages anymore?~~
- ~~This is probably going to be hideously slow on large projects, the
directory tree search will need to be parallelized before this is a
feasible approach~~
- I haven't verified that this works with other providers. We should
give the experimental python one another try

---------

Signed-off-by: Fabian von Feilitzsch <[email protected]>

external-providers/generic-external-provider/main.go

@@ -35,4 +35,22 @@ func main() {
 	s := provider.NewServer(client, *port, log)
 	ctx := context.TODO()
 	s.Start(ctx)
+	// serviceClient, err := client.Init(ctx, log, provider.InitConfig{
+	// 	Location:     "/home/fabian/projects/github.com/konveyor/analyzer-lsp/examples/golang",
+	// 	AnalysisMode: "full",
+	// 	ProviderSpecificConfig: map[string]interface{}{
+	// 		"name":          "go",
+	// 		"lspServerPath": "gopls",
+	// 		"lspArgs":       []interface{}{"-vv", "-logfile", "go-debug.log", "-rpc.trace"},
+	// 	},
+	// })
+	// if err != nil {
+	// 	panic(err)
+	// }
+
+	// response, err := serviceClient.Evaluate("referenced", []byte(`{"referenced":{pattern: ".*v1beta1.CustomResourceDefinition"}}`))
+	// if err != nil {
+	// 	panic(err)
+	// }
+	// fmt.Println(response)
 }

external-providers/generic-external-provider/pkg/generic/service_client.go

@@ -1,11 +1,15 @@
 package generic
 
 import (
+	"bufio"
 	"context"
 	"fmt"
+	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
+	"sync"
 
 	"github.com/go-logr/logr"
 	"github.com/konveyor/analyzer-lsp/jsonrpc2"
@@ -45,7 +49,7 @@ func (p *genericServiceClient) Evaluate(cap string, conditionInfo []byte) (provi
 
 	incidents := []provider.IncidentContext{}
 	for _, s := range symbols {
-		references := p.GetAllReferences(s)
+		references := p.GetAllReferences(s.Location)
 		for _, ref := range references {
 			// Look for things that are in the location loaded, //Note may need to filter out vendor at some point
 			if strings.Contains(ref.URI, p.config.Location) {
@@ -74,30 +78,133 @@ func (p *genericServiceClient) Evaluate(cap string, conditionInfo []byte) (provi
 	}, nil
 }
 
-func (p *genericServiceClient) GetAllSymbols(query string) []protocol.WorkspaceSymbol {
+func processFile(path string, regex *regexp.Regexp, positionsChan chan<- protocol.TextDocumentPositionParams, wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	content, err := os.ReadFile(path)
+	if err != nil {
+		return
+	}
+
+	if regex.Match(content) {
+		scanner := bufio.NewScanner(strings.NewReader(string(content)))
+		lineNumber := 0
+		for scanner.Scan() {
+			matchLocations := regex.FindAllStringIndex(scanner.Text(), -1)
+			for _, loc := range matchLocations {
+				absPath, err := filepath.Abs(path)
+				if err != nil {
+					return
+				}
+				positionsChan <- protocol.TextDocumentPositionParams{
+					TextDocument: protocol.TextDocumentIdentifier{
+						URI: fmt.Sprintf("file://%s", absPath),
+					},
+					Position: protocol.Position{
+						Line:      float64(lineNumber),
+						Character: float64(loc[1]),
+					},
+				}
+			}
+			lineNumber++
+		}
+	}
+}
+
+func parallelWalk(location string, regex *regexp.Regexp) ([]protocol.TextDocumentPositionParams, error) {
+	var positions []protocol.TextDocumentPositionParams
+	positionsChan := make(chan protocol.TextDocumentPositionParams)
+	wg := &sync.WaitGroup{}
+
+	go func() {
+		err := filepath.Walk(location, func(path string, f os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if f.Mode().IsRegular() {
+				wg.Add(1)
+				go processFile(path, regex, positionsChan, wg)
+			}
+
+			return nil
+		})
+
+		if err != nil {
+			return
+		}
+
+		wg.Wait()
+		close(positionsChan)
+	}()
+
+	for pos := range positionsChan {
+		positions = append(positions, pos)
+	}
+
+	return positions, nil
+}
 
+func (p *genericServiceClient) GetAllSymbols(query string) []protocol.WorkspaceSymbol {
 	wsp := &protocol.WorkspaceSymbolParams{
 		Query: query,
 	}
 
-	var refs []protocol.WorkspaceSymbol
+	var symbols []protocol.WorkspaceSymbol
 	fmt.Printf("\nrpc call\n")
-	err := p.rpc.Call(context.TODO(), "workspace/symbol", wsp, &refs)
+	err := p.rpc.Call(context.TODO(), "workspace/symbol", wsp, &symbols)
 	fmt.Printf("\nrpc called\n")
 	if err != nil {
 		fmt.Printf("\n\nerror: %v\n", err)
 	}
-
-	return refs
+	regex, err := regexp.Compile(query)
+	if err != nil {
+		// Not a valid regex, can't do anything more
+		return symbols
+	}
+	if len(symbols) == 0 {
+		// Run empty string query and manually search using the query as a regex
+		var allSymbols []protocol.WorkspaceSymbol
+		err = p.rpc.Call(context.TODO(), "workspace/symbol", &protocol.WorkspaceSymbolParams{Query: ""}, &allSymbols)
+		if err != nil {
+			fmt.Printf("\n\nerror: %v\n", err)
+		}
+		for _, s := range allSymbols {
+			if regex.MatchString(s.Name) {
+				symbols = append(symbols, s)
+			}
+		}
+	}
+	if len(symbols) == 0 {
+		var positions []protocol.TextDocumentPositionParams
+		// Fallback to manually searching for an occurrence and performing a GotoDefinition call
+		positions, err := parallelWalk(p.config.Location, regex)
+		if err != nil {
+			fmt.Printf("\n\nerror: %v\n", err)
+			return nil
+		}
+		for _, position := range positions {
+			fmt.Println(position)
+			res := []protocol.Location{}
+			err := p.rpc.Call(p.ctx, "textDocument/definition", position, &res)
+			if err != nil {
+				fmt.Printf("Error rpc: %v", err)
+			}
+			for _, r := range res {
+				symbols = append(symbols, protocol.WorkspaceSymbol{Location: r})
+			}
+		}
+	}
+	return symbols
 }
 
-func (p *genericServiceClient) GetAllReferences(symbol protocol.WorkspaceSymbol) []protocol.Location {
+func (p *genericServiceClient) GetAllReferences(location protocol.Location) []protocol.Location {
 	params := &protocol.ReferenceParams{
 		TextDocumentPositionParams: protocol.TextDocumentPositionParams{
 			TextDocument: protocol.TextDocumentIdentifier{
-				URI: symbol.Location.URI,
+				URI: location.URI,
 			},
-			Position: symbol.Location.Range.Start,
+			Position: location.Range.Start,
 		},
 	}
 

rule-example.yaml

@@ -43,12 +43,12 @@
         pattern: "*apiextensions.v1beta1.CustomResourceDefinition*"
         location: TYPE
     - go.referenced: 
-        pattern: "v1beta1.CustomResourceDefinition"
+        pattern: ".*v1beta1.CustomResourceDefinition"
 - message: 'golang apiextensions/v1/customresourcedefinitions found {{file}}:{{lineNumber}}'
   ruleID: go-lang-ref-001
   when:
     go.referenced: 
-        pattern: "v1beta1.CustomResourceDefinition"
+        pattern: ".*v1beta1.CustomResourceDefinition"
 - message: testing nested conditions
   ruleID: lang-ref-002
   when: