Merge pull request #17900 from github/mbg/go/fix/project-files-in-vendor

go/extractor/configurebaseline/configurebaseline.go

@@ -3,24 +3,12 @@ package configurebaseline
 import (
 	"encoding/json"
 	"io/fs"
-	"os"
 	"path"
 	"path/filepath"
 
 	"github.com/github/codeql-go/extractor/util"
 )
 
-func fileExists(path string) bool {
-	stat, err := os.Stat(path)
-	return err == nil && stat.Mode().IsRegular()
-}
-
-// Decides if `dirPath` is a vendor directory by testing whether it is called `vendor`
-// and contains a `modules.txt` file.
-func isGolangVendorDirectory(dirPath string) bool {
-	return filepath.Base(dirPath) == "vendor" && fileExists(filepath.Join(dirPath, "modules.txt"))
-}
-
 type BaselineConfig struct {
 	PathsIgnore []string `json:"paths-ignore"`
 }
@@ -38,7 +26,7 @@ func GetConfigBaselineAsJSON(rootDir string) ([]byte, error) {
 				// it will not be extracted either.
 				return nil
 			}
-			if isGolangVendorDirectory(dirPath) {
+			if util.IsGolangVendorDirectory(dirPath) {
 				// Note that CodeQL expects a forward-slash-separated path, even on Windows.
 				vendorDirs = append(vendorDirs, path.Join(filepath.ToSlash(dirPath), "**"))
 				return filepath.SkipDir

go/extractor/project/project.go

@@ -184,12 +184,12 @@ func RemoveTemporaryExtractorFiles() {
 
 // Find all go.work files in the working directory and its subdirectories
 func findGoWorkFiles() []string {
-	return util.FindAllFilesWithName(".", "go.work", "vendor")
+	return util.FindAllFilesWithName(".", "go.work", util.SkipVendorChecks...)
 }
 
 // Find all go.mod files in the specified directory and its subdirectories
 func findGoModFiles(root string) []string {
-	return util.FindAllFilesWithName(root, "go.mod", "vendor")
+	return util.FindAllFilesWithName(root, "go.mod", util.SkipVendorChecks...)
 }
 
 // A regular expression for the Go toolchain version syntax.
@@ -315,6 +315,11 @@ func discoverWorkspaces(emitDiagnostics bool) []GoWorkspace {
 		goModFiles := findGoModFiles(".")
 
 		// Return a separate workspace for each `go.mod` file that we found.
+		if len(goModFiles) > 0 {
+			log.Printf("Found %d go.mod files in: %s.\n", len(goModFiles), strings.Join(goModFiles, ", "))
+		} else {
+			log.Println("Found no go.mod files in the workspace.")
+		}
 		results := make([]GoWorkspace, len(goModFiles))
 
 		for i, goModFile := range goModFiles {
@@ -547,8 +552,8 @@ func startsWithAnyOf(str string, prefixes []string) bool {
 // Finds Go workspaces in the current working directory.
 func GetWorkspaceInfo(emitDiagnostics bool) []GoWorkspace {
 	bazelPaths := slices.Concat(
-		util.FindAllFilesWithName(".", "BUILD", "vendor"),
-		util.FindAllFilesWithName(".", "BUILD.bazel", "vendor"),
+		util.FindAllFilesWithName(".", "BUILD", util.SkipVendorChecks...),
+		util.FindAllFilesWithName(".", "BUILD.bazel", util.SkipVendorChecks...),
 	)
 	if len(bazelPaths) > 0 {
 		// currently not supported

go/extractor/util/util.go

@@ -152,15 +152,24 @@ func FindGoFiles(root string) bool {
 	return found
 }
 
-func FindAllFilesWithName(root string, name string, dirsToSkip ...string) []string {
+// The type of check function used by `FindAllFilesWithName` to decide whether to skip the directory named by `path`.
+type FindAllFilesWithNameSkipCheck func(path string) bool
+
+// Commonly we only want to skip `vendor` directories in `FindAllFilesWithName`. This array is a suitable
+// argument for `dirsToSkip` which skips `vendor` directories.
+var SkipVendorChecks = []FindAllFilesWithNameSkipCheck{IsGolangVendorDirectory}
+
+// Returns an array of all files matching `name` within the path at `root`.
+// The `dirsToSkip` array contains check functions used to decide which directories to skip.
+func FindAllFilesWithName(root string, name string, dirsToSkip ...FindAllFilesWithNameSkipCheck) []string {
 	paths := make([]string, 0, 1)
 	filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
 		if d.IsDir() {
 			for _, dirToSkip := range dirsToSkip {
-				if path == dirToSkip {
+				if dirToSkip(path) {
 					return filepath.SkipDir
 				}
 			}
@@ -287,3 +296,16 @@ func getImportPathFromRepoURL(repourl string) string {
 	path = regexp.MustCompile(`^/+|\.git$`).ReplaceAllString(path, "")
 	return host + "/" + path
 }
+
+// Decides if `path` refers to a file that exists.
+func fileExists(path string) bool {
+	stat, err := os.Stat(path)
+	return err == nil && stat.Mode().IsRegular()
+}
+
+// Decides if `dirPath` is a vendor directory by testing whether it is called `vendor`
+// and contains a `modules.txt` file.
+func IsGolangVendorDirectory(dirPath string) bool {
+	return filepath.Base(dirPath) == "vendor" &&
+		(fileExists(filepath.Join(dirPath, "modules.txt")) || fileExists(filepath.Join(dirPath, "../glide.yaml")))
+}

go/ql/integration-tests/mixed-layout/src/module/go.mod

@@ -1,5 +1,3 @@
 go 1.14
 
-require golang.org/x/net v0.23.0
-
 module module

go/ql/integration-tests/mixed-layout/src/module/go.sum

@@ -1,45 +0,0 @@
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
-golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

go/ql/integration-tests/mixed-layout/src/module/test.go

@@ -2,12 +2,8 @@ package subdir
 
 import (
 	"fmt"
-
-	"golang.org/x/net/ipv4"
 )
 
 func test() {
-
-	header := ipv4.Header{}
-	fmt.Print(header.String())
+	fmt.Print("Hello world")
 }

go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt

@@ -0,0 +1,2 @@
+# example.com/test v0.1.0
+example.com/test

go/ql/integration-tests/mixed-layout/src/workspace/go.work.sum

@@ -0,0 +1,3 @@
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=