diff --git a/go/extractor/configurebaseline/configurebaseline.go b/go/extractor/configurebaseline/configurebaseline.go index 04037d61425e..062289f7baa4 100644 --- a/go/extractor/configurebaseline/configurebaseline.go +++ b/go/extractor/configurebaseline/configurebaseline.go @@ -3,24 +3,12 @@ package configurebaseline import ( "encoding/json" "io/fs" - "os" "path" "path/filepath" "github.com/github/codeql-go/extractor/util" ) -func fileExists(path string) bool { - stat, err := os.Stat(path) - return err == nil && stat.Mode().IsRegular() -} - -// Decides if `dirPath` is a vendor directory by testing whether it is called `vendor` -// and contains a `modules.txt` file. -func isGolangVendorDirectory(dirPath string) bool { - return filepath.Base(dirPath) == "vendor" && fileExists(filepath.Join(dirPath, "modules.txt")) -} - type BaselineConfig struct { PathsIgnore []string `json:"paths-ignore"` } @@ -38,7 +26,7 @@ func GetConfigBaselineAsJSON(rootDir string) ([]byte, error) { // it will not be extracted either. return nil } - if isGolangVendorDirectory(dirPath) { + if util.IsGolangVendorDirectory(dirPath) { // Note that CodeQL expects a forward-slash-separated path, even on Windows. vendorDirs = append(vendorDirs, path.Join(filepath.ToSlash(dirPath), "**")) return filepath.SkipDir diff --git a/go/extractor/project/project.go b/go/extractor/project/project.go index 3745ca844028..eff69c0e9cac 100644 --- a/go/extractor/project/project.go +++ b/go/extractor/project/project.go @@ -184,12 +184,12 @@ func RemoveTemporaryExtractorFiles() { // Find all go.work files in the working directory and its subdirectories func findGoWorkFiles() []string { - return util.FindAllFilesWithName(".", "go.work", "vendor") + return util.FindAllFilesWithName(".", "go.work", util.SkipVendorChecks...) } // Find all go.mod files in the specified directory and its subdirectories func findGoModFiles(root string) []string { - return util.FindAllFilesWithName(root, "go.mod", "vendor") + return util.FindAllFilesWithName(root, "go.mod", util.SkipVendorChecks...) } // A regular expression for the Go toolchain version syntax. @@ -315,6 +315,11 @@ func discoverWorkspaces(emitDiagnostics bool) []GoWorkspace { goModFiles := findGoModFiles(".") // Return a separate workspace for each `go.mod` file that we found. + if len(goModFiles) > 0 { + log.Printf("Found %d go.mod files in: %s.\n", len(goModFiles), strings.Join(goModFiles, ", ")) + } else { + log.Println("Found no go.mod files in the workspace.") + } results := make([]GoWorkspace, len(goModFiles)) for i, goModFile := range goModFiles { @@ -547,8 +552,8 @@ func startsWithAnyOf(str string, prefixes []string) bool { // Finds Go workspaces in the current working directory. func GetWorkspaceInfo(emitDiagnostics bool) []GoWorkspace { bazelPaths := slices.Concat( - util.FindAllFilesWithName(".", "BUILD", "vendor"), - util.FindAllFilesWithName(".", "BUILD.bazel", "vendor"), + util.FindAllFilesWithName(".", "BUILD", util.SkipVendorChecks...), + util.FindAllFilesWithName(".", "BUILD.bazel", util.SkipVendorChecks...), ) if len(bazelPaths) > 0 { // currently not supported diff --git a/go/extractor/util/util.go b/go/extractor/util/util.go index 595326496d8a..fd3eed3e187f 100644 --- a/go/extractor/util/util.go +++ b/go/extractor/util/util.go @@ -152,7 +152,16 @@ func FindGoFiles(root string) bool { return found } -func FindAllFilesWithName(root string, name string, dirsToSkip ...string) []string { +// The type of check function used by `FindAllFilesWithName` to decide whether to skip the directory named by `path`. +type FindAllFilesWithNameSkipCheck func(path string) bool + +// Commonly we only want to skip `vendor` directories in `FindAllFilesWithName`. This array is a suitable +// argument for `dirsToSkip` which skips `vendor` directories. +var SkipVendorChecks = []FindAllFilesWithNameSkipCheck{IsGolangVendorDirectory} + +// Returns an array of all files matching `name` within the path at `root`. +// The `dirsToSkip` array contains check functions used to decide which directories to skip. +func FindAllFilesWithName(root string, name string, dirsToSkip ...FindAllFilesWithNameSkipCheck) []string { paths := make([]string, 0, 1) filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -160,7 +169,7 @@ func FindAllFilesWithName(root string, name string, dirsToSkip ...string) []stri } if d.IsDir() { for _, dirToSkip := range dirsToSkip { - if path == dirToSkip { + if dirToSkip(path) { return filepath.SkipDir } } @@ -287,3 +296,16 @@ func getImportPathFromRepoURL(repourl string) string { path = regexp.MustCompile(`^/+|\.git$`).ReplaceAllString(path, "") return host + "/" + path } + +// Decides if `path` refers to a file that exists. +func fileExists(path string) bool { + stat, err := os.Stat(path) + return err == nil && stat.Mode().IsRegular() +} + +// Decides if `dirPath` is a vendor directory by testing whether it is called `vendor` +// and contains a `modules.txt` file. +func IsGolangVendorDirectory(dirPath string) bool { + return filepath.Base(dirPath) == "vendor" && + (fileExists(filepath.Join(dirPath, "modules.txt")) || fileExists(filepath.Join(dirPath, "../glide.yaml"))) +} diff --git a/go/ql/integration-tests/mixed-layout/src/module/go.mod b/go/ql/integration-tests/mixed-layout/src/module/go.mod index 290a5c5c289e..c34a5e4715d6 100644 --- a/go/ql/integration-tests/mixed-layout/src/module/go.mod +++ b/go/ql/integration-tests/mixed-layout/src/module/go.mod @@ -1,5 +1,3 @@ go 1.14 -require golang.org/x/net v0.23.0 - module module diff --git a/go/ql/integration-tests/mixed-layout/src/module/go.sum b/go/ql/integration-tests/mixed-layout/src/module/go.sum index a8e1b59ae4b1..e69de29bb2d1 100644 --- a/go/ql/integration-tests/mixed-layout/src/module/go.sum +++ b/go/ql/integration-tests/mixed-layout/src/module/go.sum @@ -1,45 +0,0 @@ -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/go/ql/integration-tests/mixed-layout/src/module/test.go b/go/ql/integration-tests/mixed-layout/src/module/test.go index afc86ac3a126..502061dc123c 100644 --- a/go/ql/integration-tests/mixed-layout/src/module/test.go +++ b/go/ql/integration-tests/mixed-layout/src/module/test.go @@ -2,12 +2,8 @@ package subdir import ( "fmt" - - "golang.org/x/net/ipv4" ) func test() { - - header := ipv4.Header{} - fmt.Print(header.String()) + fmt.Print("Hello world") } diff --git a/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/add.go b/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/add.go new file mode 100644 index 000000000000..b1ce6a2a3a39 --- /dev/null +++ b/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/add.go @@ -0,0 +1,5 @@ +package test + +func Add(a, b int) int { + return a + b +} diff --git a/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/go.work b/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/go.work new file mode 100644 index 000000000000..96b89a39cb9a --- /dev/null +++ b/go/ql/integration-tests/mixed-layout/src/module/vendor/example.com/test/go.work @@ -0,0 +1,3 @@ +go 1.22.0 + +use . diff --git a/go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt b/go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt new file mode 100644 index 000000000000..a7181aa2289e --- /dev/null +++ b/go/ql/integration-tests/mixed-layout/src/module/vendor/modules.txt @@ -0,0 +1,2 @@ +# example.com/test v0.1.0 +example.com/test diff --git a/go/ql/integration-tests/mixed-layout/src/workspace/go.work.sum b/go/ql/integration-tests/mixed-layout/src/workspace/go.work.sum new file mode 100644 index 000000000000..d89ea795aab4 --- /dev/null +++ b/go/ql/integration-tests/mixed-layout/src/workspace/go.work.sum @@ -0,0 +1,3 @@ +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=