diff --git a/extractor/filesystem/language/golang/gomod/gomod.go b/extractor/filesystem/language/golang/gomod/gomod.go index 442a639f..886944f7 100644 --- a/extractor/filesystem/language/golang/gomod/gomod.go +++ b/extractor/filesystem/language/golang/gomod/gomod.go @@ -16,14 +16,17 @@ package gomod import ( + "bufio" "context" "fmt" + "go/version" "io" "path/filepath" "strings" "github.com/google/osv-scalibr/extractor" "github.com/google/osv-scalibr/extractor/filesystem" + "github.com/google/osv-scalibr/log" "github.com/google/osv-scalibr/plugin" "github.com/google/osv-scalibr/purl" "golang.org/x/exp/maps" @@ -112,8 +115,25 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] } } + isGoVersionSpecified := parsedLockfile.Go != nil && parsedLockfile.Go.Version != "" + + // At go 1.17 and above, the go command adds an indirect requirement for each module that provides any + // package imported (even indirectly) by a package or test in the main module or passed as an argument to go get. + // + // for versions below extract indirect dependencies from the go.sum file + if isGoVersionSpecified && version.Compare("go"+parsedLockfile.Go.Version, "go1.17") < 0 { + sumPackages, err := extractFromSum(input) + if err != nil { + log.Warnf("Error reading go.sum file: %s", err) + } else { + for _, p := range sumPackages { + packages[mapKey{name: p.Name, version: p.Version}] = p + } + } + } + // Add the Go stdlib as an explicit dependency. - if parsedLockfile.Go != nil && parsedLockfile.Go.Version != "" { + if isGoVersionSpecified { packages[mapKey{name: "stdlib"}] = &extractor.Inventory{ Name: "stdlib", Version: parsedLockfile.Go.Version, @@ -144,4 +164,52 @@ func (e Extractor) Ecosystem(i *extractor.Inventory) string { return "Go" } +// extractFromSum extracts dependencies from the go.sum file. +// +// Note: This function may produce false positives, as the go.sum file might be outdated. +func extractFromSum(input *filesystem.ScanInput) ([]*extractor.Inventory, error) { + goSumPath := strings.TrimSuffix(input.Path, ".mod") + ".sum" + f, err := input.FS.Open(goSumPath) + if err != nil { + return nil, err + } + + scanner := bufio.NewScanner(f) + packages := []*extractor.Inventory{} + + for lineNumber := 0; scanner.Scan(); lineNumber++ { + line := scanner.Text() + + if line == "" { + continue + } + + parts := strings.Fields(line) + if len(parts) != 3 { + return nil, fmt.Errorf("Error reading go.sum file: wrongly formatted line %s:%d", goSumPath, lineNumber) + } + + name := parts[0] + version := strings.TrimPrefix(parts[1], "v") + + // skip a line if the version contains "/go.mod" because lines + // containing "/go.mod" are duplicates used to verify the hash of the go.mod file + if strings.Contains(version, "/go.mod") { + continue + } + + packages = append(packages, &extractor.Inventory{ + Name: name, + Version: version, + Locations: []string{goSumPath}, + }) + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return packages, nil +} + var _ filesystem.Extractor = Extractor{} diff --git a/extractor/filesystem/language/golang/gomod/gomod_test.go b/extractor/filesystem/language/golang/gomod/gomod_test.go index f6a13f7e..b231c754 100644 --- a/extractor/filesystem/language/golang/gomod/gomod_test.go +++ b/extractor/filesystem/language/golang/gomod/gomod_test.go @@ -256,6 +256,72 @@ func TestExtractor_Extract(t *testing.T) { }, }, }, + { + Name: "test extractor for go > 1.16", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/indirect-1.23.mod", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "github.com/sirupsen/logrus", + Version: "1.9.3", + Locations: []string{"testdata/indirect-1.23.mod"}, + }, + { + Name: "golang.org/x/sys", + Version: "0.0.0-20220715151400-c0bba94af5f8", + Locations: []string{"testdata/indirect-1.23.mod"}, + }, + { + Name: "stdlib", + Version: "1.23", + Locations: []string{"testdata/indirect-1.23.mod"}, + }, + }, + }, + { + Name: "test extractor for go <=1.16", + InputConfig: extracttest.ScanInputMockConfig{ + Path: "testdata/indirect-1.16.mod", + }, + WantInventory: []*extractor.Inventory{ + { + Name: "github.com/davecgh/go-spew", + Version: "1.1.1", + Locations: []string{"testdata/indirect-1.16.sum"}, + }, + { + Name: "github.com/pmezard/go-difflib", + Version: "1.0.0", + Locations: []string{"testdata/indirect-1.16.sum"}, + }, + { + Name: "github.com/sirupsen/logrus", + Version: "1.9.3", + Locations: []string{"testdata/indirect-1.16.sum"}, + }, + { + Name: "github.com/stretchr/testify", + Version: "1.7.0", + Locations: []string{"testdata/indirect-1.16.sum"}, + }, + { + Name: "golang.org/x/sys", + Version: "0.0.0-20220715151400-c0bba94af5f8", + Locations: []string{"testdata/indirect-1.16.sum"}, + }, + { + Name: "gopkg.in/yaml.v3", + Version: "3.0.0-20200313102051-9f266ea9e77c", + Locations: []string{"testdata/indirect-1.16.sum"}, + }, + { + Name: "stdlib", + Version: "1.16", + Locations: []string{"testdata/indirect-1.16.mod"}, + }, + }, + }, } for _, tt := range tests { diff --git a/extractor/filesystem/language/golang/gomod/testdata/indirect-1.16.mod b/extractor/filesystem/language/golang/gomod/testdata/indirect-1.16.mod new file mode 100644 index 00000000..79f9f937 --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/testdata/indirect-1.16.mod @@ -0,0 +1,5 @@ +module test + +go 1.16 + +require github.com/sirupsen/logrus v1.9.3 diff --git a/extractor/filesystem/language/golang/gomod/testdata/indirect-1.16.sum b/extractor/filesystem/language/golang/gomod/testdata/indirect-1.16.sum new file mode 100644 index 00000000..21f9bfb3 --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/testdata/indirect-1.16.sum @@ -0,0 +1,15 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/extractor/filesystem/language/golang/gomod/testdata/indirect-1.23.mod b/extractor/filesystem/language/golang/gomod/testdata/indirect-1.23.mod new file mode 100644 index 00000000..f090c2d2 --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/testdata/indirect-1.23.mod @@ -0,0 +1,7 @@ +module test + +go 1.23 + +require github.com/sirupsen/logrus v1.9.3 + +require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect diff --git a/extractor/filesystem/language/golang/gomod/testdata/indirect-1.23.sum b/extractor/filesystem/language/golang/gomod/testdata/indirect-1.23.sum new file mode 100644 index 00000000..8d844fd1 --- /dev/null +++ b/extractor/filesystem/language/golang/gomod/testdata/indirect-1.23.sum @@ -0,0 +1 @@ +Wrongly formatted, this file should not be used!!!