Gosum support #475
+163
−1
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -16,14 +16,17 @@ | |
| package gomod | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "context" | ||
| "fmt" | ||
| "go/version" | ||
| "io" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "github.com/google/osv-scalibr/extractor" | ||
| "github.com/google/osv-scalibr/extractor/filesystem" | ||
| "github.com/google/osv-scalibr/log" | ||
| "github.com/google/osv-scalibr/plugin" | ||
| "github.com/google/osv-scalibr/purl" | ||
| "golang.org/x/exp/maps" | ||
|
|
@@ -112,8 +115,25 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([] | |
| } | ||
| } | ||
|
|
||
| isGoVersionSpecified := parsedLockfile.Go != nil && parsedLockfile.Go.Version != "" | ||
|
|
||
| // At go 1.17 and above, the go command adds an indirect requirement for each module that provides any | ||
| // package imported (even indirectly) by a package or test in the main module or passed as an argument to go get. | ||
| // | ||
| // for versions below extract indirect dependencies from the go.sum file | ||
| if isGoVersionSpecified && version.Compare("go"+parsedLockfile.Go.Version, "go1.17") < 0 { | ||
| sumPackages, err := extractFromSum(input) | ||
| if err != nil { | ||
| log.Warnf("Error reading go.sum file: %s", err) | ||
| } else { | ||
| for _, p := range sumPackages { | ||
| packages[mapKey{name: p.Name, version: p.Version}] = p | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // Add the Go stdlib as an explicit dependency. | ||
| if isGoVersionSpecified { | ||
| packages[mapKey{name: "stdlib"}] = &extractor.Inventory{ | ||
| Name: "stdlib", | ||
| Version: parsedLockfile.Go.Version, | ||
|
|
@@ -144,4 +164,52 @@ func (e Extractor) Ecosystem(i *extractor.Inventory) string { | |
| return "Go" | ||
| } | ||
|
|
||
| // extractFromSum extracts dependencies from the go.sum file. | ||
| // | ||
| // Note: This function may produce false positives, as the go.sum file might be outdated. | ||
| func extractFromSum(input *filesystem.ScanInput) ([]*extractor.Inventory, error) { | ||
|
There was a problem hiding this comment. Does this mean go.sum files only exist when a go.mod file is present, no matter what go version? There was a problem hiding this comment. Yes, the
|
||
| goSumPath := strings.TrimSuffix(input.Path, ".mod") + ".sum" | ||
| f, err := input.FS.Open(goSumPath) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| scanner := bufio.NewScanner(f) | ||
| packages := []*extractor.Inventory{} | ||
|
|
||
| for lineNumber := 0; scanner.Scan(); lineNumber++ { | ||
| line := scanner.Text() | ||
|
|
||
| if line == "" { | ||
| continue | ||
| } | ||
|
|
||
| parts := strings.Fields(line) | ||
| if len(parts) != 3 { | ||
| return nil, fmt.Errorf("Error reading go.sum file: wrongly formatted line %s:%d", goSumPath, lineNumber) | ||
| } | ||
|
|
||
| name := parts[0] | ||
| version := strings.TrimPrefix(parts[1], "v") | ||
|
|
||
| // skip a line if the version contains "/go.mod" because lines | ||
| // containing "/go.mod" are duplicates used to verify the hash of the go.mod file | ||
| if strings.Contains(version, "/go.mod") { | ||
| continue | ||
| } | ||
|
|
||
| packages = append(packages, &extractor.Inventory{ | ||
| Name: name, | ||
| Version: version, | ||
| Locations: []string{goSumPath}, | ||
| }) | ||
| } | ||
|
|
||
| if err := scanner.Err(); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return packages, nil | ||
| } | ||
|
|
||
| var _ filesystem.Extractor = Extractor{} | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| module test | ||
|
|
||
| go 1.16 | ||
|
|
||
| require github.com/sirupsen/logrus v1.9.3 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | ||
| github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= | ||
| github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | ||
| github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= | ||
| github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | ||
| github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= | ||
| github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= | ||
| github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= | ||
| github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= | ||
| github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= | ||
| golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= | ||
| golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= | ||
| gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= | ||
| gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= | ||
| gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| module test | ||
|
|
||
| go 1.23 | ||
|
|
||
| require github.com/sirupsen/logrus v1.9.3 | ||
|
|
||
| require golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| Wrongly formatted, this file should not be used!!! |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Adding this will overwrite packages from go.mod, right? I think we don't want to do this deduplication. If we skip it we can also add go.sum in FileRequired and replace Extract with a "if" between extractGoMod and extractGoSum. WDYT?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes
Should the extractor keep only the
go.moddependencies or both (in case of a collision)?I considered that approach, but since the feature request specifically involves handling
go.sumfiles for Golang versions below 1.17, the extractor must first read thego.modfile to determine the specified Go version. Then, if the version is below 1.17, it should open thego.sumfile to search for indirect dependencies.The main issue with including
go.sumin the FileRequired logic is that thego.sumfile doesn’t include the Go version, which is necessary for deciding whether or not to return the dependencies from thego.sumAm I missing something here?