From 307c17df05bbf37439c04bd67ae7f9a977af973d Mon Sep 17 00:00:00 2001 From: Niklas Date: Wed, 29 Sep 2021 20:20:08 +0200 Subject: [PATCH] fix: use main package instead of main file as entrypoint for app command (#78) * use main package instead of main file as entrypoint for app command also add more debug logging and improve existing logging messages Signed-off-by: nscuro * fix examples dockerfile and regenerate example sboms Signed-off-by: nscuro * update readme Signed-off-by: nscuro * minor logging adjustments Signed-off-by: nscuro * update changelog Signed-off-by: nscuro Closes #75 --- CHANGELOG.md | 2 + Dockerfile.examples | 2 +- README.md | 10 ++--- e2e/cmd_app_test.go | 12 +++--- examples/app_minikube-v1.23.1.bom.json | 16 +++---- examples/bin_minikube-v1.23.1.bom.json | 16 +++---- examples/mod_minikube-v1.23.1.bom.json | 16 +++---- internal/cli/cmd/app/app.go | 29 +++++++------ internal/cli/cmd/app/options.go | 57 ++++++++---------------- internal/cli/cmd/app/options_test.go | 39 ++++++++--------- internal/cli/cmd/bin/bin.go | 2 +- internal/cli/cmd/mod/mod.go | 2 +- internal/cli/util/util.go | 8 ++++ internal/gocmd/gocmd.go | 14 ++++-- internal/gomod/binary.go | 4 +- internal/gomod/filter.go | 20 +++++++-- internal/gomod/graph.go | 14 ++++-- internal/gomod/module.go | 21 ++++++--- internal/gomod/package.go | 60 ++++++++++++++++++++++++-- internal/gomod/vendor.go | 9 +++- internal/gomod/version.go | 11 ++++- internal/sbom/convert/file/file.go | 5 +++ internal/sbom/convert/module/module.go | 4 ++ internal/sbom/sbom.go | 3 ++ main.go | 8 +++- 25 files changed, 246 insertions(+), 138 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d27b19a2..bb732fdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ * Fix annotated tags not being recognized as versions ([#56](https://github.com/CycloneDX/cyclonedx-gomod/issues/56) via [#57](https://github.com/CycloneDX/cyclonedx-gomod/pull/57)) * Fix normalized versions interfering with hash calculation ([#58](https://github.com/CycloneDX/cyclonedx-gomod/issues/58) via [#60](https://github.com/CycloneDX/cyclonedx-gomod/pull/60)) +* Fix `app` command missing dependencies when `main` package is spread across multiple files ([#75](https://github.com/CycloneDX/cyclonedx-gomod/issues/75) via [#78](https://github.com/CycloneDX/cyclonedx-gomod/pull/78)) + * Also addresses [#76](https://github.com/CycloneDX/cyclonedx-gomod/issues/76) (thanks [TheDiveO](https://github.com/TheDiveO) for reporting!) ### Breaking Changes diff --git a/Dockerfile.examples b/Dockerfile.examples index 5c966aa2..ad5117ce 100644 --- a/Dockerfile.examples +++ b/Dockerfile.examples @@ -26,7 +26,7 @@ RUN apt update && \ # Create generation script RUN echo "#!/bin/bash\n\n\ -cyclonedx-gomod app -json -output /examples/app_minikube-v1.23.1.bom.json -licenses -main cmd/minikube/main.go /home/cdx/minikube \n\ +cyclonedx-gomod app -json -output /examples/app_minikube-v1.23.1.bom.json -licenses -main cmd/minikube /home/cdx/minikube \n\ cyclonedx-gomod mod -json -output /examples/mod_minikube-v1.23.1.bom.json -licenses /home/cdx/minikube \n\ cyclonedx-gomod bin -json -output /examples/bin_minikube-v1.23.1.bom.json -licenses -version v1.23.1 /home/cdx/minikube-linux-amd64 \n\ cyclonedx validate --input-file /examples/app_minikube-v1.23.1.bom.json --input-format json_v1_3 --fail-on-errors \n\ diff --git a/README.md b/README.md index 45aae775..841e5711 100644 --- a/README.md +++ b/README.md @@ -99,8 +99,8 @@ Applicable build constraints are included as properties of the main component. Because build constraints influence Go's module selection, an SBOM should be generated for each target in the build matrix. -The -main flag should be used to specify the path to the application's main file. -It must point to a go file within MODULE_PATH. The go file must have a "package main" declaration. +The -main flag should be used to specify the path to the application's main package. +It must point to a directory within MODULE_PATH. If not set, MODULE_PATH is assumed. By passing -files, all files that would be included in a binary will be attached as subcomponents of their respective module. File versions follow the v0.0.0-SHORTHASH pattern, @@ -108,13 +108,13 @@ where SHORTHASH is the first 12 characters of the file's SHA1 hash. Examples: $ GOARCH=arm64 GOOS=linux GOFLAGS="-tags=foo,bar" cyclonedx-gomod app -output linux-arm64.bom.xml - $ cyclonedx-gomod app -json -output acme-app.bom.json -files -licenses -main cmd/acme-app/main.go /usr/src/acme-module + $ cyclonedx-gomod app -json -output acme-app.bom.json -files -licenses -main cmd/acme-app /usr/src/acme-module FLAGS -files=false Include files -json=false Output in JSON -licenses=false Perform license detection - -main main.go Path to the application's main file, relative to MODULE_PATH + -main ... Path to the application's main package, relative to MODULE_PATH -noserial=false Omit serial number -output - Output file path (or - for STDOUT) -serial ... Serial number @@ -255,7 +255,7 @@ $ docker run -it --rm \ ``` > The image is based on `golang:1.17-alpine`. -> Please keep in mind that the Go version may influence module selection. +> When using the `app` command, please keep in mind that the Go version may influence module selection. > We generally recommend using a [precompiled binary](https://github.com/CycloneDX/cyclonedx-gomod/releases) instead. ## Important Notes diff --git a/e2e/cmd_app_test.go b/e2e/cmd_app_test.go index 3a7e982f..2d006fb2 100644 --- a/e2e/cmd_app_test.go +++ b/e2e/cmd_app_test.go @@ -34,7 +34,7 @@ func TestAppCmdSimple(t *testing.T) { SerialNumber: zeroUUID.String(), }, ModuleDir: fixturePath, - Main: "main.go", + Main: "", } runSnapshotIT(t, &appOptions.OutputOptions, func() error { return appcmd.Exec(appOptions) }) @@ -50,7 +50,7 @@ func TestAppCmdSimpleWithFiles(t *testing.T) { SerialNumber: zeroUUID.String(), }, ModuleDir: fixturePath, - Main: "main.go", + Main: "", IncludeFiles: true, } @@ -67,7 +67,7 @@ func TestAppCmdSimpleMultiCommandUUID(t *testing.T) { SerialNumber: zeroUUID.String(), }, ModuleDir: fixturePath, - Main: "cmd/uuid/main.go", + Main: "cmd/uuid", } runSnapshotIT(t, &appOptions.OutputOptions, func() error { return appcmd.Exec(appOptions) }) @@ -83,7 +83,7 @@ func TestAppCmdSimpleMultiCommandPURL(t *testing.T) { SerialNumber: zeroUUID.String(), }, ModuleDir: fixturePath, - Main: "cmd/purl/main.go", + Main: "cmd/purl", } runSnapshotIT(t, &appOptions.OutputOptions, func() error { return appcmd.Exec(appOptions) }) @@ -99,7 +99,7 @@ func TestAppCmdVendored(t *testing.T) { SerialNumber: zeroUUID.String(), }, ModuleDir: fixturePath, - Main: "main.go", + Main: "", } runSnapshotIT(t, &appOptions.OutputOptions, func() error { return appcmd.Exec(appOptions) }) @@ -115,7 +115,7 @@ func TestAppCmdVendoredWithFiles(t *testing.T) { SerialNumber: zeroUUID.String(), }, ModuleDir: fixturePath, - Main: "main.go", + Main: "", IncludeFiles: true, } diff --git a/examples/app_minikube-v1.23.1.bom.json b/examples/app_minikube-v1.23.1.bom.json index b1ee093c..54bf642e 100644 --- a/examples/app_minikube-v1.23.1.bom.json +++ b/examples/app_minikube-v1.23.1.bom.json @@ -1,35 +1,35 @@ { "bomFormat": "CycloneDX", "specVersion": "1.3", - "serialNumber": "urn:uuid:89b6f930-138f-47a2-bae1-effe046d6545", + "serialNumber": "urn:uuid:9a4f199c-bff6-4fa1-a34d-4b7586617568", "version": 1, "metadata": { - "timestamp": "2021-09-27T18:10:47Z", + "timestamp": "2021-09-29T18:02:11Z", "tools": [ { "vendor": "CycloneDX", "name": "cyclonedx-gomod", - "version": "v0.0.0-20210927200930-566af677a239", + "version": "v0.0.0-20210929195822-2add6b416eb9", "hashes": [ { "alg": "MD5", - "content": "a769725a7cf4d002c88e0b59a2a3c6a4" + "content": "b6ccbe5ff272355fed1803c4c17ff161" }, { "alg": "SHA-1", - "content": "b49e5f1259ce630e3b7c678de45c600a313973fb" + "content": "2232bdbfd4ef618303a59c9d42f81ccfd2fa4e8a" }, { "alg": "SHA-256", - "content": "c5493aaa2f9f288550f87243cf9953f475e967982d5f004b1ed4836843462f3a" + "content": "69f0bce5fd9cdae1e3bd65b6abddd5bc966d4842dd69e33e8433783ee069c23c" }, { "alg": "SHA-384", - "content": "fee0fd5c99ecd1c6ec8028c8496778ea932f1601605d56601f29bee6c0e1470908bd7f737aaea640c88f06f8abfa00e1" + "content": "a310e367777f74d68b822c78f3d6a72aa888daae18d967be614f666a7b5916c9d1c86dbda0adc91f3c92309ddb5489d4" }, { "alg": "SHA-512", - "content": "52897bbd73eec9ce6a96fafd596a1e43568040d9208553786f0c7d60d5bd82b8d328648b66f9e556ed26f1666eabfef75ea191fab2ebeaa4636d249a73047910" + "content": "8924da342ab6da849631f2f5eb875b40eb62c54a6573e37e31e5ed5ac0f1f576d6eb266038695fec6514fe8973cc980ef7d8d7fb1ea5488fe7ec98a6ed848553" } ] } diff --git a/examples/bin_minikube-v1.23.1.bom.json b/examples/bin_minikube-v1.23.1.bom.json index b484ecf1..ecd6fc28 100644 --- a/examples/bin_minikube-v1.23.1.bom.json +++ b/examples/bin_minikube-v1.23.1.bom.json @@ -1,35 +1,35 @@ { "bomFormat": "CycloneDX", "specVersion": "1.3", - "serialNumber": "urn:uuid:a557bfa0-02dd-450a-b350-23a8639aeff8", + "serialNumber": "urn:uuid:2eadc368-815b-420c-8886-f9ad4730fdc7", "version": 1, "metadata": { - "timestamp": "2021-09-27T18:11:30Z", + "timestamp": "2021-09-29T18:02:39Z", "tools": [ { "vendor": "CycloneDX", "name": "cyclonedx-gomod", - "version": "v0.0.0-20210927200930-566af677a239", + "version": "v0.0.0-20210929195822-2add6b416eb9", "hashes": [ { "alg": "MD5", - "content": "a769725a7cf4d002c88e0b59a2a3c6a4" + "content": "b6ccbe5ff272355fed1803c4c17ff161" }, { "alg": "SHA-1", - "content": "b49e5f1259ce630e3b7c678de45c600a313973fb" + "content": "2232bdbfd4ef618303a59c9d42f81ccfd2fa4e8a" }, { "alg": "SHA-256", - "content": "c5493aaa2f9f288550f87243cf9953f475e967982d5f004b1ed4836843462f3a" + "content": "69f0bce5fd9cdae1e3bd65b6abddd5bc966d4842dd69e33e8433783ee069c23c" }, { "alg": "SHA-384", - "content": "fee0fd5c99ecd1c6ec8028c8496778ea932f1601605d56601f29bee6c0e1470908bd7f737aaea640c88f06f8abfa00e1" + "content": "a310e367777f74d68b822c78f3d6a72aa888daae18d967be614f666a7b5916c9d1c86dbda0adc91f3c92309ddb5489d4" }, { "alg": "SHA-512", - "content": "52897bbd73eec9ce6a96fafd596a1e43568040d9208553786f0c7d60d5bd82b8d328648b66f9e556ed26f1666eabfef75ea191fab2ebeaa4636d249a73047910" + "content": "8924da342ab6da849631f2f5eb875b40eb62c54a6573e37e31e5ed5ac0f1f576d6eb266038695fec6514fe8973cc980ef7d8d7fb1ea5488fe7ec98a6ed848553" } ] } diff --git a/examples/mod_minikube-v1.23.1.bom.json b/examples/mod_minikube-v1.23.1.bom.json index d3515116..1a6d3e0a 100644 --- a/examples/mod_minikube-v1.23.1.bom.json +++ b/examples/mod_minikube-v1.23.1.bom.json @@ -1,35 +1,35 @@ { "bomFormat": "CycloneDX", "specVersion": "1.3", - "serialNumber": "urn:uuid:f100c46e-2edb-4177-a4d9-2185af3313d2", + "serialNumber": "urn:uuid:984105ea-edc3-439b-bc11-3659aa99950c", "version": 1, "metadata": { - "timestamp": "2021-09-27T18:11:08Z", + "timestamp": "2021-09-29T18:02:27Z", "tools": [ { "vendor": "CycloneDX", "name": "cyclonedx-gomod", - "version": "v0.0.0-20210927200930-566af677a239", + "version": "v0.0.0-20210929195822-2add6b416eb9", "hashes": [ { "alg": "MD5", - "content": "a769725a7cf4d002c88e0b59a2a3c6a4" + "content": "b6ccbe5ff272355fed1803c4c17ff161" }, { "alg": "SHA-1", - "content": "b49e5f1259ce630e3b7c678de45c600a313973fb" + "content": "2232bdbfd4ef618303a59c9d42f81ccfd2fa4e8a" }, { "alg": "SHA-256", - "content": "c5493aaa2f9f288550f87243cf9953f475e967982d5f004b1ed4836843462f3a" + "content": "69f0bce5fd9cdae1e3bd65b6abddd5bc966d4842dd69e33e8433783ee069c23c" }, { "alg": "SHA-384", - "content": "fee0fd5c99ecd1c6ec8028c8496778ea932f1601605d56601f29bee6c0e1470908bd7f737aaea640c88f06f8abfa00e1" + "content": "a310e367777f74d68b822c78f3d6a72aa888daae18d967be614f666a7b5916c9d1c86dbda0adc91f3c92309ddb5489d4" }, { "alg": "SHA-512", - "content": "52897bbd73eec9ce6a96fafd596a1e43568040d9208553786f0c7d60d5bd82b8d328648b66f9e556ed26f1666eabfef75ea191fab2ebeaa4636d249a73047910" + "content": "8924da342ab6da849631f2f5eb875b40eb62c54a6573e37e31e5ed5ac0f1f576d6eb266038695fec6514fe8973cc980ef7d8d7fb1ea5488fe7ec98a6ed848553" } ] } diff --git a/internal/cli/cmd/app/app.go b/internal/cli/cmd/app/app.go index 8ddc3220..51a17bf1 100644 --- a/internal/cli/cmd/app/app.go +++ b/internal/cli/cmd/app/app.go @@ -21,6 +21,7 @@ import ( "context" "flag" "fmt" + "os" "path/filepath" "strings" @@ -64,8 +65,8 @@ Applicable build constraints are included as properties of the main component. Because build constraints influence Go's module selection, an SBOM should be generated for each target in the build matrix. -The -main flag should be used to specify the path to the application's main file. -It must point to a go file within MODULE_PATH. The go file must have a "package main" declaration. +The -main flag should be used to specify the path to the application's main package. +It must point to a directory within MODULE_PATH. If not set, MODULE_PATH is assumed. By passing -files, all files that would be included in a binary will be attached as subcomponents of their respective module. File versions follow the v0.0.0-SHORTHASH pattern, @@ -73,7 +74,7 @@ where SHORTHASH is the first 12 characters of the file's SHA1 hash. Examples: $ GOARCH=arm64 GOOS=linux GOFLAGS="-tags=foo,bar" cyclonedx-gomod app -output linux-arm64.bom.xml - $ cyclonedx-gomod app -json -output acme-app.bom.json -files -licenses -main cmd/acme-app/main.go /usr/src/acme-module`, + $ cyclonedx-gomod app -json -output acme-app.bom.json -files -licenses -main cmd/acme-app /usr/src/acme-module`, FlagSet: fs, Exec: func(_ context.Context, args []string) error { if len(args) > 1 { @@ -98,9 +99,9 @@ func Exec(options Options) error { return err } - modules, err := gomod.GetModulesFromPackages(options.ModuleDir, options.Main) + modules, err := gomod.LoadModulesFromPackages(options.ModuleDir, options.Main) if err != nil { - return fmt.Errorf("failed to collect modules: %w", err) + return fmt.Errorf("failed to load modules: %w", err) } // Dependencies need to be applied prior to determining the main @@ -253,23 +254,23 @@ func parseTagsFromGoFlags(goflags string) (tags []string) { // // If the package URL is updated, the BOM reference is as well. // All places within the BOM that reference the main component will be updated accordingly. -func enrichWithApplicationDetails(bom *cdx.BOM, moduleDir, mainFile string) { - // Resolve absolute paths to moduleDir and mainFile. +func enrichWithApplicationDetails(bom *cdx.BOM, moduleDir, mainPkgDir string) { + // Resolve absolute paths to moduleDir and mainPkgDir. // Both may contain traversals or similar elements we don't care about. // This procedure is done during options validation already, // which is why we don't check for errors here. moduleDirAbs, _ := filepath.Abs(moduleDir) - mainFileAbs, _ := filepath.Abs(filepath.Join(moduleDirAbs, mainFile)) + mainPkgDirAbs, _ := filepath.Abs(filepath.Join(moduleDirAbs, mainPkgDir)) - // Construct path to mainFile relative to moduleDir - mainFileRel := strings.TrimPrefix(mainFileAbs, moduleDirAbs) - mainFileRel = strings.TrimPrefix(mainFileRel, "/") + // Construct path to mainPkgDir relative to moduleDir + mainPkgDirRel := strings.TrimPrefix(mainPkgDirAbs, moduleDirAbs) + mainPkgDirRel = strings.TrimPrefix(mainPkgDirRel, string(os.PathSeparator)) - if mainDir, _ := filepath.Split(mainFileRel); mainDir != "" { - mainDir = strings.TrimSuffix(mainDir, "/") + if mainPkgDirRel != "" { + mainPkgDirRel = strings.TrimSuffix(mainPkgDirRel, string(os.PathSeparator)) oldPURL := bom.Metadata.Component.PackageURL - newPURL := oldPURL + "#" + mainDir + newPURL := oldPURL + "#" + mainPkgDirRel log.Debug(). Str("old", oldPURL). diff --git a/internal/cli/cmd/app/options.go b/internal/cli/cmd/app/options.go index dc1deae7..96269bd7 100644 --- a/internal/cli/cmd/app/options.go +++ b/internal/cli/cmd/app/options.go @@ -18,17 +18,14 @@ package app import ( - "bufio" "errors" "flag" "fmt" - "io" - "os" - "path/filepath" - "strings" - "github.com/CycloneDX/cyclonedx-gomod/internal/cli/options" + "github.com/CycloneDX/cyclonedx-gomod/internal/gomod" "github.com/CycloneDX/cyclonedx-gomod/internal/util" + "os" + "path/filepath" ) type Options struct { @@ -47,7 +44,7 @@ func (o *Options) RegisterFlags(fs *flag.FlagSet) { o.SBOMOptions.RegisterFlags(fs) fs.BoolVar(&o.IncludeFiles, "files", false, "Include files") - fs.StringVar(&o.Main, "main", "main.go", "Path to the application's main file, relative to MODULE_PATH") + fs.StringVar(&o.Main, "main", "", "Path to the application's main package, relative to MODULE_PATH") } func (o Options) Validate() error { @@ -82,15 +79,14 @@ func (o Options) Validate() error { return nil } -func (o Options) validateMain(mainFilePath string, errs *[]error) error { - mainFilePath = filepath.Join(o.ModuleDir, mainFilePath) - - if filepath.Ext(mainFilePath) != ".go" { - *errs = append(*errs, fmt.Errorf("main: must be a go source file, but \"%s\" is not", mainFilePath)) +func (o Options) validateMain(mainPkgDir string, errs *[]error) error { + if filepath.IsAbs(mainPkgDir) { + *errs = append(*errs, fmt.Errorf("main: must be a relative path")) return nil } - isSubPath, err := util.IsSubPath(mainFilePath, o.ModuleDir) + mainPkgDir = filepath.Join(o.ModuleDir, mainPkgDir) + isSubPath, err := util.IsSubPath(mainPkgDir, o.ModuleDir) if err != nil { return err } @@ -99,46 +95,27 @@ func (o Options) validateMain(mainFilePath string, errs *[]error) error { return nil } - fileInfo, err := os.Stat(mainFilePath) + fileInfo, err := os.Stat(mainPkgDir) if err != nil { if errors.Is(err, os.ErrNotExist) { - *errs = append(*errs, fmt.Errorf("main: \"%s\" does not exist", mainFilePath)) + *errs = append(*errs, fmt.Errorf("main: \"%s\" does not exist", mainPkgDir)) return nil } return err } - - if fileInfo.IsDir() { - *errs = append(*errs, fmt.Errorf("main: must be a go file, but \"%s\" is a directory", mainFilePath)) + if !fileInfo.IsDir() { + *errs = append(*errs, fmt.Errorf("main: must be a directory, but \"%s\" is a file", mainPkgDir)) return nil } - isMain, err := checkForMainPackage(mainFilePath) + pkg, err := gomod.LoadPackage(o.ModuleDir, o.Main) if err != nil { - return err + return fmt.Errorf("failed to load package: %w", err) } - if !isMain { - *errs = append(*errs, fmt.Errorf("main: \"%s\" is not a main file", mainFilePath)) + if pkg.Name != "main" { + *errs = append(*errs, fmt.Errorf("main: must be main package, but is \"%s\"", pkg.Name)) return nil } return nil } - -func checkForMainPackage(filePath string) (bool, error) { - mainFile, err := os.Open(filePath) - if err != nil { - return false, err - } - defer mainFile.Close() - - scanner := bufio.NewScanner(io.LimitReader(mainFile, 1024)) - for scanner.Scan() { - fields := strings.Fields(scanner.Text()) - if len(fields) >= 2 && fields[0] == "package" && fields[1] == "main" { - return true, nil - } - } - - return false, nil -} diff --git a/internal/cli/cmd/app/options_test.go b/internal/cli/cmd/app/options_test.go index 7495f106..17801b20 100644 --- a/internal/cli/cmd/app/options_test.go +++ b/internal/cli/cmd/app/options_test.go @@ -26,16 +26,7 @@ import ( ) func TestOptions_Validate(t *testing.T) { - t.Run("Main Isnt A Go File", func(t *testing.T) { - var options Options - options.Main = "./notGo.txt" - - err := options.Validate() - require.Error(t, err) - require.Contains(t, err.Error(), "must be a go source file") - }) - - t.Run("Main Isnt Subpath Of MODDIR", func(t *testing.T) { + t.Run("Main Isnt Subpath Of MODULE_PATH", func(t *testing.T) { var options Options options.ModuleDir = "/path/to/module" options.Main = "../main.go" @@ -48,16 +39,16 @@ func TestOptions_Validate(t *testing.T) { t.Run("Main Doesnt Exist", func(t *testing.T) { var options Options options.ModuleDir = "/path/to/module" - options.Main = "main.go" + options.Main = "cmd/app" err := options.Validate() require.Error(t, err) require.Contains(t, err.Error(), "does not exist") }) - t.Run("Main Is Directory", func(t *testing.T) { + t.Run("Main Is File", func(t *testing.T) { tmpDir := t.TempDir() - err := os.Mkdir(filepath.Join(tmpDir, "main.go"), os.ModePerm) + err := os.WriteFile(filepath.Join(tmpDir, "main.go"), []byte("package main"), os.ModePerm) require.NoError(t, err) var options Options @@ -66,31 +57,39 @@ func TestOptions_Validate(t *testing.T) { err = options.Validate() require.Error(t, err) - require.Contains(t, err.Error(), "is a directory") + require.Contains(t, err.Error(), "must be a directory") }) - t.Run("Main Isnt A Main File", func(t *testing.T) { + t.Run("Main Isnt A Main Package", func(t *testing.T) { tmpDir := t.TempDir() - err := os.WriteFile(filepath.Join(tmpDir, "main.go"), []byte(`package notmain`), os.ModePerm) + err := os.MkdirAll(filepath.Join(tmpDir, "cmd/app"), os.ModePerm) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(tmpDir, "go.mod"), []byte("module foobar"), os.ModePerm) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(tmpDir, "cmd/app/main.go"), []byte("package baz"), os.ModePerm) require.NoError(t, err) var options Options options.ModuleDir = tmpDir - options.Main = "main.go" + options.Main = "cmd/app" err = options.Validate() require.Error(t, err) - require.Contains(t, err.Error(), "not a main file") + require.Contains(t, err.Error(), "must be main package") }) t.Run("Success", func(t *testing.T) { tmpDir := t.TempDir() - err := os.WriteFile(filepath.Join(tmpDir, "main.go"), []byte(`package main // somecomment`), os.ModePerm) + err := os.MkdirAll(filepath.Join(tmpDir, "cmd/app"), os.ModePerm) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(tmpDir, "go.mod"), []byte("module foobar"), os.ModePerm) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(tmpDir, "cmd/app/main.go"), []byte("package main"), os.ModePerm) require.NoError(t, err) var options Options options.ModuleDir = tmpDir - options.Main = "main.go" + options.Main = "cmd/app" err = options.Validate() require.NoError(t, err) diff --git a/internal/cli/cmd/bin/bin.go b/internal/cli/cmd/bin/bin.go index f0f1c247..e5798893 100644 --- a/internal/cli/cmd/bin/bin.go +++ b/internal/cli/cmd/bin/bin.go @@ -81,7 +81,7 @@ func Exec(options Options) error { return err } - modules, hashes, err := gomod.GetModulesFromBinary(options.BinaryPath) + modules, hashes, err := gomod.LoadModulesFromBinary(options.BinaryPath) if err != nil { return fmt.Errorf("failed to extract modules: %w", err) } else if len(modules) == 0 { diff --git a/internal/cli/cmd/mod/mod.go b/internal/cli/cmd/mod/mod.go index 16f71c39..301942a8 100644 --- a/internal/cli/cmd/mod/mod.go +++ b/internal/cli/cmd/mod/mod.go @@ -84,7 +84,7 @@ func Exec(options Options) error { modules, err := gomod.GetVendoredModules(options.ModuleDir, options.IncludeTest) if err != nil { if errors.Is(err, gomod.ErrNotVendoring) { - modules, err = gomod.GetModules(options.ModuleDir, options.IncludeTest) + modules, err = gomod.LoadModules(options.ModuleDir, options.IncludeTest) if err != nil { return fmt.Errorf("failed to collect modules: %w", err) } diff --git a/internal/cli/util/util.go b/internal/cli/util/util.go index 1e303dbc..b16fad54 100644 --- a/internal/cli/util/util.go +++ b/internal/cli/util/util.go @@ -52,6 +52,9 @@ func AddCommonMetadata(bom *cdx.BOM, sbomOptions options.SBOMOptions) error { } func AddStdComponent(bom *cdx.BOM) error { + log.Debug(). + Msg("adding std component") + stdComponent, err := sbom.BuildStdComponent() if err != nil { return fmt.Errorf("failed to build std component: %w", err) @@ -113,6 +116,11 @@ func SetSerialNumber(bom *cdx.BOM, sbomOptions options.SBOMOptions) error { // WriteBOM writes the given bom according to the provided OutputOptions. func WriteBOM(bom *cdx.BOM, outputOptions options.OutputOptions) error { + log.Debug(). + Str("output", outputOptions.OutputFilePath). + Bool("json", outputOptions.UseJSON). + Msg("writing sbom") + var outputFormat cdx.BOMFileFormat if outputOptions.UseJSON { outputFormat = cdx.BOMFileFormatJSON diff --git a/internal/gocmd/gocmd.go b/internal/gocmd/gocmd.go index c5392bf3..a183073f 100644 --- a/internal/gocmd/gocmd.go +++ b/internal/gocmd/gocmd.go @@ -79,7 +79,15 @@ func ListModules(moduleDir string, writer io.Writer) error { return executeGoCommand([]string{"list", "-mod", "readonly", "-json", "-m", "all"}, withDir(moduleDir), withStdout(writer)) } -// ListPackages executed `go list -deps -json` and writes the output to a given writer. +// ListPackage executes `go list -json -e ` and writes the output to a given writer. +func ListPackage(moduleDir, packagePattern string, writer io.Writer) error { + return executeGoCommand([]string{"list", "-json", "-e", packagePattern}, + withDir(moduleDir), + withStdout(writer), + withStderr(os.Stderr)) +} + +// ListPackages executes `go list -deps -json ` and writes the output to a given writer. // See https://golang.org/cmd/go/#hdr-List_packages_or_modules func ListPackages(moduleDir, packagePattern string, writer io.Writer) error { return executeGoCommand([]string{"list", "-deps", "-json", packagePattern}, @@ -112,8 +120,8 @@ func ModWhy(moduleDir string, modules []string, writer io.Writer) error { ) } -// GetModulesFromBinary executes `go version -m` and writes the output to a given writer. -func GetModulesFromBinary(binaryPath string, writer io.Writer) error { +// LoadModulesFromBinary executes `go version -m` and writes the output to a given writer. +func LoadModulesFromBinary(binaryPath string, writer io.Writer) error { return executeGoCommand([]string{"version", "-m", binaryPath}, withStdout(writer)) } diff --git a/internal/gomod/binary.go b/internal/gomod/binary.go index 49dc9c8d..65f04465 100644 --- a/internal/gomod/binary.go +++ b/internal/gomod/binary.go @@ -26,9 +26,9 @@ import ( "github.com/CycloneDX/cyclonedx-gomod/internal/gocmd" ) -func GetModulesFromBinary(binaryPath string) ([]Module, map[string]string, error) { +func LoadModulesFromBinary(binaryPath string) ([]Module, map[string]string, error) { buf := new(bytes.Buffer) - if err := gocmd.GetModulesFromBinary(binaryPath, buf); err != nil { + if err := gocmd.LoadModulesFromBinary(binaryPath, buf); err != nil { return nil, nil, err } diff --git a/internal/gomod/filter.go b/internal/gomod/filter.go index bfa7adc0..e9ac5258 100644 --- a/internal/gomod/filter.go +++ b/internal/gomod/filter.go @@ -45,7 +45,13 @@ import ( // See: // - https://github.com/golang/go/issues/30720 // - https://github.com/golang/go/issues/26904 -func FilterModules(mainModulePath string, modules []Module, includeTest bool) ([]Module, error) { +func FilterModules(moduleDir string, modules []Module, includeTest bool) ([]Module, error) { + log.Debug(). + Str("moduleDir", moduleDir). + Int("moduleCount", len(modules)). + Bool("includeTest", includeTest). + Msg("filtering modules") + buf := new(bytes.Buffer) filtered := make([]Module, 0) chunks := chunkModules(modules, 20) @@ -56,13 +62,16 @@ func FilterModules(mainModulePath string, modules []Module, includeTest bool) ([ paths[i] = chunk[i].Path } - if err := gocmd.ModWhy(mainModulePath, paths, buf); err != nil { + if err := gocmd.ModWhy(moduleDir, paths, buf); err != nil { return nil, err } for modPath, modPkgs := range parseModWhy(buf) { if len(modPkgs) == 0 { - log.Debug().Str("module", modPath).Msg("filtering unneeded module") + log.Debug(). + Str("module", modPath). + Str("reason", "not needed"). + Msg("filtering module") continue } @@ -75,7 +84,10 @@ func FilterModules(mainModulePath string, modules []Module, includeTest bool) ([ } } if !includeTest && testOnly { - log.Debug().Str("module", modPath).Msg("filtering test-only module") + log.Debug(). + Str("module", modPath). + Str("reason", "test only"). + Msg("filtering module") continue } diff --git a/internal/gomod/graph.go b/internal/gomod/graph.go index 740d5b19..6e64fbac 100644 --- a/internal/gomod/graph.go +++ b/internal/gomod/graph.go @@ -31,8 +31,12 @@ import ( ) func ApplyModuleGraph(moduleDir string, modules []Module) error { - buf := new(bytes.Buffer) + log.Debug(). + Str("moduleDir", moduleDir). + Int("moduleCount", len(modules)). + Msg("applying module graph") + buf := new(bytes.Buffer) err := gocmd.GetModuleGraph(moduleDir, buf) if err != nil { return err @@ -51,7 +55,7 @@ func ApplyModuleGraph(moduleDir string, modules []Module) error { } // parseModuleGraph parses the output of `go mod graph` and populates -// the .Dependencies field of a given Module slice. +// the Dependencies field of a given Module slice. // // The Module slice is expected to contain only "effective" modules, // with only a single version per module, as provided by `go list -m` or `go list -deps`. @@ -83,7 +87,8 @@ func parseModuleGraph(reader io.Reader, modules []Module) error { log.Debug(). Str("dependant", dependant.Coordinates()). Str("dependency", fields[1]). - Msg("dependency not found") + Str("reason", "dependency not in list of selected modules"). + Msg("skipping graph edge") continue } @@ -91,7 +96,8 @@ func parseModuleGraph(reader io.Reader, modules []Module) error { log.Debug(). Str("dependant", dependant.Coordinates()). Str("dependency", dependency.Coordinates()). - Msg("pruning graph edge to indirect dependency") + Str("reason", "indirect dependency"). + Msg("skipping graph edge") continue } diff --git a/internal/gomod/module.go b/internal/gomod/module.go index 31cc8b67..cd56fc15 100644 --- a/internal/gomod/module.go +++ b/internal/gomod/module.go @@ -80,9 +80,12 @@ func IsModule(dir string) bool { // ErrNoModule indicates that a given path is not a valid Go module var ErrNoModule = errors.New("not a go module") -func GetModule(moduleDir string) (*Module, error) { - buf := new(bytes.Buffer) +func LoadModule(moduleDir string) (*Module, error) { + log.Debug(). + Str("moduleDir", moduleDir). + Msg("loading module") + buf := new(bytes.Buffer) err := gocmd.GetModule(moduleDir, buf) if err != nil { return nil, fmt.Errorf("listing module failed: %w", err) @@ -97,13 +100,17 @@ func GetModule(moduleDir string) (*Module, error) { return &module, nil } -func GetModules(moduleDir string, includeTest bool) ([]Module, error) { +func LoadModules(moduleDir string, includeTest bool) ([]Module, error) { + log.Debug(). + Str("moduleDir", moduleDir). + Bool("includeTest", includeTest). + Msg("loading modules") + if !IsModule(moduleDir) { return nil, ErrNoModule } buf := new(bytes.Buffer) - err := gocmd.ListModules(moduleDir, buf) if err != nil { return nil, fmt.Errorf("listing modules failed: %w", err) @@ -205,7 +212,11 @@ func ResolveLocalReplacements(mainModuleDir string, modules []Module) error { } func resolveLocalReplacement(localModuleDir string, module *Module) error { - localModule, err := GetModule(localModuleDir) + log.Debug(). + Str("moduleDir", localModuleDir). + Msg("resolving local replacement module") + + localModule, err := LoadModule(localModuleDir) if err != nil { return err } diff --git a/internal/gomod/package.go b/internal/gomod/package.go index eb383cda..0e11ffd0 100644 --- a/internal/gomod/package.go +++ b/internal/gomod/package.go @@ -51,15 +51,54 @@ type Package struct { SwigCXXFiles []string // .swigcxx files SysoFiles []string // .syso object files to add to archive EmbedFiles []string // files matched by EmbedPatterns + + Error *PackageError // error loading package +} + +type PackageError struct { + Err string } -func GetModulesFromPackages(moduleDir, packagePattern string) ([]Module, error) { +func (pe PackageError) Error() string { + return pe.Err +} + +func LoadPackage(moduleDir, packagePattern string) (*Package, error) { + log.Debug(). + Str("moduleDir", moduleDir). + Str("packagePattern", packagePattern). + Msg("loading package") + + buf := new(bytes.Buffer) + err := gocmd.ListPackage(moduleDir, toRelativePackagePath(packagePattern), buf) + if err != nil { + return nil, err + } + + var pkg Package + err = json.NewDecoder(buf).Decode(&pkg) + if err != nil { + return nil, err + } + + if pkg.Error != nil { + return nil, pkg.Error + } + + return &pkg, nil +} + +func LoadModulesFromPackages(moduleDir, packagePattern string) ([]Module, error) { + log.Debug(). + Str("moduleDir", moduleDir). + Msg("loading modules") + if !IsModule(moduleDir) { return nil, ErrNoModule } buf := new(bytes.Buffer) - err := gocmd.ListPackages(moduleDir, packagePattern, buf) + err := gocmd.ListPackages(moduleDir, toRelativePackagePath(packagePattern), buf) if err != nil { return nil, fmt.Errorf("failed to list packages for pattern \"%s\": %w", packagePattern, err) } @@ -103,13 +142,15 @@ func parsePackages(reader io.Reader) (map[string][]Package, error) { if pkg.Standard { log.Debug(). Str("package", pkg.ImportPath). - Msg("skipping standard library package") + Str("reason", "part of standard library"). + Msg("skipping package") continue } if pkg.Module == nil { log.Debug(). Str("package", pkg.ImportPath). - Msg("skipping package without module") + Str("reason", "no associated module"). + Msg("skipping package") continue } @@ -214,3 +255,14 @@ func makePackageFileRelativeToModule(moduleDir, pkgDir, file string) (string, er return strings.ReplaceAll(fp, "\\", "/"), nil } + +// toRelativePackagePath ensures that Go will interpret the given packagePattern +// as relative package path. This is done by prefixing it with "./" if it isn't already. +// See also: `go help packages` +func toRelativePackagePath(packagePattern string) string { + packagePattern = filepath.ToSlash(packagePattern) + if !strings.HasPrefix(packagePattern, "./") { + packagePattern = "./" + packagePattern + } + return packagePattern +} diff --git a/internal/gomod/vendor.go b/internal/gomod/vendor.go index d080d24d..197718cb 100644 --- a/internal/gomod/vendor.go +++ b/internal/gomod/vendor.go @@ -22,6 +22,7 @@ import ( "bytes" "errors" "fmt" + "github.com/rs/zerolog/log" "io" "path/filepath" "strings" @@ -45,8 +46,12 @@ func GetVendoredModules(moduleDir string, includeTest bool) ([]Module, error) { return nil, ErrNotVendoring } - buf := new(bytes.Buffer) + log.Debug(). + Str("moduleDir", moduleDir). + Bool("includeTest", includeTest). + Msg("loading vendored modules") + buf := new(bytes.Buffer) err := gocmd.ListVendoredModules(moduleDir, buf) if err != nil { return nil, fmt.Errorf("listing vendored modules failed: %w", err) @@ -68,7 +73,7 @@ func GetVendoredModules(moduleDir string, includeTest bool) ([]Module, error) { } // Main module is not included in vendored module list, so we have to get it separately - mainModule, err := GetModule(moduleDir) + mainModule, err := LoadModule(moduleDir) if err != nil { return nil, fmt.Errorf("failed to get main module: %w", err) } diff --git a/internal/gomod/version.go b/internal/gomod/version.go index c9e4128f..1918fd9b 100644 --- a/internal/gomod/version.go +++ b/internal/gomod/version.go @@ -37,6 +37,10 @@ import ( // upwards until the root directory is reached. This is done to accommodate // for multi-module repositories, where modules are not placed in the repo root. func GetModuleVersion(moduleDir string) (string, error) { + log.Debug(). + Str("moduleDir", moduleDir). + Msg("detecting module version") + repoDir, err := filepath.Abs(moduleDir) if err != nil { return "", err @@ -109,6 +113,10 @@ type tag struct { // GetLatestTag determines the latest tag relative to HEAD. // Only tags with valid semver are considered. func GetLatestTag(repo *git.Repository, headCommit *object.Commit) (*tag, error) { + log.Debug(). + Str("headCommit", headCommit.Hash.String()). + Msg("getting latest tag for head commit") + tagRefs, err := repo.Tags() if err != nil { return nil, err @@ -141,7 +149,8 @@ func GetLatestTag(repo *git.Repository, headCommit *object.Commit) (*tag, error) log.Debug(). Str("tag", ref.Name().Short()). Str("hash", ref.Hash().String()). - Msg("tag is not a valid semver") + Str("reason", "not a valid semver"). + Msg("skipping tag") } return nil diff --git a/internal/sbom/convert/file/file.go b/internal/sbom/convert/file/file.go index 88908803..c946c3c5 100644 --- a/internal/sbom/convert/file/file.go +++ b/internal/sbom/convert/file/file.go @@ -19,6 +19,7 @@ package file import ( "fmt" + "github.com/rs/zerolog/log" cdx "github.com/CycloneDX/cyclonedx-go" "github.com/CycloneDX/cyclonedx-gomod/internal/sbom" @@ -46,6 +47,10 @@ func WithScope(scope cdx.Scope) Option { } func ToComponent(absoluteFilePath, relativeFilePath string, options ...Option) (*cdx.Component, error) { + log.Debug(). + Str("file", absoluteFilePath). + Msg("converting file to component") + component := cdx.Component{ Type: cdx.ComponentTypeFile, Name: relativeFilePath, diff --git a/internal/sbom/convert/module/module.go b/internal/sbom/convert/module/module.go index 87431203..2680ae89 100644 --- a/internal/sbom/convert/module/module.go +++ b/internal/sbom/convert/module/module.go @@ -181,6 +181,10 @@ func ToComponent(module gomod.Module, options ...Option) (*cdx.Component, error) return ToComponent(*module.Replace, options...) } + log.Debug(). + Str("module", module.Coordinates()). + Msg("converting module to component") + component := cdx.Component{ BOMRef: module.PackageURL(), Type: cdx.ComponentTypeLibrary, diff --git a/internal/sbom/sbom.go b/internal/sbom/sbom.go index a6246fb5..30afbe3d 100644 --- a/internal/sbom/sbom.go +++ b/internal/sbom/sbom.go @@ -86,6 +86,9 @@ func BuildToolMetadata() (*cdx.Tool, error) { } func BuildStdComponent() (*cdx.Component, error) { + log.Debug(). + Msg("building std component") + goVersion, err := gocmd.GetVersion() if err != nil { return nil, fmt.Errorf("failed to determine Go version: %w", err) diff --git a/main.go b/main.go index 470352d3..f54dbc6b 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,8 @@ package main import ( "context" "fmt" + "github.com/CycloneDX/cyclonedx-gomod/internal/cli/options" + "github.com/rs/zerolog/log" "os" "github.com/CycloneDX/cyclonedx-gomod/internal/cli" @@ -28,7 +30,11 @@ import ( func main() { err := cli.New().ParseAndRun(context.Background(), os.Args[1:]) if err != nil { - _, _ = fmt.Fprintln(os.Stderr, err) + if _, ok := err.(*options.ValidationError); ok { + _, _ = fmt.Fprintln(os.Stderr, err) + } else { + log.Err(err).Msg("") + } os.Exit(1) } }