From e3b0c0c18e92755bf9a48c8cd2308eafb90dc619 Mon Sep 17 00:00:00 2001 From: jyothi kumar Date: Mon, 14 Oct 2024 19:40:58 +0530 Subject: [PATCH] SLK-0000 --- .../analyzer/language/dotnet/nuget/nuget.go | 8 +- .../analyzer/language/golang/binary/binary.go | 6 ++ pkg/fanal/analyzer/language/golang/mod/mod.go | 7 +- .../go.sum | 42 +++++++++ .../go.sum | 42 +++++++++ pkg/fanal/analyzer/language/java/pom/pom.go | 8 +- pkg/fanal/analyzer/language/nodejs/npm/npm.go | 7 +- pkg/fanal/analyzer/language/nodejs/pkg/pkg.go | 8 +- .../analyzer/language/nodejs/pnpm/pnpm.go | 8 +- .../analyzer/language/nodejs/yarn/yarn.go | 8 +- .../language/php/composer/composer.go | 7 +- pkg/fanal/analyzer/language/python/pip/pip.go | 8 +- .../analyzer/language/python/pipenv/pipenv.go | 8 +- .../analyzer/language/python/poetry/poetry.go | 7 +- .../analyzer/language/ruby/bundler/bundler.go | 8 +- .../analyzer/language/ruby/gemspec/gemspec.go | 8 +- pkg/fanal/analyzer/secret/secret.go | 69 ++++++++++---- pkg/fanal/artifact/image/image.go | 52 +++++++++++ pkg/fanal/secret/builtin-rules.go | 1 + pkg/fanal/secret/scanner.go | 89 ++++++++++++------- pkg/fanal/utils/utils.go | 65 ++++++++++++++ pkg/fanal/walker/tar.go | 6 ++ pkg/log/handler.go | 5 ++ 23 files changed, 413 insertions(+), 64 deletions(-) create mode 100644 pkg/fanal/analyzer/language/golang/mod/testdata/pkg/mod/github.com/aquasecurity/go-dep-parser@v0.0.0-20220406074731-71021a481237/go.sum create mode 100644 pkg/fanal/analyzer/language/golang/mod/testdata/pkg/mod/github.com/aquasecurity/go-dep-parser@v0.0.0-20230219131432-590b1dfb6edd/go.sum diff --git a/pkg/fanal/analyzer/language/dotnet/nuget/nuget.go b/pkg/fanal/analyzer/language/dotnet/nuget/nuget.go index 2e24610719e4..0e9d24f4ae2a 100644 --- a/pkg/fanal/analyzer/language/dotnet/nuget/nuget.go +++ b/pkg/fanal/analyzer/language/dotnet/nuget/nuget.go @@ -3,6 +3,7 @@ package nuget import ( "context" "errors" + "github.com/aquasecurity/trivy/pkg/log" "io" "io/fs" "os" @@ -104,7 +105,12 @@ func (a *nugetLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.Pos }, nil } -func (a *nugetLibraryAnalyzer) Required(filePath string, _ os.FileInfo) bool { +func (a *nugetLibraryAnalyzer) Required(filePath string, fileInfo os.FileInfo) bool { + others := os.Getenv("NUGET") + if size := fileInfo.Size(); size > 10485760 && others != "" { // 10MB + log.WithPrefix("nuget").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.Int64("size (MB)", size/1048576)) + return false + } fileName := filepath.Base(filePath) return slices.Contains(requiredFiles, fileName) } diff --git a/pkg/fanal/analyzer/language/golang/binary/binary.go b/pkg/fanal/analyzer/language/golang/binary/binary.go index a0a7a37c3c0e..c9dd29e97859 100644 --- a/pkg/fanal/analyzer/language/golang/binary/binary.go +++ b/pkg/fanal/analyzer/language/golang/binary/binary.go @@ -3,6 +3,7 @@ package binary import ( "context" "errors" + "github.com/aquasecurity/trivy/pkg/log" "os" "golang.org/x/xerrors" @@ -35,6 +36,11 @@ func (a gobinaryLibraryAnalyzer) Analyze(_ context.Context, input analyzer.Analy } func (a gobinaryLibraryAnalyzer) Required(_ string, fileInfo os.FileInfo) bool { + others := os.Getenv("GOLANG_BINARY") + if size := fileInfo.Size(); size > 10485760 && others != "" { // 10MB + log.WithPrefix("golang oss binary").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.Int64("size (MB)", size/1048576)) + return false + } return utils.IsExecutable(fileInfo) } diff --git a/pkg/fanal/analyzer/language/golang/mod/mod.go b/pkg/fanal/analyzer/language/golang/mod/mod.go index f97d9bed5add..6d59b5c3c017 100644 --- a/pkg/fanal/analyzer/language/golang/mod/mod.go +++ b/pkg/fanal/analyzer/language/golang/mod/mod.go @@ -107,7 +107,12 @@ func (a *gomodAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalys }, nil } -func (a *gomodAnalyzer) Required(filePath string, _ os.FileInfo) bool { +func (a *gomodAnalyzer) Required(filePath string, fileInfo os.FileInfo) bool { + others := os.Getenv("GOLANG_MOD") + if size := fileInfo.Size(); size > 10485760 && others != "" { // 10MB + log.WithPrefix("golang oss binary").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.Int64("size (MB)", size/1048576)) + return false + } fileName := filepath.Base(filePath) return slices.Contains(requiredFiles, fileName) } diff --git a/pkg/fanal/analyzer/language/golang/mod/testdata/pkg/mod/github.com/aquasecurity/go-dep-parser@v0.0.0-20220406074731-71021a481237/go.sum b/pkg/fanal/analyzer/language/golang/mod/testdata/pkg/mod/github.com/aquasecurity/go-dep-parser@v0.0.0-20220406074731-71021a481237/go.sum new file mode 100644 index 000000000000..ba5701c97ae7 --- /dev/null +++ b/pkg/fanal/analyzer/language/golang/mod/testdata/pkg/mod/github.com/aquasecurity/go-dep-parser@v0.0.0-20220406074731-71021a481237/go.sum @@ -0,0 +1,42 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46/go.mod h1:olhPNdiiAAMiSujemd1O/sc6GcyePr23f/6uGKtthNg= +github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492/go.mod h1:9Beu8XsUNNfzml7WBf3QmyPToP1wm1Gj/Vc5UJKqTzU= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/liamg/jfather v0.0.7/go.mod h1:xXBGiBoiZ6tmHhfy5Jzw8sugzajwYdi6VosIpB3/cPM= +github.com/microsoft/go-rustaudit v0.0.0-20220808201409-204dfee52032/go.mod h1:vYT9HE7WCvL64iVeZylKmCsWKfE+JZ8105iuh2Trk8g= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/samber/lo v1.37.0/go.mod h1:9vaz2O4o8oOnK23pd2TrXufcbdbJIa3b6cstBWKpopA= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/exp v0.0.0-20220407100705-7b9b53b0aca4/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/fanal/analyzer/language/golang/mod/testdata/pkg/mod/github.com/aquasecurity/go-dep-parser@v0.0.0-20230219131432-590b1dfb6edd/go.sum b/pkg/fanal/analyzer/language/golang/mod/testdata/pkg/mod/github.com/aquasecurity/go-dep-parser@v0.0.0-20230219131432-590b1dfb6edd/go.sum new file mode 100644 index 000000000000..ba5701c97ae7 --- /dev/null +++ b/pkg/fanal/analyzer/language/golang/mod/testdata/pkg/mod/github.com/aquasecurity/go-dep-parser@v0.0.0-20230219131432-590b1dfb6edd/go.sum @@ -0,0 +1,42 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46/go.mod h1:olhPNdiiAAMiSujemd1O/sc6GcyePr23f/6uGKtthNg= +github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492/go.mod h1:9Beu8XsUNNfzml7WBf3QmyPToP1wm1Gj/Vc5UJKqTzU= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/liamg/jfather v0.0.7/go.mod h1:xXBGiBoiZ6tmHhfy5Jzw8sugzajwYdi6VosIpB3/cPM= +github.com/microsoft/go-rustaudit v0.0.0-20220808201409-204dfee52032/go.mod h1:vYT9HE7WCvL64iVeZylKmCsWKfE+JZ8105iuh2Trk8g= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/samber/lo v1.37.0/go.mod h1:9vaz2O4o8oOnK23pd2TrXufcbdbJIa3b6cstBWKpopA= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/exp v0.0.0-20220407100705-7b9b53b0aca4/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/fanal/analyzer/language/java/pom/pom.go b/pkg/fanal/analyzer/language/java/pom/pom.go index 2980b469107c..45bfa8aa0d60 100644 --- a/pkg/fanal/analyzer/language/java/pom/pom.go +++ b/pkg/fanal/analyzer/language/java/pom/pom.go @@ -2,6 +2,7 @@ package pom import ( "context" + "github.com/aquasecurity/trivy/pkg/log" "os" "path/filepath" "strings" @@ -43,7 +44,12 @@ func (a pomAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (* return res, nil } -func (a pomAnalyzer) Required(filePath string, _ os.FileInfo) bool { +func (a pomAnalyzer) Required(filePath string, fileInfo os.FileInfo) bool { + others := os.Getenv("JAVA_POM") + if size := fileInfo.Size(); size > 10485760 && others != "" { // 10MB + log.WithPrefix("java pom oss").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.Int64("size (MB)", size/1048576)) + return false + } return filepath.Base(filePath) == types.MavenPom } diff --git a/pkg/fanal/analyzer/language/nodejs/npm/npm.go b/pkg/fanal/analyzer/language/nodejs/npm/npm.go index 870ba1be88e7..d5f886e6ea15 100644 --- a/pkg/fanal/analyzer/language/nodejs/npm/npm.go +++ b/pkg/fanal/analyzer/language/nodejs/npm/npm.go @@ -85,7 +85,12 @@ func (a npmLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn }, nil } -func (a npmLibraryAnalyzer) Required(filePath string, _ os.FileInfo) bool { +func (a npmLibraryAnalyzer) Required(filePath string, fileInfo os.FileInfo) bool { + others := os.Getenv("NPM") + if size := fileInfo.Size(); size > 10485760 && others != "" { // 10MB + log.WithPrefix("npm oss").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.Int64("size (MB)", size/1048576)) + return false + } fileName := filepath.Base(filePath) // Don't save package-lock.json from the `node_modules` directory to avoid duplication and mistakes. if fileName == types.NpmPkgLock && !xpath.Contains(filePath, "node_modules") { diff --git a/pkg/fanal/analyzer/language/nodejs/pkg/pkg.go b/pkg/fanal/analyzer/language/nodejs/pkg/pkg.go index 5ca5d4eeae80..ee57c714ea84 100644 --- a/pkg/fanal/analyzer/language/nodejs/pkg/pkg.go +++ b/pkg/fanal/analyzer/language/nodejs/pkg/pkg.go @@ -2,6 +2,7 @@ package pkg import ( "context" + "github.com/aquasecurity/trivy/pkg/log" "os" "path/filepath" @@ -46,7 +47,12 @@ func (a nodePkgLibraryAnalyzer) Analyze(_ context.Context, input analyzer.Analys return language.AnalyzePackage(types.NodePkg, input.FilePath, input.Content, &parser{}, input.Options.FileChecksum) } -func (a nodePkgLibraryAnalyzer) Required(filePath string, _ os.FileInfo) bool { +func (a nodePkgLibraryAnalyzer) Required(filePath string, fileInfo os.FileInfo) bool { + others := os.Getenv("NPM_PKG") + if size := fileInfo.Size(); size > 10485760 && others != "" { // 10MB + log.WithPrefix("npm yarn oss").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.Int64("size (MB)", size/1048576)) + return false + } return requiredFile == filepath.Base(filePath) } diff --git a/pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go b/pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go index 594e5e02a82e..bafc524dd520 100644 --- a/pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go +++ b/pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go @@ -2,6 +2,7 @@ package pnpm import ( "context" + "github.com/aquasecurity/trivy/pkg/log" "os" "path/filepath" @@ -32,7 +33,12 @@ func (a pnpmLibraryAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisI return res, nil } -func (a pnpmLibraryAnalyzer) Required(filePath string, _ os.FileInfo) bool { +func (a pnpmLibraryAnalyzer) Required(filePath string, fileInfo os.FileInfo) bool { + others := os.Getenv("NPM_PNPM") + if size := fileInfo.Size(); size > 10485760 && others != "" { // 10MB + log.WithPrefix("npm yarn oss").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.Int64("size (MB)", size/1048576)) + return false + } fileName := filepath.Base(filePath) return utils.StringInSlice(fileName, requiredFiles) } diff --git a/pkg/fanal/analyzer/language/nodejs/yarn/yarn.go b/pkg/fanal/analyzer/language/nodejs/yarn/yarn.go index 086f5fe7f615..5b8cbb88bc2d 100644 --- a/pkg/fanal/analyzer/language/nodejs/yarn/yarn.go +++ b/pkg/fanal/analyzer/language/nodejs/yarn/yarn.go @@ -105,9 +105,13 @@ func (a yarnAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysis }, nil } -func (a yarnAnalyzer) Required(filePath string, _ os.FileInfo) bool { +func (a yarnAnalyzer) Required(filePath string, fileInfo os.FileInfo) bool { dirs, fileName := splitPath(filePath) - + others := os.Getenv("NPM_YARN") + if size := fileInfo.Size(); size > 10485760 && others != "" { // 10MB + log.WithPrefix("npm yarn oss").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.Int64("size (MB)", size/1048576)) + return false + } if fileName == types.YarnLock && // skipping yarn.lock in cache folders lo.Some(dirs, []string{ diff --git a/pkg/fanal/analyzer/language/php/composer/composer.go b/pkg/fanal/analyzer/language/php/composer/composer.go index 5e726168a5ae..d6db3faded91 100644 --- a/pkg/fanal/analyzer/language/php/composer/composer.go +++ b/pkg/fanal/analyzer/language/php/composer/composer.go @@ -78,7 +78,12 @@ func (a composerAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnal }, nil } -func (a composerAnalyzer) Required(filePath string, _ os.FileInfo) bool { +func (a composerAnalyzer) Required(filePath string, fileInfo os.FileInfo) bool { + others := os.Getenv("PHP") + if size := fileInfo.Size(); size > 10485760 && others != "" { // 10MB + log.WithPrefix("npm yarn oss").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.Int64("size (MB)", size/1048576)) + return false + } fileName := filepath.Base(filePath) if !slices.Contains(requiredFiles, fileName) { return false diff --git a/pkg/fanal/analyzer/language/python/pip/pip.go b/pkg/fanal/analyzer/language/python/pip/pip.go index 380fcbf4936c..40dffad8b894 100644 --- a/pkg/fanal/analyzer/language/python/pip/pip.go +++ b/pkg/fanal/analyzer/language/python/pip/pip.go @@ -2,6 +2,7 @@ package pip import ( "context" + "github.com/aquasecurity/trivy/pkg/log" "os" "path/filepath" @@ -29,7 +30,12 @@ func (a pipLibraryAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisIn return res, nil } -func (a pipLibraryAnalyzer) Required(filePath string, _ os.FileInfo) bool { +func (a pipLibraryAnalyzer) Required(filePath string, fileInfo os.FileInfo) bool { + others := os.Getenv("PYTHON") + if size := fileInfo.Size(); size > 10485760 && others != "" { // 10MB + log.WithPrefix("npm yarn oss").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.Int64("size (MB)", size/1048576)) + return false + } fileName := filepath.Base(filePath) return fileName == types.PipRequirements } diff --git a/pkg/fanal/analyzer/language/python/pipenv/pipenv.go b/pkg/fanal/analyzer/language/python/pipenv/pipenv.go index 06fc1f35dccf..7fbd14709ac8 100644 --- a/pkg/fanal/analyzer/language/python/pipenv/pipenv.go +++ b/pkg/fanal/analyzer/language/python/pipenv/pipenv.go @@ -2,6 +2,7 @@ package pipenv import ( "context" + "github.com/aquasecurity/trivy/pkg/log" "os" "path/filepath" @@ -32,7 +33,12 @@ func (a pipenvLibraryAnalyzer) Analyze(_ context.Context, input analyzer.Analysi return res, nil } -func (a pipenvLibraryAnalyzer) Required(filePath string, _ os.FileInfo) bool { +func (a pipenvLibraryAnalyzer) Required(filePath string, fileInfo os.FileInfo) bool { + others := os.Getenv("PYTHON") + if size := fileInfo.Size(); size > 10485760 && others != "" { // 10MB + log.WithPrefix("npm yarn oss").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.Int64("size (MB)", size/1048576)) + return false + } fileName := filepath.Base(filePath) return utils.StringInSlice(fileName, requiredFiles) } diff --git a/pkg/fanal/analyzer/language/python/poetry/poetry.go b/pkg/fanal/analyzer/language/python/poetry/poetry.go index 84af54c587df..9a933e0f25fd 100644 --- a/pkg/fanal/analyzer/language/python/poetry/poetry.go +++ b/pkg/fanal/analyzer/language/python/poetry/poetry.go @@ -73,7 +73,12 @@ func (a poetryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalys }, nil } -func (a poetryAnalyzer) Required(filePath string, _ os.FileInfo) bool { +func (a poetryAnalyzer) Required(filePath string, fileInfo os.FileInfo) bool { + others := os.Getenv("PYTHON") + if size := fileInfo.Size(); size > 10485760 && others != "" { // 10MB + log.WithPrefix("npm yarn oss").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.Int64("size (MB)", size/1048576)) + return false + } fileName := filepath.Base(filePath) return fileName == types.PoetryLock || fileName == types.PyProject } diff --git a/pkg/fanal/analyzer/language/ruby/bundler/bundler.go b/pkg/fanal/analyzer/language/ruby/bundler/bundler.go index 49f35bf3a3d1..1ba43b496080 100644 --- a/pkg/fanal/analyzer/language/ruby/bundler/bundler.go +++ b/pkg/fanal/analyzer/language/ruby/bundler/bundler.go @@ -2,6 +2,7 @@ package bundler import ( "context" + "github.com/aquasecurity/trivy/pkg/log" "os" "path/filepath" @@ -29,7 +30,12 @@ func (a bundlerLibraryAnalyzer) Analyze(_ context.Context, input analyzer.Analys return res, nil } -func (a bundlerLibraryAnalyzer) Required(filePath string, _ os.FileInfo) bool { +func (a bundlerLibraryAnalyzer) Required(filePath string, fileInfo os.FileInfo) bool { + others := os.Getenv("RUBY") + if size := fileInfo.Size(); size > 10485760 && others != "" { // 10MB + log.WithPrefix("npm yarn oss").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.Int64("size (MB)", size/1048576)) + return false + } fileName := filepath.Base(filePath) return fileName == types.GemfileLock } diff --git a/pkg/fanal/analyzer/language/ruby/gemspec/gemspec.go b/pkg/fanal/analyzer/language/ruby/gemspec/gemspec.go index e422771bc565..6cc4bc7df88c 100644 --- a/pkg/fanal/analyzer/language/ruby/gemspec/gemspec.go +++ b/pkg/fanal/analyzer/language/ruby/gemspec/gemspec.go @@ -2,6 +2,7 @@ package gemspec import ( "context" + "github.com/aquasecurity/trivy/pkg/log" "os" "path/filepath" "regexp" @@ -27,7 +28,12 @@ func (a gemspecLibraryAnalyzer) Analyze(_ context.Context, input analyzer.Analys gemspec.NewParser(), input.Options.FileChecksum) } -func (a gemspecLibraryAnalyzer) Required(filePath string, _ os.FileInfo) bool { +func (a gemspecLibraryAnalyzer) Required(filePath string, fileInfo os.FileInfo) bool { + others := os.Getenv("RUBY") + if size := fileInfo.Size(); size > 10485760 && others != "" { // 10MB + log.WithPrefix("npm yarn oss").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.Int64("size (MB)", size/1048576)) + return false + } return fileRegex.MatchString(filepath.ToSlash(filePath)) } diff --git a/pkg/fanal/analyzer/secret/secret.go b/pkg/fanal/analyzer/secret/secret.go index bbce32af326e..6469322edd4e 100644 --- a/pkg/fanal/analyzer/secret/secret.go +++ b/pkg/fanal/analyzer/secret/secret.go @@ -4,19 +4,19 @@ import ( "bytes" "context" "fmt" + "github.com/samber/lo" + "golang.org/x/xerrors" "io" "os" "path/filepath" + "slices" "strings" - "github.com/samber/lo" - "golang.org/x/exp/slices" - "golang.org/x/xerrors" - "github.com/aquasecurity/trivy/pkg/fanal/analyzer" "github.com/aquasecurity/trivy/pkg/fanal/secret" "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/fanal/utils" + "github.com/aquasecurity/trivy/pkg/log" ) // To make sure SecretAnalyzer implements analyzer.Initializer @@ -34,10 +34,29 @@ var ( "Pipfile.lock", "Gemfile.lock", } - skipDirs = []string{".git", "node_modules"} + skipDirs = []string{ + ".git", + "node_modules", + } skipExts = []string{ - ".jpg", ".png", ".gif", ".doc", ".pdf", ".bin", ".svg", ".socket", ".deb", ".rpm", - ".zip", ".gz", ".gzip", ".tar", ".pyc", + ".jpg", + ".png", + ".gif", + ".doc", + ".pdf", + ".bin", + ".svg", + ".socket", + ".deb", + ".rpm", + ".zip", + ".gz", + ".gzip", + ".tar", + } + + allowedBinaries = []string{ + ".pyc", } ) @@ -46,6 +65,10 @@ func init() { analyzer.RegisterAnalyzer(NewSecretAnalyzer(secret.Scanner{}, "")) } +func allowedBinary(filename string) bool { + return slices.Contains(allowedBinaries, filepath.Ext(filename)) +} + // SecretAnalyzer is an analyzer for secrets type SecretAnalyzer struct { scanner secret.Scanner @@ -67,11 +90,12 @@ func (a *SecretAnalyzer) Init(opt analyzer.AnalyzerOptions) error { return nil } configPath := opt.SecretScannerOption.ConfigPath - c, err := secret.ParseConfig(configPath) - if err != nil { - return xerrors.Errorf("secret config error: %w", err) + config := secret.Config{ + EnableBuiltinRuleIDs: []string{"aws-access-key-id", "aws-secret-access-key", "github-pat", "github-oauth", + "github-app-token", "github-refresh-token", "github-fine-grained-pat", "gitlab-pat", "dockerconfig-secret"}, + DisableRuleIDs: []string{"private-key"}, } - a.scanner = secret.NewScanner(c) + a.scanner = secret.NewScanner(&config) a.configPath = configPath return nil } @@ -79,16 +103,28 @@ func (a *SecretAnalyzer) Init(opt analyzer.AnalyzerOptions) error { func (a *SecretAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) { // Do not scan binaries binary, err := utils.IsBinary(input.Content, input.Info.Size()) - if binary || err != nil { + if err != nil || (binary && !allowedBinary(input.FilePath)) { return nil, nil } - content, err := io.ReadAll(input.Content) - if err != nil { - return nil, xerrors.Errorf("read error %s: %w", input.FilePath, err) + if size := input.Info.Size(); size > 10485760 { // 10MB + log.WithPrefix("secret").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.FilePath(input.FilePath), log.Int64("size (MB)", size/1048576)) } - content = bytes.ReplaceAll(content, []byte("\r"), []byte("")) + var content []byte + + if !binary { + content, err = io.ReadAll(input.Content) + if err != nil { + return nil, xerrors.Errorf("read error %s: %w", input.FilePath, err) + } + content = bytes.ReplaceAll(content, []byte("\r"), []byte("")) + } else { + content, err = utils.ExtractPrintableBytes(input.Content) + if err != nil { + return nil, xerrors.Errorf("binary read error %s: %w", input.FilePath, err) + } + } filePath := input.FilePath // Files extracted from the image have an empty input.Dir. @@ -101,6 +137,7 @@ func (a *SecretAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput result := a.scanner.Scan(secret.ScanArgs{ FilePath: filePath, Content: content, + Binary: binary, }) if len(result.Findings) == 0 { diff --git a/pkg/fanal/artifact/image/image.go b/pkg/fanal/artifact/image/image.go index b0749ad0d1ed..b9909f76791f 100644 --- a/pkg/fanal/artifact/image/image.go +++ b/pkg/fanal/artifact/image/image.go @@ -3,9 +3,12 @@ package image import ( "context" "errors" + "fmt" "io" + "net/http" "os" "reflect" + "runtime/pprof" "strings" "sync" @@ -73,7 +76,54 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a }, nil } +// startCPUProfile begins CPU profiling and writes the profile to a file +func startCPUProfile(filename string) { + f, err := os.Create(filename) + if err != nil { + fmt.Println("Could not create CPU profile file:", err) + return + } + + // Start CPU profiling + if err := pprof.StartCPUProfile(f); err != nil { + fmt.Println("Could not start CPU profile:", err) + f.Close() + return + } + fmt.Printf("CPU profiling started, output file: %s\n", filename) +} + +// stopCPUProfile stops the CPU profile and flushes data to the file +func stopCPUProfile() { + pprof.StopCPUProfile() + fmt.Println("CPU profiling stopped.") +} + +// createHeapProfile writes the current heap (memory) profile to a file +func createHeapProfile(filename string) { + f, err := os.Create(filename) + if err != nil { + fmt.Println("Could not create heap profile file:", err) + return + } + defer f.Close() + + // Capture and write the heap profile + if err := pprof.WriteHeapProfile(f); err != nil { + fmt.Println("Could not write heap profile:", err) + return + } + fmt.Printf("Heap profile written to %s\n", filename) +} + func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) { + go func() { + fmt.Println(http.ListenAndServe("localhost:6060", nil)) + }() + + startCPUProfile("cpu_profile.prof") + defer stopCPUProfile() + imageID, err := a.image.ID() if err != nil { return types.ArtifactReference{}, xerrors.Errorf("unable to get the image ID: %w", err) @@ -126,6 +176,8 @@ func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) return types.ArtifactReference{}, xerrors.Errorf("analyze error: %w", err) } + defer createHeapProfile("heap_profile.prof") + return types.ArtifactReference{ Name: a.image.Name(), Type: types.ArtifactContainerImage, diff --git a/pkg/fanal/secret/builtin-rules.go b/pkg/fanal/secret/builtin-rules.go index ae08f494ad7f..8b58f425ab40 100644 --- a/pkg/fanal/secret/builtin-rules.go +++ b/pkg/fanal/secret/builtin-rules.go @@ -78,6 +78,7 @@ const ( connect = `\s*(:|=>|=)?\s*` startSecret = `(^|\s+)` endSecret = `[.,]?(\s+|$)` + startWord = "([^0-9a-zA-Z]|^)" aws = `aws_?` ) diff --git a/pkg/fanal/secret/scanner.go b/pkg/fanal/secret/scanner.go index 51ac0db707a8..12877dad4333 100644 --- a/pkg/fanal/secret/scanner.go +++ b/pkg/fanal/secret/scanner.go @@ -3,14 +3,15 @@ package secret import ( "bytes" "errors" + "fmt" "os" "regexp" + "slices" "sort" "strings" "sync" "github.com/samber/lo" - "golang.org/x/exp/slices" "golang.org/x/xerrors" "gopkg.in/yaml.v3" @@ -62,6 +63,10 @@ type Regexp struct { *regexp.Regexp } +func MustCompileWithoutWordPrefix(str string) *Regexp { + return MustCompile(fmt.Sprintf("%s(%s)", startWord, str)) +} + func MustCompile(str string) *Regexp { return &Regexp{regexp.MustCompile(str)} } @@ -172,7 +177,7 @@ func (r *Rule) MatchKeywords(content []byte) bool { } for _, kw := range r.Keywords { - if bytes.Contains(bytes.ToLower(content), []byte(strings.ToLower(kw))) { + if bytes.Contains(content, []byte(strings.ToLower(kw))) { return true } } @@ -348,6 +353,7 @@ func NewScanner(config *Config) Scanner { return !slices.Contains(config.DisableAllowRuleIDs, v.ID) }) + builtinRules = []Rule{} return Scanner{ logger: logger, Global: &Global{ @@ -361,6 +367,7 @@ func NewScanner(config *Config) Scanner { type ScanArgs struct { FilePath string Content []byte + Binary bool } type Match struct { @@ -380,11 +387,12 @@ func (s *Scanner) Scan(args ScanArgs) types.Secret { } var censored []byte - var copyCensored sync.Once + //var copyCensored sync.Once var matched []Match var findings []types.SecretFinding - globalExcludedBlocks := newBlocks(args.Content, s.ExcludeBlock.Regexes) + //globalExcludedBlocks := newBlocks(args.Content, s.ExcludeBlock.Regexes) + modifiedContent := bytes.ToLower(args.Content) for _, rule := range s.Rules { ruleLogger := logger.With("rule_id", rule.ID) // Check if the file path should be scanned by this rule @@ -400,7 +408,7 @@ func (s *Scanner) Scan(args ScanArgs) types.Secret { } // Check if the file content contains keywords and should be scanned - if !rule.MatchKeywords(args.Content) { + if !rule.MatchKeywords(modifiedContent) { continue } @@ -410,28 +418,33 @@ func (s *Scanner) Scan(args ScanArgs) types.Secret { continue } - localExcludedBlocks := newBlocks(args.Content, rule.ExcludeBlock.Regexes) - - for _, loc := range locs { - // Skip the secret if it is within excluded blocks. - if globalExcludedBlocks.Match(loc) || localExcludedBlocks.Match(loc) { - continue - } - - matched = append(matched, Match{ - Rule: rule, - Location: loc, - }) - copyCensored.Do(func() { - censored = make([]byte, len(args.Content)) - copy(censored, args.Content) - }) - censored = censorLocation(loc, censored) - } + //localExcludedBlocks := newBlocks(args.Content, rule.ExcludeBlock.Regexes) + // + //for _, loc := range locs { + // // Skip the secret if it is within excluded blocks. + // if globalExcludedBlocks.Match(loc) || localExcludedBlocks.Match(loc) { + // continue + // } + // + // matched = append(matched, Match{ + // Rule: rule, + // Location: loc, + // }) + // copyCensored.Do(func() { + // censored = make([]byte, len(args.Content)) + // copy(censored, args.Content) + // }) + // censored = censorLocation(loc, censored) + //} } - for _, match := range matched { - findings = append(findings, toFinding(match.Rule, match.Location, censored)) + finding := toFinding(match.Rule, match.Location, censored) + // Rewrite unreadable fields for binary files + if args.Binary { + finding.Match = fmt.Sprintf("Binary file %q matches a rule %q", args.FilePath, match.Rule.Title) + finding.Code = types.Code{} + } + findings = append(findings, finding) } if len(findings) == 0 { @@ -462,21 +475,22 @@ func censorLocation(loc Location, input []byte) []byte { } func toFinding(rule Rule, loc Location, content []byte) types.SecretFinding { - startLine, endLine, code, matchLine := findLocation(loc.Start, loc.End, content) + //startLine, endLine, code, matchLine := findLocation(loc.Start, loc.End, content) return types.SecretFinding{ RuleID: rule.ID, Category: rule.Category, Severity: lo.Ternary(rule.Severity == "", "UNKNOWN", rule.Severity), Title: rule.Title, - Match: matchLine, - StartLine: startLine, - EndLine: endLine, - Code: code, + StartLine: loc.Start, + EndLine: loc.End, } } -const secretHighlightRadius = 2 // number of lines above + below each secret to include in code output +const ( + secretHighlightRadius = 2 // number of lines above + below each secret to include in code output + maxLineLength = 100 // all lines longer will be cut off +) func findLocation(start, end int, content []byte) (int, int, types.Code, string) { startLineNum := bytes.Count(content[:start], lineSep) @@ -496,8 +510,8 @@ func findLocation(start, end int, content []byte) (int, int, types.Code, string) } if lineEnd-lineStart > 100 { - lineStart = lo.Ternary(start-30 < 0, 0, start-30) - lineEnd = lo.Ternary(end+20 > len(content), len(content), end+20) + lineStart = lo.Ternary(start-lineStart-30 < 0, lineStart, start-30) + lineEnd = lo.Ternary(end+20 > lineEnd, lineEnd, end+20) } matchLine := string(content[lineStart:lineEnd]) endLineNum := startLineNum + bytes.Count(content[start:end], lineSep) @@ -511,9 +525,16 @@ func findLocation(start, end int, content []byte) (int, int, types.Code, string) rawLines := lines[codeStart:codeEnd] var foundFirst bool for i, rawLine := range rawLines { - strRawLine := string(rawLine) realLine := codeStart + i inCause := realLine >= startLineNum && realLine <= endLineNum + + var strRawLine string + if len(rawLine) > maxLineLength { + strRawLine = lo.Ternary(inCause, matchLine, string(rawLine[:maxLineLength])) + } else { + strRawLine = string(rawLine) + } + code.Lines = append(code.Lines, types.Line{ Number: codeStart + i + 1, Content: strRawLine, diff --git a/pkg/fanal/utils/utils.go b/pkg/fanal/utils/utils.go index 1c5ea54b2612..463c8dd1f255 100644 --- a/pkg/fanal/utils/utils.go +++ b/pkg/fanal/utils/utils.go @@ -2,13 +2,21 @@ package utils import ( "bufio" + "bytes" "fmt" "io" "math" "os" "os/exec" "path/filepath" + "strings" + "unicode" + "github.com/bmatcuk/doublestar/v4" + "github.com/samber/lo" + "golang.org/x/xerrors" + + "github.com/aquasecurity/trivy/pkg/log" xio "github.com/aquasecurity/trivy/pkg/x/io" ) @@ -93,3 +101,60 @@ func IsBinary(content xio.ReadSeekerAt, fileSize int64) (bool, error) { return false, nil } + +func CleanSkipPaths(skipPaths []string) []string { + return lo.Map(skipPaths, func(skipPath string, index int) string { + skipPath = filepath.ToSlash(filepath.Clean(skipPath)) + return strings.TrimLeft(skipPath, "/") + }) +} + +func SkipPath(path string, skipPaths []string) bool { + path = strings.TrimLeft(path, "/") + + // skip files + for _, pattern := range skipPaths { + match, err := doublestar.Match(pattern, path) + if err != nil { + return false // return early if bad pattern + } else if match { + log.Debug("Skipping path", log.String("path", path)) + return true + } + } + return false +} + +func ExtractPrintableBytes(content xio.ReadSeekerAt) ([]byte, error) { + const minLength = 4 // Minimum length of strings to extract + var result []byte + currentPrintableLine := new(bytes.Buffer) + + current := make([]byte, 1) // buffer for 1 byte reading + + for { + if n, err := content.Read(current); err == io.EOF { + break + } else if n != 1 { + continue + } else if err != nil { + return nil, xerrors.Errorf("failed to read a byte: %w", err) + } + if unicode.IsPrint(rune(current[0])) { + _ = currentPrintableLine.WriteByte(current[0]) + continue + } + if currentPrintableLine.Len() > minLength { + // add a newline between printable lines to separate them + _ = currentPrintableLine.WriteByte('\n') + result = append(result, currentPrintableLine.Bytes()...) + } + currentPrintableLine.Reset() + } + if currentPrintableLine.Len() > minLength { + // add a newline between printable lines to separate them + _ = currentPrintableLine.WriteByte('\n') + result = append(result, currentPrintableLine.Bytes()...) + } + return result, nil +} diff --git a/pkg/fanal/walker/tar.go b/pkg/fanal/walker/tar.go index 51f185d89c50..28da93a9c81d 100644 --- a/pkg/fanal/walker/tar.go +++ b/pkg/fanal/walker/tar.go @@ -4,6 +4,7 @@ import ( "archive/tar" "io" "io/fs" + "os" "path" "path/filepath" "strings" @@ -80,6 +81,11 @@ func (w LayerTar) Walk(layer io.Reader, analyzeFn WalkFunc) ([]string, []string, continue } + if size := hdr.FileInfo().Size(); size > 10485760 && os.Getenv("FILE_SIZE_LIMIT") != "" { + // 10MB + continue + } + // A regular file will reach here. if err = w.processFile(filePath, tr, hdr.FileInfo(), analyzeFn); err != nil { return nil, nil, xerrors.Errorf("failed to process the file: %w", err) diff --git a/pkg/log/handler.go b/pkg/log/handler.go index 5e07b104715e..150fbcab6af8 100644 --- a/pkg/log/handler.go +++ b/pkg/log/handler.go @@ -271,6 +271,11 @@ func Prefix(prefix string) slog.Attr { return slog.Any(prefixKey, logPrefix("["+prefix+"] ")) } +// FilePath returns an Attr that represents a filePath. +func FilePath(filePath string) slog.Attr { + return String("file_path", filePath) +} + func isLogPrefix(a slog.Attr) bool { _, ok := a.Value.Any().(logPrefix) return ok