Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(flag): replace '--slow' with '--parallel' #5572

Merged
merged 9 commits into from
Nov 15, 2023
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ trivy filesystem [flags] PATH
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--policy-namespaces strings Rego namespaces
Expand All @@ -76,7 +77,6 @@ trivy filesystem [flags] PATH
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files
Expand Down
2 changes: 1 addition & 1 deletion docs/docs/references/configuration/cli/trivy_image.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ trivy image [flags] IMAGE_NAME
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--platform string set platform in the form os/arch if image is multi-platform capable
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
Expand All @@ -97,7 +98,6 @@ trivy image [flags] IMAGE_NAME
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ trivy kubernetes [flags] { cluster | all | specific resources like kubectl. eg:
--node-collector-namespace string specify the namespace in which the node-collector job should be deployed (default "trivy-temp")
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number (between 1-20) of goroutines enabled for parallel scanning (default 5)
--parallel int number of goroutines enabled for parallel scanning (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--policy-namespaces strings Rego namespaces
Expand All @@ -87,7 +87,6 @@ trivy kubernetes [flags] { cluster | all | specific resources like kubectl. eg:
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ trivy repository [flags] (REPO_PATH | REPO_URL)
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--policy-namespaces strings Rego namespaces
Expand All @@ -75,7 +76,6 @@ trivy repository [flags] (REPO_PATH | REPO_URL)
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
--tag string pass the tag name to be scanned
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
Expand Down
2 changes: 1 addition & 1 deletion docs/docs/references/configuration/cli/trivy_rootfs.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ trivy rootfs [flags] ROOTDIR
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning (default 5)
--password strings password. Comma-separated passwords allowed. TRIVY_PASSWORD should be used for security reasons.
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--policy-namespaces strings Rego namespaces
Expand All @@ -77,7 +78,6 @@ trivy rootfs [flags] ROOTDIR
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--skip-policy-update skip fetching rego policy updates
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files
Expand Down
2 changes: 1 addition & 1 deletion docs/docs/references/configuration/cli/trivy_sbom.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ trivy sbom [flags] SBOM_PATH
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning (default 5)
DmitriyLewen marked this conversation as resolved.
Show resolved Hide resolved
--redis-ca string redis ca file location, if using redis as cache backend
--redis-cert string redis certificate file location, if using redis as cache backend
--redis-key string redis key file location, if using redis as cache backend
Expand All @@ -55,7 +56,6 @@ trivy sbom [flags] SBOM_PATH
--skip-dirs strings specify the directories or glob patterns to skip
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--token string for authentication in client/server mode
--token-header string specify a header name for token in client/server mode (default "Trivy-Token")
Expand Down
2 changes: 1 addition & 1 deletion docs/docs/references/configuration/cli/trivy_vm.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ trivy vm [flags] VM_IMAGE
--no-progress suppress progress bar
--offline-scan do not issue API requests to identify dependencies
-o, --output string output file name
--parallel int number of goroutines enabled for parallel scanning (default 5)
--policy-bundle-repository string OCI registry URL to retrieve policy bundle from (default "ghcr.io/aquasecurity/trivy-policies:0")
--redis-ca string redis ca file location, if using redis as cache backend
--redis-cert string redis certificate file location, if using redis as cache backend
Expand All @@ -68,7 +69,6 @@ trivy vm [flags] VM_IMAGE
--skip-dirs strings specify the directories or glob patterns to skip
--skip-files strings specify the files or glob patterns to skip
--skip-java-db-update skip updating Java index database
--slow scan over time with lower CPU and memory utilization
-t, --template string output template
--tf-exclude-downloaded-modules exclude misconfigurations for downloaded terraform modules
--tf-vars strings specify paths to override the Terraform tfvars files
Expand Down
1 change: 0 additions & 1 deletion pkg/commands/artifact/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -637,7 +637,6 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
SBOMSources: opts.SBOMSources,
RekorURL: opts.RekorURL,
//Platform: opts.Platform,
Slow: opts.Slow,
DmitriyLewen marked this conversation as resolved.
Show resolved Hide resolved
AWSRegion: opts.Region,
AWSEndpoint: opts.Endpoint,
FileChecksum: fileChecksum,
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/analyzer/analyzer.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ var (
// AnalyzerOptions is used to initialize analyzers
type AnalyzerOptions struct {
Group Group
Slow bool
Parallel int
FilePatterns []string
DisabledAnalyzers []Type
MisconfScannerOption misconf.ScannerOption
Expand Down
6 changes: 3 additions & 3 deletions pkg/fanal/analyzer/language/java/jar/jar.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,12 @@ var requiredExtensions = []string{

// javaLibraryAnalyzer analyzes jar/war/ear/par files
type javaLibraryAnalyzer struct {
slow bool
parallel int
}

func newJavaLibraryAnalyzer(options analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) {
return &javaLibraryAnalyzer{
slow: options.Slow,
parallel: options.Parallel,
}, nil
}

Expand Down Expand Up @@ -70,7 +70,7 @@ func (a *javaLibraryAnalyzer) PostAnalyze(ctx context.Context, input analyzer.Po
return nil
}

if err = parallel.WalkDir(ctx, input.FS, ".", a.slow, onFile, onResult); err != nil {
if err = parallel.WalkDir(ctx, input.FS, ".", a.parallel, onFile, onResult); err != nil {
return nil, xerrors.Errorf("walk dir error: %w", err)
}

Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/analyzer/language/java/jar/jar_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ func Test_javaLibraryAnalyzer_Analyze(t *testing.T) {
// init java-trivy-db with skip update
javadb.Init("testdata", defaultJavaDBRepository, true, false, types.RegistryOptions{Insecure: false})

a := javaLibraryAnalyzer{slow: true}
a := javaLibraryAnalyzer{}
ctx := context.Background()

mfs := mapfs.New()
Expand Down
23 changes: 22 additions & 1 deletion pkg/fanal/artifact/artifact.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ type Option struct {
AppDirs []string
SBOMSources []string
RekorURL string
Slow bool // Lower CPU and memory
Parallel int
AWSRegion string
AWSEndpoint string
FileChecksum bool // For SPDX
Expand All @@ -50,6 +50,27 @@ type WalkOption struct {
ErrorCallback walker.ErrorCallback
}

func (o *Option) AnalyzerOptions() analyzer.AnalyzerOptions {
return analyzer.AnalyzerOptions{
Group: o.AnalyzerGroup,
FilePatterns: o.FilePatterns,
Parallel: o.Parallel,
DisabledAnalyzers: o.DisabledAnalyzers,
MisconfScannerOption: o.MisconfScannerOption,
SecretScannerOption: o.SecretScannerOption,
LicenseScannerOption: o.LicenseScannerOption,
}
}

func (o *Option) ConfigAnalyzerOptions() analyzer.ConfigAnalyzerOptions {
return analyzer.ConfigAnalyzerOptions{
FilePatterns: o.FilePatterns,
DisabledAnalyzers: o.DisabledAnalyzers,
MisconfScannerOption: o.MisconfScannerOption,
SecretScannerOption: o.SecretScannerOption,
}
}

func (o *Option) Sort() {
sort.Slice(o.DisabledAnalyzers, func(i, j int) bool {
return o.DisabledAnalyzers[i] < o.DisabledAnalyzers[j]
Expand Down
24 changes: 5 additions & 19 deletions pkg/fanal/artifact/image/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,33 +49,20 @@ func NewArtifact(img types.Image, c cache.ArtifactCache, opt artifact.Option) (a
return nil, xerrors.Errorf("handler init error: %w", err)
}

a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
Group: opt.AnalyzerGroup,
Slow: opt.Slow,
FilePatterns: opt.FilePatterns,
DisabledAnalyzers: opt.DisabledAnalyzers,
MisconfScannerOption: opt.MisconfScannerOption,
SecretScannerOption: opt.SecretScannerOption,
LicenseScannerOption: opt.LicenseScannerOption,
})
a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions())
if err != nil {
return nil, xerrors.Errorf("analyzer group error: %w", err)
}

ca, err := analyzer.NewConfigAnalyzerGroup(analyzer.ConfigAnalyzerOptions{
FilePatterns: opt.FilePatterns,
DisabledAnalyzers: opt.DisabledAnalyzers,
MisconfScannerOption: opt.MisconfScannerOption,
SecretScannerOption: opt.SecretScannerOption,
})
ca, err := analyzer.NewConfigAnalyzerGroup(opt.ConfigAnalyzerOptions())
if err != nil {
return nil, xerrors.Errorf("config analyzer group error: %w", err)
}

return Artifact{
image: img,
cache: c,
walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs, opt.Slow),
walker: walker.NewLayerTar(opt.SkipFiles, opt.SkipDirs),
analyzer: a,
configAnalyzer: ca,
handlerManager: handlerManager,
Expand Down Expand Up @@ -215,8 +202,7 @@ func (a Artifact) inspect(ctx context.Context, missingImage string, layerKeys, b
layerKeyMap map[string]LayerInfo, configFile *v1.ConfigFile) error {

var osFound types.OS
workers := lo.Ternary(a.artifactOption.Slow, 1, 5)
p := parallel.NewPipeline(workers, false, layerKeys, func(ctx context.Context, layerKey string) (any, error) {
p := parallel.NewPipeline(a.artifactOption.Parallel, false, layerKeys, func(ctx context.Context, layerKey string) (any, error) {
layer := layerKeyMap[layerKey]

// If it is a base layer, secret scanning should not be performed.
Expand Down Expand Up @@ -268,7 +254,7 @@ func (a Artifact) inspectLayer(ctx context.Context, layerInfo LayerInfo, disable
FileChecksum: a.artifactOption.FileChecksum,
}
result := analyzer.NewAnalysisResult()
limit := semaphore.New(a.artifactOption.Slow)
limit := semaphore.New(a.artifactOption.Parallel)

// Prepare filesystem for post analysis
composite, err := a.analyzer.PostAnalyzerFS()
Expand Down
5 changes: 1 addition & 4 deletions pkg/fanal/artifact/image/image_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2035,11 +2035,8 @@ func TestArtifact_Inspect(t *testing.T) {
wantErr: "put layer failed",
},
{
name: "sad path, PutBlob returns an error with multiple layers and Slow enabled",
name: "sad path, PutBlob returns an error with multiple layers",
imagePath: "../../test/testdata/vuln-image.tar.gz",
artifactOpt: artifact.Option{
Slow: true,
},
missingBlobsExpectation: cache.ArtifactCacheMissingBlobsExpectation{
Args: cache.ArtifactCacheMissingBlobsArgs{
ArtifactID: "sha256:33f9415ed2cd5a9cef5d5144333619745b9ec0f851f0684dd45fa79c6b26a650",
Expand Down
14 changes: 3 additions & 11 deletions pkg/fanal/artifact/local/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a
return nil, xerrors.Errorf("handler initialize error: %w", err)
}

a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
Group: opt.AnalyzerGroup,
Slow: opt.Slow,
FilePatterns: opt.FilePatterns,
DisabledAnalyzers: opt.DisabledAnalyzers,
MisconfScannerOption: opt.MisconfScannerOption,
SecretScannerOption: opt.SecretScannerOption,
LicenseScannerOption: opt.LicenseScannerOption,
})
a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions())
if err != nil {
return nil, xerrors.Errorf("analyzer group error: %w", err)
}
Expand All @@ -56,7 +48,7 @@ func NewArtifact(rootPath string, c cache.ArtifactCache, opt artifact.Option) (a
rootPath: filepath.ToSlash(filepath.Clean(rootPath)),
cache: c,
walker: walker.NewFS(buildPathsToSkip(rootPath, opt.SkipFiles), buildPathsToSkip(rootPath, opt.SkipDirs),
opt.Slow, opt.WalkOption.ErrorCallback),
opt.Parallel, opt.WalkOption.ErrorCallback),
analyzer: a,
handlerManager: handlerManager,

Expand Down Expand Up @@ -122,7 +114,7 @@ func buildPathsToSkip(base string, paths []string) []string {
func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) {
var wg sync.WaitGroup
result := analyzer.NewAnalysisResult()
limit := semaphore.New(a.artifactOption.Slow)
limit := semaphore.New(a.artifactOption.Parallel)
opts := analyzer.AnalysisOptions{
Offline: a.artifactOption.Offline,
FileChecksum: a.artifactOption.FileChecksum,
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/artifact/local/fs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ func TestArtifact_Inspect(t *testing.T) {
fields: fields{
dir: "./testdata/unknown",
},
wantErr: "walk error",
wantErr: "walk dir error",
},
{
name: "happy path with single file",
Expand Down
13 changes: 3 additions & 10 deletions pkg/fanal/artifact/vm/vm.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ type Storage struct {

func (a *Storage) Analyze(ctx context.Context, r *io.SectionReader) (types.BlobInfo, error) {
var wg sync.WaitGroup
limit := semaphore.New(a.artifactOption.Slow)
limit := semaphore.New(a.artifactOption.Parallel)
result := analyzer.NewAnalysisResult()

opts := analyzer.AnalysisOptions{
Expand Down Expand Up @@ -119,14 +119,7 @@ func NewArtifact(target string, c cache.ArtifactCache, opt artifact.Option) (art
if err != nil {
return nil, xerrors.Errorf("handler init error: %w", err)
}
a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
Group: opt.AnalyzerGroup,
FilePatterns: opt.FilePatterns,
DisabledAnalyzers: opt.DisabledAnalyzers,
MisconfScannerOption: opt.MisconfScannerOption,
SecretScannerOption: opt.SecretScannerOption,
LicenseScannerOption: opt.LicenseScannerOption,
})
a, err := analyzer.NewAnalyzerGroup(opt.AnalyzerOptions())
if err != nil {
return nil, xerrors.Errorf("analyzer group error: %w", err)
}
Expand All @@ -135,7 +128,7 @@ func NewArtifact(target string, c cache.ArtifactCache, opt artifact.Option) (art
cache: c,
analyzer: a,
handlerManager: handlerManager,
walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs, opt.Slow),
walker: walker.NewVM(opt.SkipFiles, opt.SkipDirs),
artifactOption: opt,
}

Expand Down
8 changes: 5 additions & 3 deletions pkg/fanal/walker/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,11 @@ type ErrorCallback func(pathname string, err error) error

type FS struct {
walker
parallel int
errCallback ErrorCallback
}

func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) FS {
func NewFS(skipFiles, skipDirs []string, parallel int, errCallback ErrorCallback) FS {
if errCallback == nil {
errCallback = func(pathname string, err error) error {
// ignore permission errors
Expand All @@ -32,7 +33,8 @@ func NewFS(skipFiles, skipDirs []string, slow bool, errCallback ErrorCallback) F
}

return FS{
walker: newWalker(skipFiles, skipDirs, slow),
walker: newWalker(skipFiles, skipDirs),
parallel: parallel,
errCallback: errCallback,
}
}
Expand Down Expand Up @@ -69,7 +71,7 @@ func (w FS) Walk(root string, fn WalkFunc) error {
return nil
}

if w.slow {
if w.parallel <= 1 {
// In series: fast, with higher CPU/memory
return w.walkSlow(root, walkFn)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/fanal/walker/fs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func TestDir_Walk(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, true, tt.fields.errCallback)
w := walker.NewFS(tt.fields.skipFiles, tt.fields.skipDirs, 1, tt.fields.errCallback)

err := w.Walk(tt.rootDir, tt.analyzeFn)
if tt.wantErr != "" {
Expand Down
Loading