use yarascanner

deepfence · Jul 9, 2024 · 3addd90 · 3addd90
1 parent 7bc1268
commit 3addd90
Show file tree

Hide file tree

Showing 6 changed files with 169 additions and 217 deletions.
diff --git a/core/config.go b/core/config.go
@@ -1,14 +1,11 @@
 package core
 
 import (
-	"fmt"
 	"os"
-	"path"
 	"path/filepath"
 	"regexp"
 
 	"github.com/deepfence/match-scanner/pkg/config"
-	"gopkg.in/yaml.v3"
 )
 
 type Config struct {
@@ -62,81 +59,6 @@ func mergeStringSlices(old, new []string) []string {
 	return old
 }
 
-func ParseConfig(options *Options) (*Config, error) {
-	configFileDirs := options.RulesPath.Values()
-
-	if len(configFileDirs) > 0 {
-		if *options.MergeConfigs {
-			// merge them together onto default config in order of specification
-			config, err := getDefaultConfig()
-			if err != nil {
-				return nil, err
-			}
-
-			var subConfig *Config
-			for _, dir := range configFileDirs {
-				subConfig, err = loadConfigFile(dir)
-				if err != nil {
-					return nil, err
-				}
-				config.Merge(subConfig)
-			}
-
-			return config, nil
-		} else {
-			if len(configFileDirs) > 1 {
-				return nil, fmt.Errorf("error: Multiple config paths specified, but --merge-configs is not specified")
-			}
-
-			return loadConfigFile(configFileDirs[0])
-		}
-
-	}
-
-	return getDefaultConfig()
-}
-
-// Trying to first find the configuration next to executable
-// Helps e.g. with Drone where workdir is different than shhgit dir
-func getDefaultConfig() (*Config, error) {
-	ex, err := os.Executable()
-	if err != nil {
-		return nil, fmt.Errorf("os.Executable: %w", err)
-	}
-	dir := filepath.Dir(ex)
-	config, err := loadConfigFile(dir)
-	if err != nil {
-		dir, _ = os.Getwd()
-		return loadConfigFile(dir)
-	}
-	return config, nil
-}
-
-func loadConfigFile(configPath string) (*Config, error) {
-	fstat, err := os.Stat(configPath)
-	if err != nil {
-		return nil, err
-	}
-
-	var data []byte
-	if fstat.IsDir() {
-		data, err = os.ReadFile(path.Join(configPath, "config.yaml"))
-	} else {
-		data, err = os.ReadFile(configPath)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	config := &Config{}
-	err = yaml.Unmarshal(data, config)
-	if err != nil {
-		return nil, err
-	}
-
-	return config, nil
-}
-
 func loadExtractorConfigFile(options *Options) (config.Config, error) {
 	configPath := *options.ConfigPath
 	fstat, err := os.Stat(configPath)

diff --git a/core/options.go b/core/options.go
@@ -14,31 +14,34 @@ const (
 )
 
 type Options struct {
-	Threads           *int
-	Debug             *bool
-	MaximumFileSize   *uint
-	TempDirectory     *string
-	Local             *string
-	HostMountPath     *string
-	ConfigPath        *string
-	RulesPath         *repeatableStringValue
-	MergeConfigs      *bool
-	ImageName         *string
-	MultipleMatch     *bool
-	MaxMultiMatch     *uint
-	MaxSecrets        *uint
-	ContainerID       *string
-	ContainerNS       *string
-	WorkersPerScan    *int
-	InactiveThreshold *int
-	OutFormat         *string
-	ConsoleURL        *string
-	ConsolePort       *int
-	DeepfenceKey      *string
-	FailOnCount       *int
-	FailOnHighCount   *int
-	FailOnMediumCount *int
-	FailOnLowCount    *int
+	Threads              *int
+	Debug                *bool
+	MaximumFileSize      *uint
+	TempDirectory        *string
+	Local                *string
+	HostMountPath        *string
+	ConfigPath           *string
+	RulesPath            *string
+	RulesListingURL      *string
+	FailOnCompileWarning *bool
+	EnableUpdater        *bool
+	MergeConfigs         *bool
+	ImageName            *string
+	MultipleMatch        *bool
+	MaxMultiMatch        *uint
+	MaxSecrets           *uint
+	ContainerID          *string
+	ContainerNS          *string
+	WorkersPerScan       *int
+	InactiveThreshold    *int
+	OutFormat            *string
+	ConsoleURL           *string
+	ConsolePort          *int
+	DeepfenceKey         *string
+	FailOnCount          *int
+	FailOnHighCount      *int
+	FailOnMediumCount    *int
+	FailOnLowCount       *int
 }
 
 type repeatableStringValue struct {
@@ -60,33 +63,35 @@ func (v *repeatableStringValue) Values() []string {
 
 func ParseOptions() (*Options, error) {
 	options := &Options{
-		Threads:           flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"),
-		Debug:             flag.Bool("debug", false, "enable debug logs"),
-		MaximumFileSize:   flag.Uint("maximum-file-size", 256, "Maximum file size to process in KB"),
-		TempDirectory:     flag.String("temp-directory", os.TempDir(), "Directory to process and store repositories/matches"),
-		Local:             flag.String("local", "", "Specify local directory (absolute path) which to scan. Scans only given directory recursively."),
-		HostMountPath:     flag.String("host-mount-path", "", "If scanning the host, specify the host mount path for path exclusions to work correctly."),
-		ConfigPath:        flag.String("config-path", "", "yaml config path"),
-		RulesPath:         &repeatableStringValue{},
-		MergeConfigs:      flag.Bool("merge-configs", false, "Merge config files specified by --config-path into the default config"),
-		ImageName:         flag.String("image-name", "", "Name of the image along with tag to scan for secrets"),
-		MultipleMatch:     flag.Bool("multi-match", false, "Output multiple matches of same pattern in one file. By default, only one match of a pattern is output for a file for better performance"),
-		MaxMultiMatch:     flag.Uint("max-multi-match", 3, "Maximum number of matches of same pattern in one file. This is used only when multi-match option is enabled."),
-		MaxSecrets:        flag.Uint("max-secrets", 1000, "Maximum number of secrets to find in one container image or file system."),
-		ContainerID:       flag.String("container-id", "", "Id of existing container ID"),
-		ContainerNS:       flag.String("container-ns", "", "Namespace of existing container to scan, empty for docker runtime"),
-		WorkersPerScan:    flag.Int("workers-per-scan", 1, "Number of concurrent workers per scan"),
-		InactiveThreshold: flag.Int("inactive-threshold", 600, "Threshold for Inactive scan in seconds"),
-		OutFormat:         flag.String("output", TableOutput, "Output format: json or table"),
-		ConsoleURL:        flag.String("console-url", "", "Deepfence Management Console URL"),
-		ConsolePort:       flag.Int("console-port", 443, "Deepfence Management Console Port"),
-		DeepfenceKey:      flag.String("deepfence-key", "", "Deepfence key for auth"),
-		FailOnCount:       flag.Int("fail-on-count", -1, "Exit with status 1 if number of secrets found is >= this value (Default: -1)"),
-		FailOnHighCount:   flag.Int("fail-on-high-count", -1, "Exit with status 1 if number of high secrets found is >= this value (Default: -1)"),
-		FailOnMediumCount: flag.Int("fail-on-medium-count", -1, "Exit with status 1 if number of medium secrets found is >= this value (Default: -1)"),
-		FailOnLowCount:    flag.Int("fail-on-low-count", -1, "Exit with status 1 if number of low secrets found is >= this value (Default: -1)"),
+		Threads:              flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"),
+		Debug:                flag.Bool("debug", false, "enable debug logs"),
+		MaximumFileSize:      flag.Uint("maximum-file-size", 256, "Maximum file size to process in KB"),
+		TempDirectory:        flag.String("temp-directory", os.TempDir(), "Directory to process and store repositories/matches"),
+		Local:                flag.String("local", "", "Specify local directory (absolute path) which to scan. Scans only given directory recursively."),
+		HostMountPath:        flag.String("host-mount-path", "", "If scanning the host, specify the host mount path for path exclusions to work correctly."),
+		ConfigPath:           flag.String("config-path", "", "yaml config path"),
+		RulesPath:            flag.String("rules-path", "", "yara rules path"),
+		RulesListingURL:      flag.String("rules-listing-url", "", "yara rules listing url"),
+		FailOnCompileWarning: flag.Bool("fail-warning", false, "fail if compilation warning"),
+		EnableUpdater:        flag.Bool("enable-updated", false, "Enable rule updater"),
+		MergeConfigs:         flag.Bool("merge-configs", false, "Merge config files specified by --config-path into the default config"),
+		ImageName:            flag.String("image-name", "", "Name of the image along with tag to scan for secrets"),
+		MultipleMatch:        flag.Bool("multi-match", false, "Output multiple matches of same pattern in one file. By default, only one match of a pattern is output for a file for better performance"),
+		MaxMultiMatch:        flag.Uint("max-multi-match", 3, "Maximum number of matches of same pattern in one file. This is used only when multi-match option is enabled."),
+		MaxSecrets:           flag.Uint("max-secrets", 1000, "Maximum number of secrets to find in one container image or file system."),
+		ContainerID:          flag.String("container-id", "", "Id of existing container ID"),
+		ContainerNS:          flag.String("container-ns", "", "Namespace of existing container to scan, empty for docker runtime"),
+		WorkersPerScan:       flag.Int("workers-per-scan", 1, "Number of concurrent workers per scan"),
+		InactiveThreshold:    flag.Int("inactive-threshold", 600, "Threshold for Inactive scan in seconds"),
+		OutFormat:            flag.String("output", TableOutput, "Output format: json or table"),
+		ConsoleURL:           flag.String("console-url", "", "Deepfence Management Console URL"),
+		ConsolePort:          flag.Int("console-port", 443, "Deepfence Management Console Port"),
+		DeepfenceKey:         flag.String("deepfence-key", "", "Deepfence key for auth"),
+		FailOnCount:          flag.Int("fail-on-count", -1, "Exit with status 1 if number of secrets found is >= this value (Default: -1)"),
+		FailOnHighCount:      flag.Int("fail-on-high-count", -1, "Exit with status 1 if number of high secrets found is >= this value (Default: -1)"),
+		FailOnMediumCount:    flag.Int("fail-on-medium-count", -1, "Exit with status 1 if number of medium secrets found is >= this value (Default: -1)"),
+		FailOnLowCount:       flag.Int("fail-on-low-count", -1, "Exit with status 1 if number of low secrets found is >= this value (Default: -1)"),
 	}
-	flag.Var(options.RulesPath, "rules-path", "Searches for rules.yaml from given directory. If not set, tries to find it from SecretScanner binary's and current directory.  Can be specified multiple times.")
 	flag.Parse()
 	return options, nil
 }
diff --git a/core/session.go b/core/session.go
@@ -49,11 +49,6 @@ func GetSession() *Session {
 			os.Exit(1)
 		}
 
-		if session.Config, err = ParseConfig(session.Options); err != nil {
-			log.Error(err)
-			os.Exit(1)
-		}
-
 		if session.ExtractorConfig, err = loadExtractorConfigFile(session.Options); err != nil {
 			log.Error(err)
 			os.Exit(1)

diff --git a/go.mod b/go.mod
@@ -4,50 +4,54 @@ go 1.21.0
 
 replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc
 
+replace github.com/deepfence/YaraHunter => ../YaraHunter
+
 require (
+	github.com/deepfence/YaraHunter v0.0.0-00010101000000-000000000000
 	github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000
-	github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20231201173641-092afefd00a2
-	github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20231201173641-092afefd00a2
-	github.com/deepfence/match-scanner v0.0.0-20240627065846-d2405fb72cfb
+	github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9
+	github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9
+	github.com/deepfence/match-scanner v0.0.0-20240701181002-ece6f13f296f
 	github.com/fatih/color v1.16.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/sirupsen/logrus v1.9.3
-	google.golang.org/grpc v1.63.2
-	gopkg.in/yaml.v3 v3.0.1
+	google.golang.org/grpc v1.64.0
 )
 
 require (
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20231105174938-2b5cbb29f3e2 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
-	github.com/Microsoft/hcsshim v0.12.3 // indirect
-	github.com/containerd/cgroups/v3 v3.0.2 // indirect
-	github.com/containerd/containerd v1.7.16 // indirect
+	github.com/Microsoft/hcsshim v0.12.4 // indirect
+	github.com/containerd/cgroups/v3 v3.0.3 // indirect
+	github.com/containerd/containerd v1.7.18 // indirect
 	github.com/containerd/continuity v0.4.3 // indirect
 	github.com/containerd/errdefs v0.1.0 // indirect
 	github.com/containerd/fifo v1.1.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
-	github.com/containerd/ttrpc v1.2.3 // indirect
+	github.com/containerd/ttrpc v1.2.5 // indirect
 	github.com/containerd/typeurl/v2 v2.1.1 // indirect
 	github.com/deepfence/vessel v0.12.3 // indirect
 	github.com/distribution/reference v0.6.0 // indirect
-	github.com/docker/docker v26.1.1+incompatible // indirect
+	github.com/docker/docker v27.0.2+incompatible // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
-	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.4 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.5 // indirect
-	github.com/klauspost/compress v1.17.8 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
+	github.com/hillu/go-yara/v4 v4.3.2 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/locker v1.0.1 // indirect
 	github.com/moby/sys/mountinfo v0.7.1 // indirect
@@ -60,16 +64,18 @@ require (
 	github.com/opencontainers/runtime-spec v1.2.0 // indirect
 	github.com/opencontainers/selinux v1.11.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect
-	go.opentelemetry.io/otel v1.26.0 // indirect
-	go.opentelemetry.io/otel/metric v1.26.0 // indirect
-	go.opentelemetry.io/otel/trace v1.26.0 // indirect
-	golang.org/x/net v0.24.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.52.0 // indirect
+	go.opentelemetry.io/otel v1.27.0 // indirect
+	go.opentelemetry.io/otel/metric v1.27.0 // indirect
+	go.opentelemetry.io/otel/trace v1.27.0 // indirect
+	golang.org/x/net v0.26.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.20.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
-	google.golang.org/genproto v0.0.0-20240429193739-8cf5692501f6 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect
-	google.golang.org/protobuf v1.34.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )