feat(secret): enhance secret scanning for python binary files (#7223)

Signed-off-by: knqyf263 <[email protected]>
Co-authored-by: knqyf263 <[email protected]>

docs/docs/scanner/secret.md

@@ -3,7 +3,9 @@
 Trivy scans any container image, filesystem and git repository to detect exposed secrets like passwords, api keys, and tokens.
 Secret scanning is enabled by default.
 
-Trivy will scan every plaintext file, according to builtin rules or configuration. There are plenty of builtin rules:
+Trivy will scan every plaintext file, according to builtin rules or configuration. Also, Trivy can detect secrets in compiled Python files (`.pyc`).
+
+There are plenty of builtin rules:
 
 - AWS access key
 - GCP service account

pkg/fanal/analyzer/secret/secret.go

@@ -54,6 +54,9 @@ var (
 		".gz",
 		".gzip",
 		".tar",
+	}
+
+	allowedBinaries = []string{
 		".pyc",
 	}
 )
@@ -63,6 +66,10 @@ func init() {
 	analyzer.RegisterAnalyzer(NewSecretAnalyzer(secret.Scanner{}, ""))
 }
 
+func allowedBinary(filename string) bool {
+	return slices.Contains(allowedBinaries, filepath.Ext(filename))
+}
+
 // SecretAnalyzer is an analyzer for secrets
 type SecretAnalyzer struct {
 	scanner    secret.Scanner
@@ -96,20 +103,28 @@ func (a *SecretAnalyzer) Init(opt analyzer.AnalyzerOptions) error {
 func (a *SecretAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) {
 	// Do not scan binaries
 	binary, err := utils.IsBinary(input.Content, input.Info.Size())
-	if binary || err != nil {
+	if err != nil || (binary && !allowedBinary(input.FilePath)) {
 		return nil, nil
 	}
 
 	if size := input.Info.Size(); size > 10485760 { // 10MB
 		log.WithPrefix("secret").Warn("The size of the scanned file is too large. It is recommended to use `--skip-files` for this file to avoid high memory consumption.", log.FilePath(input.FilePath), log.Int64("size (MB)", size/1048576))
 	}
 
-	content, err := io.ReadAll(input.Content)
-	if err != nil {
-		return nil, xerrors.Errorf("read error %s: %w", input.FilePath, err)
-	}
+	var content []byte
 
-	content = bytes.ReplaceAll(content, []byte("\r"), []byte(""))
+	if !binary {
+		content, err = io.ReadAll(input.Content)
+		if err != nil {
+			return nil, xerrors.Errorf("read error %s: %w", input.FilePath, err)
+		}
+		content = bytes.ReplaceAll(content, []byte("\r"), []byte(""))
+	} else {
+		content, err = utils.ExtractPrintableBytes(input.Content)
+		if err != nil {
+			return nil, xerrors.Errorf("binary read error %s: %w", input.FilePath, err)
+		}
+	}
 
 	filePath := input.FilePath
 	// Files extracted from the image have an empty input.Dir.
@@ -122,6 +137,7 @@ func (a *SecretAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput
 	result := a.scanner.Scan(secret.ScanArgs{
 		FilePath: filePath,
 		Content:  content,
+		Binary:   binary,
 	})
 
 	if len(result.Findings) == 0 {

pkg/fanal/analyzer/secret/secret_test.go

@@ -95,6 +95,16 @@ func TestSecretAnalyzer(t *testing.T) {
 			},
 		},
 	}
+	wantFindingGH_PAT := types.SecretFinding{
+		RuleID:    "github-fine-grained-pat",
+		Category:  "GitHub",
+		Title:     "GitHub Fine-grained personal access tokens",
+		Severity:  "CRITICAL",
+		StartLine: 1,
+		EndLine:   1,
+		Match:     "Binary file \"/testdata/secret.cpython-310.pyc\" matches a rule \"GitHub Fine-grained personal access tokens\"",
+	}
+
 	tests := []struct {
 		name       string
 		configPath string
@@ -153,6 +163,21 @@ func TestSecretAnalyzer(t *testing.T) {
 			filePath:   "testdata/binaryfile",
 			want:       nil,
 		},
+		{
+			name:       "python binary file",
+			configPath: "testdata/skip-tests-config.yaml",
+			filePath:   "testdata/secret.cpython-310.pyc",
+			want: &analyzer.AnalysisResult{
+				Secrets: []types.Secret{
+					{
+						FilePath: "/testdata/secret.cpython-310.pyc",
+						Findings: []types.SecretFinding{
+							wantFindingGH_PAT,
+						},
+					},
+				},
+			},
+		},
 	}
 
 	for _, tt := range tests {

pkg/fanal/secret/scanner.go

@@ -366,6 +366,7 @@ func NewScanner(config *Config) Scanner {
 type ScanArgs struct {
 	FilePath string
 	Content  []byte
+	Binary   bool
 }
 
 type Match struct {
@@ -434,9 +435,14 @@ func (s *Scanner) Scan(args ScanArgs) types.Secret {
 			censored = censorLocation(loc, censored)
 		}
 	}
-
 	for _, match := range matched {
-		findings = append(findings, toFinding(match.Rule, match.Location, censored))
+		finding := toFinding(match.Rule, match.Location, censored)
+		// Rewrite unreadable fields for binary files
+		if args.Binary {
+			finding.Match = fmt.Sprintf("Binary file %q matches a rule %q", args.FilePath, match.Rule.Title)
+			finding.Code = types.Code{}
+		}
+		findings = append(findings, finding)
 	}
 
 	if len(findings) == 0 {

pkg/fanal/utils/utils.go

@@ -2,12 +2,16 @@ package utils
 
 import (
 	"bufio"
+	"bytes"
 	"fmt"
 	"io"
 	"math"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"unicode"
+
+	"golang.org/x/xerrors"
 
 	xio "github.com/aquasecurity/trivy/pkg/x/io"
 )
@@ -93,3 +97,37 @@ func IsBinary(content xio.ReadSeekerAt, fileSize int64) (bool, error) {
 
 	return false, nil
 }
+
+func ExtractPrintableBytes(content xio.ReadSeekerAt) ([]byte, error) {
+	const minLength = 4 // Minimum length of strings to extract
+	var result []byte
+	currentPrintableLine := new(bytes.Buffer)
+
+	current := make([]byte, 1) // buffer for 1 byte reading
+
+	for {
+		if n, err := content.Read(current); err == io.EOF {
+			break
+		} else if n != 1 {
+			continue
+		} else if err != nil {
+			return nil, xerrors.Errorf("failed to read a byte: %w", err)
+		}
+		if unicode.IsPrint(rune(current[0])) {
+			_ = currentPrintableLine.WriteByte(current[0])
+			continue
+		}
+		if currentPrintableLine.Len() > minLength {
+			// add a newline between printable lines to separate them
+			_ = currentPrintableLine.WriteByte('\n')
+			result = append(result, currentPrintableLine.Bytes()...)
+		}
+		currentPrintableLine.Reset()
+	}
+	if currentPrintableLine.Len() > minLength {
+		// add a newline between printable lines to separate them
+		_ = currentPrintableLine.WriteByte('\n')
+		result = append(result, currentPrintableLine.Bytes()...)
+	}
+	return result, nil
+}