From bac1aa98957b49cdb19733bebe66337fd4cfe0b0 Mon Sep 17 00:00:00 2001 From: Thomas Legris Date: Mon, 8 Jul 2024 17:35:06 +0900 Subject: [PATCH] use new grpc abstraction --- go.mod | 4 +- go.sum | 2 + jobs/scan.go | 64 +------------------------- main.go | 17 ++++++- output/output.go | 25 +++++----- rules/yara.rules | 11 +++++ server/grpc.go | 116 ++++++++++++++++------------------------------- 7 files changed, 85 insertions(+), 154 deletions(-) create mode 100644 rules/yara.rules diff --git a/go.mod b/go.mod index f4262ae..2efda4e 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,8 @@ go 1.21.0 replace github.com/deepfence/agent-plugins-grpc => ./agent-plugins-grpc -replace github.com/deepfence/YaraHunter => ../YaraHunter - require ( - github.com/deepfence/YaraHunter v0.0.0-00010101000000-000000000000 + github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1 github.com/deepfence/agent-plugins-grpc v0.0.0-00010101000000-000000000000 github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9 github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9 diff --git a/go.sum b/go.sum index 6270cf7..0903485 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/containerd/typeurl/v2 v2.1.1/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3H github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1 h1:VuKkE8OKmM+hII/fViwYuLL1DTq5ncczMKWuOKg+EZc= +github.com/deepfence/YaraHunter v0.0.0-20240708090804-4196e3bbd2c1/go.mod h1:SSZ34MXU0qR30p6/rJ/PpY7OgwljWF3go6K/ZfYqHZ0= github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9 h1:hI/Fv6XabkERGza4E8g7XLhlkuzWjoXQwTmtq0WIC+Y= github.com/deepfence/golang_deepfence_sdk/client v0.0.0-20240626143546-e4ec9311fdf9/go.mod h1:+rchMc4YNjCoHo0YAwKsT+DRBNr1hdDG0WrvAOOCc5k= github.com/deepfence/golang_deepfence_sdk/utils v0.0.0-20240626143546-e4ec9311fdf9 h1:/1olNPTiYUFxuYsP79DKLEEieNWIdPDFxs+B58jysRA= diff --git a/jobs/scan.go b/jobs/scan.go index 61606b4..4bde5e0 100644 --- a/jobs/scan.go +++ b/jobs/scan.go @@ -2,77 +2,15 @@ package jobs import ( "encoding/json" - "fmt" "strings" "sync" - "time" - - "github.com/deepfence/SecretScanner/core" - "github.com/deepfence/SecretScanner/output" - "github.com/deepfence/SecretScanner/scan" - "github.com/deepfence/golang_deepfence_sdk/utils/tasks" pb "github.com/deepfence/agent-plugins-grpc/srcgo" - cfg "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" ) var ScanMap sync.Map -func DispatchScan(r *pb.FindRequest) { - go func() { - startScanJob() - defer stopScanJob() - - var err error - res, scanCtx := tasks.StartStatusReporter( - r.ScanId, - func(ss tasks.ScanStatus) error { - return writeSecretScanStatus(ss.ScanStatus, ss.ScanId, ss.ScanMessage) - }, - tasks.StatusValues{ - IN_PROGRESS: "IN_PROGRESS", - CANCELLED: "CANCELLED", - FAILED: "ERROR", - SUCCESS: "COMPLETE", - }, - time.Minute*20, - ) - - ScanMap.Store(r.ScanId, scanCtx) - - defer func() { - ScanMap.Delete(r.ScanId) - res <- err - close(res) - }() - - var ( - scanType scan.ScanType - nodeID string - ) - - if r.GetPath() != "" { - scanType = scan.DirScan - nodeID = r.GetPath() - } else if r.GetImage() != nil && r.GetImage().Name != "" { - scanType = scan.ImageScan - nodeID = r.GetImage().Name - } else if r.GetContainer() != nil && r.GetContainer().Id != "" { - scanType = scan.ContainerScan - nodeID = r.GetContainer().Id - } else { - err = fmt.Errorf("Invalid request") - return - } - - filters := cfg.Config2Filter(core.GetSession().ExtractorConfig) - err = scan.Scan(scanCtx, scanType, filters, "", nodeID, r.GetScanId(), func(sf output.SecretFound, s string) { - writeSingleScanData(output.SecretToSecretInfo(sf), r.ScanId) - }) - }() -} - type SecretScanDoc struct { pb.SecretInfo ScanID string `json:"scan_id,omitempty"` @@ -100,7 +38,7 @@ func writeMultiScanData(secrets []*pb.SecretInfo, scan_id string) { } } -func writeSingleScanData(secret *pb.SecretInfo, scan_id string) { +func WriteSingleScanData(secret *pb.SecretInfo, scan_id string) { if SecretScanDir == HostMountDir { secret.GetMatch().FullFilename = strings.Replace(secret.GetMatch().GetFullFilename(), SecretScanDir, "", 1) } diff --git a/main.go b/main.go index c82101b..9f7e913 100644 --- a/main.go +++ b/main.go @@ -39,12 +39,17 @@ import ( "github.com/deepfence/SecretScanner/core" "github.com/deepfence/SecretScanner/output" "github.com/deepfence/SecretScanner/scan" + "github.com/deepfence/SecretScanner/server" "github.com/deepfence/SecretScanner/signature" "github.com/deepfence/golang_deepfence_sdk/utils/tasks" "github.com/deepfence/match-scanner/pkg/config" log "github.com/sirupsen/logrus" + "google.golang.org/grpc" "github.com/deepfence/YaraHunter/pkg/runner" + yaraserver "github.com/deepfence/YaraHunter/pkg/server" + + pb "github.com/deepfence/agent-plugins-grpc/srcgo" ) const ( @@ -206,5 +211,15 @@ func main() { go runner.ScheduleYaraHunterUpdater(ctx, runnerOpts) } - runner.StartYaraHunter(ctx, runnerOpts, core.GetSession().ExtractorConfig) + runner.StartYaraHunter(ctx, runnerOpts, core.GetSession().ExtractorConfig, + + func(base *yaraserver.GRPCScannerServer) server.SecretGRPCServer { + return server.SecretGRPCServer{ + GRPCScannerServer: base, + UnimplementedSecretScannerServer: pb.UnimplementedSecretScannerServer{}, + } + }, + func(s grpc.ServiceRegistrar, impl any) { + pb.RegisterSecretScannerServer(s, impl.(pb.SecretScannerServer)) + }) } diff --git a/output/output.go b/output/output.go index 8db7250..a47f445 100644 --- a/output/output.go +++ b/output/output.go @@ -6,6 +6,7 @@ import ( "os" "time" + "github.com/deepfence/YaraHunter/pkg/output" pb "github.com/deepfence/agent-plugins-grpc/srcgo" "github.com/fatih/color" tw "github.com/olekukonko/tablewriter" @@ -180,7 +181,7 @@ func removeFirstLastChar(input string) string { return input[1 : len(input)-1] } -func SecretsToSecretInfos(out []SecretFound) []*pb.SecretInfo { +func SecretsToSecretInfos(out []output.IOCFound) []*pb.SecretInfo { res := make([]*pb.SecretInfo, 0) for _, v := range out { res = append(res, SecretToSecretInfo(v)) @@ -188,22 +189,24 @@ func SecretsToSecretInfos(out []SecretFound) []*pb.SecretInfo { return res } -func SecretToSecretInfo(out SecretFound) *pb.SecretInfo { +func SecretToSecretInfo(out output.IOCFound) *pb.SecretInfo { + matchedContent := "" + if len(out.Meta) != 0 { + matchedContent = out.Meta[0] + } + signature := "" + if len(out.StringsToMatch) != 0 { + signature = out.StringsToMatch[0] + } return &pb.SecretInfo{ ImageLayerId: out.LayerID, Rule: &pb.MatchRule{ - Id: int32(out.RuleID), Name: out.RuleName, - Part: out.PartToMatch, - StringToMatch: out.Match, - SignatureToMatch: out.Regex, + SignatureToMatch: signature, }, Match: &pb.Match{ - StartingIndex: int64(out.PrintBufferStartIndex), - RelativeStartingIndex: int64(out.MatchFromByte), - RelativeEndingIndex: int64(out.MatchToByte), - FullFilename: out.CompleteFilename, - MatchedContent: jsonMarshal(out.MatchedContents), + FullFilename: out.CompleteFilename, + MatchedContent: matchedContent, }, Severity: &pb.Severity{ Level: out.Severity, diff --git a/rules/yara.rules b/rules/yara.rules new file mode 100644 index 0000000..088487a --- /dev/null +++ b/rules/yara.rules @@ -0,0 +1,11 @@ +rule github_personal_access_token { + meta: + description = "Rule to match GitHub Personal Access Tokens (classic), Fine-grained & Github Actions Token" + author = "deepfence.io" + + strings: + $github_pat = /^gh[ps]_[a-zA-Z0-9]{36}|github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}/ + + condition: + $github_pat +} diff --git a/server/grpc.go b/server/grpc.go index 0cdf8bd..7f6b081 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -2,95 +2,59 @@ package server import ( "context" - "fmt" - "net" - "sync" "github.com/deepfence/SecretScanner/jobs" + "github.com/deepfence/SecretScanner/output" + out "github.com/deepfence/YaraHunter/pkg/output" + "github.com/deepfence/YaraHunter/pkg/server" pb "github.com/deepfence/agent-plugins-grpc/srcgo" - + "github.com/sirupsen/logrus" //nolint:typecheck - "github.com/deepfence/golang_deepfence_sdk/utils/tasks" - log "github.com/sirupsen/logrus" - "google.golang.org/grpc" ) -type gRPCServer struct { - socket_path string - plugin_name string +type SecretGRPCServer struct { + *server.GRPCScannerServer pb.UnimplementedSecretScannerServer - pb.UnimplementedAgentPluginServer - pb.UnimplementedScannersServer -} - -func (s *gRPCServer) ReportJobsStatus(context.Context, *pb.Empty) (*pb.JobReports, error) { - return &pb.JobReports{ - RunningJobs: jobs.GetRunningJobCount(), - }, nil -} - -func (s *gRPCServer) StopScan(c context.Context, req *pb.StopScanRequest) (*pb.StopScanResult, error) { - log.Errorf("Received StopScanRequest: %v", *req) - scanID := req.ScanId - result := &pb.StopScanResult{ - Success: true, - Description: "", - } - - obj, found := jobs.ScanMap.Load(scanID) - if !found { - log.Errorf("SecretScanner::Failed to Stop scan, may have already completed successfully or errored out, scan_id: %s", scanID) - result.Success = false - result.Description = "SecretScanner::Failed to Stop scan" - return result, nil - } else { - log.Errorf("SecretScanner::Stop request submitted") - result.Description = "SecretScanner::Stop request submitted" - } - - scanCtx := obj.(*tasks.ScanContext) - scanCtx.StopTriggered.Store(true) - scanCtx.Cancel() - return result, nil -} - -func (s *gRPCServer) GetName(context.Context, *pb.Empty) (*pb.Name, error) { - return &pb.Name{Str: s.plugin_name}, nil } -func (s *gRPCServer) GetUID(context.Context, *pb.Empty) (*pb.Uid, error) { - return &pb.Uid{Str: fmt.Sprintf("%s-%s", s.plugin_name, s.socket_path)}, nil -} - -func (s *gRPCServer) FindSecretInfo(c context.Context, r *pb.FindRequest) (*pb.FindResult, error) { - jobs.DispatchScan(r) - return &pb.FindResult{}, nil -} - -func RunServer(ctx context.Context, socket_path string, plugin_name string) error { - - lis, err := net.Listen("unix", fmt.Sprintf("%s", socket_path)) +func (s *SecretGRPCServer) FindSecretInfo(c context.Context, r *pb.FindRequest) (*pb.FindResult, error) { + yaraScanner, err := s.YaraRules.NewScanner() if err != nil { - return err + return &pb.FindResult{}, err } - s := grpc.NewServer() - jobs.ScanMap = sync.Map{} - - impl := &gRPCServer{socket_path: socket_path, plugin_name: plugin_name} - pb.RegisterAgentPluginServer(s, impl) - pb.RegisterSecretScannerServer(s, impl) - pb.RegisterScannersServer(s, impl) - log.Infof("main: server listening at %v", lis.Addr()) go func() { - if err := s.Serve(lis); err != nil { - log.Errorf("server: %v", err) + logrus.Infof("request to scan %+v", r) + + namespace := "" + container := "" + image := "" + path := "" + switch { + case r.GetContainer() != nil: + namespace = r.GetContainer().GetNamespace() + container = r.GetContainer().GetId() + case r.GetImage() != nil: + image = r.GetImage().GetName() + default: + path = r.GetPath() } - }() - - <-ctx.Done() - s.GracefulStop() - log.Infof("main: exiting gracefully") - return nil + server.DoScan( + r.ScanId, + s.HostMountPath, + s.ExtractorConfig, + s.InactiveThreshold, + &s.ScanMap, + namespace, + path, + image, + container, + yaraScanner, + func(res out.IOCFound, scanID string) { + jobs.WriteSingleScanData(output.SecretToSecretInfo(res), scanID) + }, + ) + }() + return &pb.FindResult{}, nil }