From 9295f1a4fa2b6bdaf4e170bf9681d75eba5feda7 Mon Sep 17 00:00:00 2001 From: Robi Nino Date: Wed, 4 Sep 2024 19:18:13 +0300 Subject: [PATCH] Aggregate code scanning sarifs from command summaries (#2683) --- general/summary/cli.go | 80 ++++++++++++++++++++++++++++++++++-------- go.mod | 8 ++--- go.sum | 8 ++--- main.go | 2 +- transfer_test.go | 1 + 5 files changed, 76 insertions(+), 23 deletions(-) diff --git a/general/summary/cli.go b/general/summary/cli.go index a3d282a0a..495b5a34a 100644 --- a/general/summary/cli.go +++ b/general/summary/cli.go @@ -27,8 +27,8 @@ const ( ) const ( - JfrogCliSummaryDir = "jfrog-command-summary" - MarkdownFileName = "markdown.md" + markdownFileName = "markdown.md" + finalSarifFileName = "final.sarif" ) var markdownSections = []MarkdownSection{Security, BuildInfo, Upload} @@ -37,13 +37,22 @@ func (ms MarkdownSection) String() string { return string(ms) } -// GenerateSummaryMarkdown creates a summary of recorded CLI commands in Markdown format. -func GenerateSummaryMarkdown(c *cli.Context) error { +// Generates a combined markdown from all sections, and aggregates multiple SARIF files into one. +func FinalizeCommandSummaries(c *cli.Context) error { if !shouldGenerateSummary() { return fmt.Errorf("unable to generate the command summary because the output directory is not specified."+ " Please ensure that the environment variable '%s' is set before running your commands to enable summary generation", coreutils.SummaryOutputDirPathEnv) } + if err := generateSummaryMarkdown(c); err != nil { + return err + } + + return aggregatedCodeScanningSarifs() +} + +// generateSummaryMarkdown creates a summary of recorded CLI commands in Markdown format. +func generateSummaryMarkdown(c *cli.Context) error { // Get URL and Version to generate summary links serverUrl, majorVersion, err := extractServerUrlAndVersion(c) if err != nil { @@ -71,6 +80,26 @@ func GenerateSummaryMarkdown(c *cli.Context) error { return saveMarkdownToFileSystem(finalMarkdown) } +func aggregatedCodeScanningSarifs() error { + files, err := getSarifFiles() + if err != nil { + return err + } + if len(files) == 0 { + log.Debug("No sarif reports were found") + return nil + } + finalSarif, err := securityUtils.CombineSarifOutputFiles(files) + if err != nil { + return err + } + return saveFinalSarifToFileSystem(string(finalSarif)) +} + +func getSarifReportsDir() string { + return filepath.Join(os.Getenv(coreutils.SummaryOutputDirPathEnv), commandsummary.OutputDirName, string(Security), string(commandsummary.SarifReport)) +} + // The CLI generates summaries in sections, with each section as a separate Markdown file. // This function merges all sections into a single Markdown file and saves it in the root of the // command summary output directory. @@ -93,23 +122,29 @@ func saveMarkdownToFileSystem(finalMarkdown string) (err error) { if finalMarkdown == "" { return nil } - filePath := filepath.Join(os.Getenv(coreutils.SummaryOutputDirPathEnv), JfrogCliSummaryDir, MarkdownFileName) + filePath := filepath.Join(os.Getenv(coreutils.SummaryOutputDirPathEnv), commandsummary.OutputDirName, markdownFileName) + return saveFile(finalMarkdown, filePath) +} + +func saveFile(content, filePath string) (err error) { + if content == "" { + return nil + } file, err := os.Create(filePath) if err != nil { - return fmt.Errorf("error creating markdown file: %w", err) + return err } defer func() { err = errors.Join(err, file.Close()) }() - // Write to file - if _, err := file.WriteString(finalMarkdown); err != nil { - return fmt.Errorf("error writing to markdown file: %w", err) + if _, err = file.WriteString(content); err != nil { + return err } return nil } func getSectionMarkdownContent(section MarkdownSection) (string, error) { - sectionFilepath := filepath.Join(os.Getenv(coreutils.SummaryOutputDirPathEnv), JfrogCliSummaryDir, string(section), MarkdownFileName) + sectionFilepath := filepath.Join(os.Getenv(coreutils.SummaryOutputDirPathEnv), commandsummary.OutputDirName, string(section), markdownFileName) if _, err := os.Stat(sectionFilepath); os.IsNotExist(err) { return "", nil } @@ -124,6 +159,23 @@ func getSectionMarkdownContent(section MarkdownSection) (string, error) { return string(contentBytes), nil } +func getSarifFiles() (files []string, err error) { + indexedFiles, err := commandsummary.GetIndexedDataFilesPaths() + if err != nil { + return + } + sarifsMap := indexedFiles[commandsummary.SarifReport] + for i := range sarifsMap { + files = append(files, sarifsMap[i]) + } + return +} + +func saveFinalSarifToFileSystem(finalSarif string) (err error) { + filePath := filepath.Join(getSarifReportsDir(), finalSarifFileName) + return saveFile(finalSarif, filePath) +} + // Initiate the desired command summary implementation and invoke its Markdown generation. func invokeSectionMarkdownGeneration(section MarkdownSection) error { switch section { @@ -151,7 +203,7 @@ func generateBuildInfoMarkdown() error { if err != nil { return fmt.Errorf("error generating build-info markdown: %w", err) } - if err = mapScanResults(buildInfoSummary); err != nil { + if err = mapScanResults(); err != nil { return fmt.Errorf("error mapping scan results: %w", err) } return buildInfoSummary.GenerateMarkdown() @@ -170,9 +222,9 @@ func generateUploadMarkdown() error { } // mapScanResults maps the scan results saved during runtime into scan components. -func mapScanResults(commandSummary *commandsummary.CommandSummary) (err error) { +func mapScanResults() (err error) { // Gets the saved scan results file paths. - indexedFiles, err := commandSummary.GetIndexedDataFilesPaths() + indexedFiles, err := commandsummary.GetIndexedDataFilesPaths() if err != nil { return err } @@ -216,7 +268,7 @@ func processScan(index commandsummary.Index, filePath string, scannedName string // shouldGenerateUploadSummary checks if upload summary should be generated. func shouldGenerateUploadSummary() (bool, error) { - buildInfoPath := filepath.Join(os.Getenv(coreutils.SummaryOutputDirPathEnv), JfrogCliSummaryDir, string(BuildInfo)) + buildInfoPath := filepath.Join(os.Getenv(coreutils.SummaryOutputDirPathEnv), commandsummary.OutputDirName, string(BuildInfo)) if _, err := os.Stat(buildInfoPath); os.IsNotExist(err) { return true, nil } diff --git a/go.mod b/go.mod index 6ce594f19..f97e42df8 100644 --- a/go.mod +++ b/go.mod @@ -19,9 +19,9 @@ require ( github.com/jfrog/build-info-go v1.9.35 github.com/jfrog/gofrog v1.7.5 github.com/jfrog/jfrog-cli-artifactory v0.1.6 - github.com/jfrog/jfrog-cli-core/v2 v2.55.6 + github.com/jfrog/jfrog-cli-core/v2 v2.55.7 github.com/jfrog/jfrog-cli-platform-services v1.3.0 - github.com/jfrog/jfrog-cli-security v1.7.2 + github.com/jfrog/jfrog-cli-security v1.8.0 github.com/jfrog/jfrog-client-go v1.46.1 github.com/jszwec/csvutil v1.10.0 github.com/stretchr/testify v1.9.0 @@ -171,9 +171,9 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -// replace github.com/jfrog/jfrog-cli-core/v2 => github.com/eyaldelarea/jfrog-cli-core/v2 v2.0.0-20240829171158-7b0f89df2c0c +// replace github.com/jfrog/jfrog-cli-core/v2 => github.com/RobiNino/jfrog-cli-core/v2 v2.0.0-20240904105726-775a1614224e -// replace github.com/jfrog/jfrog-cli-security => github.com/attiasas/jfrog-cli-security v0.0.0-20240829151632-3a7a90969eca +// replace github.com/jfrog/jfrog-cli-security => github.com/attiasas/jfrog-cli-security v0.0.0-20240904061406-f368939ce3a0 // replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240806162439-01bb7dcd43fc diff --git a/go.sum b/go.sum index 5e4b116dd..6c23b575e 100644 --- a/go.sum +++ b/go.sum @@ -941,12 +941,12 @@ github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYL github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= github.com/jfrog/jfrog-cli-artifactory v0.1.6 h1:bMfJsrLQJw0dZp4nqUf1xOmtY0rpCatW/I5q88x+fhQ= github.com/jfrog/jfrog-cli-artifactory v0.1.6/go.mod h1:jbNb22ebtupcjdhrdGq0VBew2vWG6VUK04xxGNDfynE= -github.com/jfrog/jfrog-cli-core/v2 v2.55.6 h1:3tQuEdYgS2q7fkrrSG66OnO0S998FXGaY9BVsxSLst4= -github.com/jfrog/jfrog-cli-core/v2 v2.55.6/go.mod h1:DPO5BfWAeOByahFMMy+PcjmbPlcyoRy7Bf2C5sGKVi0= +github.com/jfrog/jfrog-cli-core/v2 v2.55.7 h1:V4dO2FMNIH49lov3dMj3jYRg8KBTG7hyhHI8ftYByf8= +github.com/jfrog/jfrog-cli-core/v2 v2.55.7/go.mod h1:DPO5BfWAeOByahFMMy+PcjmbPlcyoRy7Bf2C5sGKVi0= github.com/jfrog/jfrog-cli-platform-services v1.3.0 h1:IblSDZFBjL7WLRi37Ni2DmHrXJJ6ysSMxx7t41AvyDA= github.com/jfrog/jfrog-cli-platform-services v1.3.0/go.mod h1:Ky4SDXuMeaiNP/5zMT1YSzIuXG+cNYYOl8BaEA7Awbc= -github.com/jfrog/jfrog-cli-security v1.7.2 h1:Kvabj/6LhM+WEb6woIqqbv2VmIj69IFwz859Sys1Tgs= -github.com/jfrog/jfrog-cli-security v1.7.2/go.mod h1:4eztJ+gBb7Xtq/TtnOvIodBOMZutPIAZOuLxqHWXrOo= +github.com/jfrog/jfrog-cli-security v1.8.0 h1:jp/AVaQcItUNXRCud5PMyl8VVjPuzfrNHJWQvWAMnms= +github.com/jfrog/jfrog-cli-security v1.8.0/go.mod h1:DjufYZpsTwILOFJlx7tR/y63oLBRmtPtFIz1WgiP/X4= github.com/jfrog/jfrog-client-go v1.46.1 h1:ExqOF8ClOG9LO3vbm6jTIwQHHhprbu8lxB2RrM6mMI0= github.com/jfrog/jfrog-client-go v1.46.1/go.mod h1:UCu2JNBfMp9rypEmCL84DCooG79xWIHVadZQR3Ab+BQ= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= diff --git a/main.go b/main.go index 0f13d25e7..82af701a2 100644 --- a/main.go +++ b/main.go @@ -300,7 +300,7 @@ func getCommands() ([]cli.Command, error) { Usage: summaryDocs.GetDescription(), HelpName: corecommon.CreateUsage("gsm", summaryDocs.GetDescription(), summaryDocs.Usage), Category: otherCategory, - Action: summary.GenerateSummaryMarkdown, + Action: summary.FinalizeCommandSummaries, }, } diff --git a/transfer_test.go b/transfer_test.go index 592dc9ea3..2fe91bd22 100644 --- a/transfer_test.go +++ b/transfer_test.go @@ -369,6 +369,7 @@ func generateTestRepoSnapshotFile(t *testing.T, repoKey, repoSnapshotFilePath st func addChildWithFiles(t *testing.T, parent *reposnapshot.Node, dirName string, explored, checkCompleted bool, filesCount int) *reposnapshot.Node { childNode := reposnapshot.CreateNewNode(dirName, nil) for i := 0; i < filesCount; i++ { + //#nosec G115 assert.NoError(t, childNode.IncrementFilesCount(uint64(i))) }