Merge pull request #263 from vardhaman22/v2.9-add-field-for-avmap-data

[release/v0.4] add field in report to store compressed base64 encoded avmap data

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/aquasecurity/kube-bench v0.8.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/viper v1.19.0
+	github.com/stretchr/testify v1.9.0
 	github.com/urfave/cli/v2 v2.27.4
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -15,6 +16,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/securityhub v1.37.0 // indirect
 	github.com/aws/smithy-go v1.20.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/golang/glog v1.2.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
@@ -23,6 +25,7 @@ require (
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/onsi/gomega v1.27.10 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect

pkg/kb-summarizer/report/report.go

@@ -63,6 +63,8 @@ type Report struct {
 	NotApplicable int                   `json:"notApplicable"`
 	Nodes         map[NodeType][]string `json:"nodes"`
 	Results       []*Group              `json:"results"`
+	// ActualValueMapData is the base64-encoded gzipped-compressed avmap data of all checks.
+	ActualValueMapData string `json:"actual_value_map_data"`
 }
 
 func nodeTypeMapper(nodeType summarizer.NodeType) NodeType {
@@ -163,7 +165,7 @@ func mapReport(internalReport *summarizer.SummarizedReport) (*Report, error) {
 	externalReport.Warn = internalReport.Warn
 	externalReport.NotApplicable = internalReport.NotApplicable
 	externalReport.Nodes = mapNodes(internalReport.Nodes)
-
+	externalReport.ActualValueMapData = internalReport.ActualValueMapData
 	return externalReport, nil
 }
 

pkg/kb-summarizer/summarizer/summarizer.go

@@ -1,6 +1,9 @@
 package summarizer
 
 import (
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -121,6 +124,20 @@ type SummarizedReport struct {
 	NotApplicable int                   `json:"na"`
 	Nodes         map[NodeType][]string `json:"n"`
 	GroupWrappers []*GroupWrapper       `json:"o"`
+	// ActualValueMapData is the base64-encoded gzipped-compressed avmap data of all checks.
+	ActualValueMapData string `json:"actual_value_map_data"`
+}
+
+type ActualValueGroup struct {
+	ID                string              `yaml:"id" json:"id"`
+	Text              string              `json:"description"`
+	ActualValueChecks []*ActualValueCheck `json:"actual_value_checks"`
+}
+
+type ActualValueCheck struct {
+	ID                 string            `yaml:"id" json:"id"`
+	Text               string            `json:"description"`
+	ActualValueNodeMap map[string]string `json:"actual_value_node_map"`
 }
 
 type skipConfig struct {
@@ -360,10 +377,17 @@ func (s *Summarizer) save() error {
 	jsonWriter := io.Writer(jsonFile)
 	encoder := json.NewEncoder(jsonWriter)
 	encoder.SetIndent("", " ")
+
+	err = s.handleAvMapData()
+	if err != nil {
+		return fmt.Errorf("failed to update avmap data, err: %w", err)
+	}
+
 	err = encoder.Encode(s.fullReport)
 	if err != nil {
 		return fmt.Errorf("error encoding: %v", err)
 	}
+
 	logrus.Infof("successfully saved report file: %v", outputFilePath)
 	return nil
 }
@@ -778,3 +802,87 @@ func printCheck(check *kb.Check) {
 func printCheckWrapper(cw *CheckWrapper) {
 	logrus.Debugf("checkWrapper: %+v", cw)
 }
+
+// handleAvMapData sets ActualValueMapData field for the fullReport and also set ActualValueNodeMap to nil for each CheckWrapper
+// in the report.
+func (s *Summarizer) handleAvMapData() error {
+	err := s.setFullReportActualValueMapData()
+	if err != nil {
+		return fmt.Errorf("failed to set actualValueMapData, err: %w", err)
+	}
+
+	// because of ActualValueNodeMap values the size of clusterscan report was exceeding the 1 MB limit for large clusters
+	// so this data is aggregated for all the checks and then set to ActualValueMapData field of the report after compression.
+	// and ActualValueNodeMap is set to nil for each check wrapper.
+	s.resetAvmapPerCheck()
+
+	return nil
+}
+
+// setFullReportActualValueMapData sets ActualValueMapData field for the fullReport
+func (s *Summarizer) setFullReportActualValueMapData() error {
+	avgroups := mapGroupWrappersToActualValueGroups(s.fullReport.GroupWrappers)
+
+	jsonData, err := json.Marshal(avgroups)
+	if err != nil {
+		return fmt.Errorf("error encoding avgroups: %w", err)
+	}
+
+	var buf bytes.Buffer
+	gzipWriter := gzip.NewWriter(&buf)
+
+	_, err = gzipWriter.Write(jsonData)
+	if err != nil {
+		return fmt.Errorf("error writing compressed data: %w", err)
+	}
+
+	if err := gzipWriter.Close(); err != nil {
+		return fmt.Errorf("error closing gzip writer: %w", err)
+	}
+
+	compressedData := buf.Bytes()
+
+	base64Data := base64.StdEncoding.EncodeToString(compressedData)
+	s.fullReport.ActualValueMapData = base64Data
+
+	return nil
+}
+
+func mapGroupWrappersToActualValueGroups(grpWrappers []*GroupWrapper) []*ActualValueGroup {
+	avgroups := make([]*ActualValueGroup, len(grpWrappers))
+
+	for gwIdx, gw := range grpWrappers {
+		avchecks := make([]*ActualValueCheck, len(gw.CheckWrappers))
+
+		for cwIdx, cw := range gw.CheckWrappers {
+
+			avchecks[cwIdx] = &ActualValueCheck{
+				ID:                 cw.ID,
+				Text:               cw.Text,
+				ActualValueNodeMap: make(map[string]string, len(cw.ActualValueNodeMap)),
+			}
+
+			for k, v := range cw.ActualValueNodeMap {
+				avchecks[cwIdx].ActualValueNodeMap[k] = v
+			}
+
+		}
+
+		avgroups[gwIdx] = &ActualValueGroup{
+			ID:                gw.ID,
+			Text:              gw.Text,
+			ActualValueChecks: avchecks,
+		}
+	}
+
+	return avgroups
+}
+
+// resetAvmapPerCheck sets the ActualValueNodeMap to nil for each CheckWrapper in the fullReport
+func (s *Summarizer) resetAvmapPerCheck() {
+	for _, gw := range s.fullReport.GroupWrappers {
+		for _, cw := range gw.CheckWrappers {
+			cw.ActualValueNodeMap = nil
+		}
+	}
+}