From 285c06bcaf608f0d3289e371b2d27417abe3d517 Mon Sep 17 00:00:00 2001 From: vardhaman22 Date: Fri, 11 Oct 2024 13:26:31 +0530 Subject: [PATCH] add field in report to store compressed base64 encoded avmap data --- go.mod | 3 + pkg/kb-summarizer/report/report.go | 4 +- pkg/kb-summarizer/summarizer/summarizer.go | 108 ++++++++ .../summarizer/summarizer_test.go | 230 ++++++++++++++++++ 4 files changed, 344 insertions(+), 1 deletion(-) create mode 100644 pkg/kb-summarizer/summarizer/summarizer_test.go diff --git a/go.mod b/go.mod index 434f8f25..b231bb84 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/aquasecurity/kube-bench v0.8.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.19.0 + github.com/stretchr/testify v1.9.0 github.com/urfave/cli/v2 v2.27.4 gopkg.in/yaml.v3 v3.0.1 ) @@ -15,6 +16,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/securityhub v1.37.0 // indirect github.com/aws/smithy-go v1.20.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/golang/glog v1.2.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -23,6 +25,7 @@ require ( github.com/onsi/ginkgo v1.16.5 // indirect github.com/onsi/gomega v1.27.10 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect diff --git a/pkg/kb-summarizer/report/report.go b/pkg/kb-summarizer/report/report.go index 628b1db2..d492630b 100644 --- a/pkg/kb-summarizer/report/report.go +++ b/pkg/kb-summarizer/report/report.go @@ -63,6 +63,8 @@ type Report struct { NotApplicable int `json:"notApplicable"` Nodes map[NodeType][]string `json:"nodes"` Results []*Group `json:"results"` + // ActualValueMapData is the base64-encoded gzipped-compressed avmap data of all checks. + ActualValueMapData string `json:"actual_value_map_data"` } func nodeTypeMapper(nodeType summarizer.NodeType) NodeType { @@ -163,7 +165,7 @@ func mapReport(internalReport *summarizer.SummarizedReport) (*Report, error) { externalReport.Warn = internalReport.Warn externalReport.NotApplicable = internalReport.NotApplicable externalReport.Nodes = mapNodes(internalReport.Nodes) - + externalReport.ActualValueMapData = internalReport.ActualValueMapData return externalReport, nil } diff --git a/pkg/kb-summarizer/summarizer/summarizer.go b/pkg/kb-summarizer/summarizer/summarizer.go index 964b9ba0..45113c5e 100644 --- a/pkg/kb-summarizer/summarizer/summarizer.go +++ b/pkg/kb-summarizer/summarizer/summarizer.go @@ -1,6 +1,9 @@ package summarizer import ( + "bytes" + "compress/gzip" + "encoding/base64" "encoding/json" "fmt" "io" @@ -121,6 +124,20 @@ type SummarizedReport struct { NotApplicable int `json:"na"` Nodes map[NodeType][]string `json:"n"` GroupWrappers []*GroupWrapper `json:"o"` + // ActualValueMapData is the base64-encoded gzipped-compressed avmap data of all checks. + ActualValueMapData string `json:"actual_value_map_data"` +} + +type ActualValueGroup struct { + ID string `yaml:"id" json:"id"` + Text string `json:"description"` + ActualValueChecks []*ActualValueCheck `json:"actual_value_checks"` +} + +type ActualValueCheck struct { + ID string `yaml:"id" json:"id"` + Text string `json:"description"` + ActualValueNodeMap map[string]string `json:"actual_value_node_map"` } type skipConfig struct { @@ -360,10 +377,17 @@ func (s *Summarizer) save() error { jsonWriter := io.Writer(jsonFile) encoder := json.NewEncoder(jsonWriter) encoder.SetIndent("", " ") + + err = s.handleAvMapData() + if err != nil { + return fmt.Errorf("failed to update avmap data, err: %w", err) + } + err = encoder.Encode(s.fullReport) if err != nil { return fmt.Errorf("error encoding: %v", err) } + logrus.Infof("successfully saved report file: %v", outputFilePath) return nil } @@ -778,3 +802,87 @@ func printCheck(check *kb.Check) { func printCheckWrapper(cw *CheckWrapper) { logrus.Debugf("checkWrapper: %+v", cw) } + +// handleAvMapData sets ActualValueMapData field for the fullReport and also set ActualValueNodeMap to nil for each CheckWrapper +// in the report. +func (s *Summarizer) handleAvMapData() error { + err := s.setFullReportActualValueMapData() + if err != nil { + return fmt.Errorf("failed to set actualValueMapData, err: %w", err) + } + + // because of ActualValueNodeMap values the size of clusterscan report was exceeding the 1 MB limit for large clusters + // so this data is aggregated for all the checks and then set to ActualValueMapData field of the report after compression. + // and ActualValueNodeMap is set to nil for each check wrapper. + s.resetAvmapPerCheck() + + return nil +} + +// setFullReportActualValueMapData sets ActualValueMapData field for the fullReport +func (s *Summarizer) setFullReportActualValueMapData() error { + avgroups := mapGroupWrappersToActualValueGroups(s.fullReport.GroupWrappers) + + jsonData, err := json.Marshal(avgroups) + if err != nil { + return fmt.Errorf("error encoding avgroups: %w", err) + } + + var buf bytes.Buffer + gzipWriter := gzip.NewWriter(&buf) + + _, err = gzipWriter.Write(jsonData) + if err != nil { + return fmt.Errorf("error writing compressed data: %w", err) + } + + if err := gzipWriter.Close(); err != nil { + return fmt.Errorf("error closing gzip writer: %w", err) + } + + compressedData := buf.Bytes() + + base64Data := base64.StdEncoding.EncodeToString(compressedData) + s.fullReport.ActualValueMapData = base64Data + + return nil +} + +func mapGroupWrappersToActualValueGroups(grpWrappers []*GroupWrapper) []*ActualValueGroup { + avgroups := make([]*ActualValueGroup, len(grpWrappers)) + + for gwIdx, gw := range grpWrappers { + avchecks := make([]*ActualValueCheck, len(gw.CheckWrappers)) + + for cwIdx, cw := range gw.CheckWrappers { + + avchecks[cwIdx] = &ActualValueCheck{ + ID: cw.ID, + Text: cw.Text, + ActualValueNodeMap: make(map[string]string, len(cw.ActualValueNodeMap)), + } + + for k, v := range cw.ActualValueNodeMap { + avchecks[cwIdx].ActualValueNodeMap[k] = v + } + + } + + avgroups[gwIdx] = &ActualValueGroup{ + ID: gw.ID, + Text: gw.Text, + ActualValueChecks: avchecks, + } + } + + return avgroups +} + +// resetAvmapPerCheck sets the ActualValueNodeMap to nil for each CheckWrapper in the fullReport +func (s *Summarizer) resetAvmapPerCheck() { + for _, gw := range s.fullReport.GroupWrappers { + for _, cw := range gw.CheckWrappers { + cw.ActualValueNodeMap = nil + } + } +} diff --git a/pkg/kb-summarizer/summarizer/summarizer_test.go b/pkg/kb-summarizer/summarizer/summarizer_test.go new file mode 100644 index 00000000..01f44c4c --- /dev/null +++ b/pkg/kb-summarizer/summarizer/summarizer_test.go @@ -0,0 +1,230 @@ +package summarizer + +import ( + "bytes" + "compress/gzip" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var ( + testAvNodeMap1 = map[string]string{ + "node1": "permission=644", + "node2": "permission=640", + "node3": "permission=600", + } + + testAvNodeMap2 = map[string]string{ + "node1": "testvalue", + "node2": "testvalue1", + "node3": "testvalue2", + } + + testAvNodeMap3 = map[string]string{ + "node1": "true", + "node2": "false", + "node3": "true", + } +) + +func TestSummarizer_handleAvMapData(t *testing.T) { + gwTestData, err := getGroupWrappersTestData() + require.Nil(t, err, "error while getting groupwrappers test data") + + avGroupsTestData, err := getAvGroupTestData() + require.Nil(t, err, "error while getting avgroups test data") + + s := Summarizer{ + fullReport: &SummarizedReport{ + GroupWrappers: gwTestData, + }, + } + + err = s.handleAvMapData() + require.Nil(t, err, "failed to update ActualValueMapData for fullReport") + + require.NotEmpty(t, s.fullReport.ActualValueMapData, "empty actualValueMapData found") + + compressedAvMapData, err := base64.StdEncoding.DecodeString(s.fullReport.ActualValueMapData) + require.Nil(t, err, "error while decoding ActualValueMapData") + require.NotNil(t, compressedAvMapData, "compressed ActualValueMapData should not be empty") + + r, err := gzip.NewReader(bytes.NewBuffer(compressedAvMapData)) + require.Nil(t, err, "error while reading compressed avMapData") + defer r.Close() + + avgroupsJSON, err := io.ReadAll(r) + require.Nil(t, err, "error while reading avMapData json") + + expectedAvGroupsJSON, err := json.Marshal(avGroupsTestData) + require.Nil(t, err, "error while encoding expected avgroups data") + + // verify ActualValueMapData + require.Equal(t, string(expectedAvGroupsJSON), string(avgroupsJSON), "avmapData is not correctly encoded") + + // check if ActualValueNodeMap is set to nil for each check + for _, gw := range s.fullReport.GroupWrappers { + for _, cw := range gw.CheckWrappers { + require.Nil(t, cw.ActualValueNodeMap, nil, "ActualValueNodeMap is not set to nil for the check") + } + } +} + +func TestMapGroupWrappersToActualValueGroups(t *testing.T) { + gwTestData, err := getGroupWrappersTestData() + require.Nil(t, err, "error while getting groupwrappers test data") + + avGroupsTestData, err := getAvGroupTestData() + require.Nil(t, err, "error while getting avgroups test data") + + avgroups := mapGroupWrappersToActualValueGroups(gwTestData) + avgroupsJSON, err := json.Marshal(avgroups) + require.Nil(t, err, "error while encoding avgroups data") + + expectedAvGroupsJSON, err := json.Marshal(avGroupsTestData) + require.Nil(t, err, "error while encoding expected avgroups data") + + assert.EqualValues(t, string(expectedAvGroupsJSON), string(avgroupsJSON), "GroupWrappers are not correctly mapped to ActualValueGroups") +} + +func getGroupWrappersTestData() ([]*GroupWrapper, error) { + groupWrappersTestData := []*GroupWrapper{ + &GroupWrapper{ + ID: "1.1", + Text: "Checks for group 1.1", + CheckWrappers: []*CheckWrapper{ + &CheckWrapper{ + ID: "1.1.1", + Text: "Check 1.1.1", + ActualValueNodeMap: testAvNodeMap1, + }, + &CheckWrapper{ + ID: "1.1.2", + Text: "Check 1.1.2", + ActualValueNodeMap: testAvNodeMap2, + }, + }, + }, + &GroupWrapper{ + ID: "1.2", + Text: "Checks for group 1.2", + CheckWrappers: []*CheckWrapper{ + &CheckWrapper{ + ID: "1.2.1", + Text: "Check 1.2.1", + ActualValueNodeMap: testAvNodeMap2, + }, + }, + }, + &GroupWrapper{ + ID: "2.1", + Text: "Checks for group 2.1", + CheckWrappers: []*CheckWrapper{ + &CheckWrapper{ + ID: "2.1.1", + Text: "Check 2.1.1", + ActualValueNodeMap: testAvNodeMap2, + }, + }, + }, + &GroupWrapper{ + ID: "3.1", + Text: "Checks for group 2.2", + CheckWrappers: []*CheckWrapper{ + &CheckWrapper{ + ID: "3.2", + Text: "Check 3.1", + ActualValueNodeMap: testAvNodeMap3, + }, + }, + }, + } + + groupWrappersTestDataJSON, err := json.Marshal(groupWrappersTestData) + if err != nil { + return nil, fmt.Errorf("error while json encoding group wrappers test data: %w", err) + } + + testData := []*GroupWrapper{} + + err = json.Unmarshal(groupWrappersTestDataJSON, &testData) + if err != nil { + return nil, fmt.Errorf("error while json decoding group wrappers test data: %w", err) + } + + return testData, nil +} + +func getAvGroupTestData() ([]*ActualValueGroup, error) { + avGroupsTestData := []*ActualValueGroup{ + &ActualValueGroup{ + ID: "1.1", + Text: "Checks for group 1.1", + ActualValueChecks: []*ActualValueCheck{ + &ActualValueCheck{ + ID: "1.1.1", + Text: "Check 1.1.1", + ActualValueNodeMap: testAvNodeMap1, + }, + &ActualValueCheck{ + ID: "1.1.2", + Text: "Check 1.1.2", + ActualValueNodeMap: testAvNodeMap2, + }, + }, + }, + &ActualValueGroup{ + ID: "1.2", + Text: "Checks for group 1.2", + ActualValueChecks: []*ActualValueCheck{ + &ActualValueCheck{ + ID: "1.2.1", + Text: "Check 1.2.1", + ActualValueNodeMap: testAvNodeMap2, + }, + }, + }, + &ActualValueGroup{ + ID: "2.1", + Text: "Checks for group 2.1", + ActualValueChecks: []*ActualValueCheck{ + &ActualValueCheck{ + ID: "2.1.1", + Text: "Check 2.1.1", + ActualValueNodeMap: testAvNodeMap2, + }, + }, + }, + &ActualValueGroup{ + ID: "3.1", + Text: "Checks for group 2.2", + ActualValueChecks: []*ActualValueCheck{ + &ActualValueCheck{ + ID: "3.2", + Text: "Check 3.1", + ActualValueNodeMap: testAvNodeMap3, + }, + }, + }, + } + + groupWrappersTestDataJSON, err := json.Marshal(avGroupsTestData) + if err != nil { + return nil, fmt.Errorf("error while json encoding group wrappers test data: %w", err) + } + + testData := []*ActualValueGroup{} + + err = json.Unmarshal(groupWrappersTestDataJSON, &testData) + if err != nil { + return nil, fmt.Errorf("error while json decoding group wrappers test data: %w", err) + } + + return testData, nil +}