Improve marshal performance by using pointers within CycloneDX Vulner…

…ability data structures (#65) * Improve performance by using pointers within CycloneDX Vulnerability structs Signed-off-by: Matt Rutkowski <[email protected]> * Migrate Vuln. struct members to pointers and update marhsal routines Signed-off-by: Matt Rutkowski <[email protected]> * Introduce const for JSON indent. spacing and set to conventional defaults Signed-off-by: Matt Rutkowski <[email protected]> --------- Signed-off-by: Matt Rutkowski <[email protected]>
CycloneDX · Nov 14, 2023 · e421ef1 · e421ef1
1 parent a30779e
commit e421ef1
Show file tree

Hide file tree

Showing 13 changed files with 404 additions and 267 deletions.
diff --git a/cmd/query_test.go b/cmd/query_test.go
@@ -61,7 +61,7 @@ func innerQuery(t *testing.T, filename string, queryRequest *common.QueryRequest
 	}
 
 	// This will print results ONLY if --quiet mode is `false`
-	printResult(result)
+	printMarshaledResultOnlyIfNotQuiet(result)
 	return
 }
 
@@ -122,15 +122,6 @@ func VerifySelectedFieldsInJsonMap(t *testing.T, keys []string, results interfac
 	return
 }
 
-func printResult(iResult interface{}) {
-	if !*TestLogQuiet {
-		// Format results in JSON
-		fResult, _ := utils.MarshalAnyToFormattedJsonString(iResult)
-		// Output the JSON data directly to stdout (not subject to log-level)
-		fmt.Printf("%s\n", fResult)
-	}
-}
-
 // ----------------------------------------
 // Command flag tests
 // ----------------------------------------

diff --git a/cmd/root_test.go b/cmd/root_test.go
@@ -283,3 +283,13 @@ func bufferContainsValues(buffer bytes.Buffer, values ...string) bool {
 	}
 	return true
 }
+
+// TODO: find a better way using some log package feature
+func printMarshaledResultOnlyIfNotQuiet(iResult interface{}) {
+	if !*TestLogQuiet {
+		// Format results in JSON
+		fResult, _ := utils.MarshalAnyToFormattedJsonString(iResult)
+		// Output the JSON data directly to stdout (not subject to log-level)
+		fmt.Printf("%s\n", fResult)
+	}
+}
diff --git a/cmd/trim.go b/cmd/trim.go
@@ -34,6 +34,14 @@ const (
 	FLAG_TRIM_MAP_KEYS   = "keys"
 )
 
+// TODO: make flag configurable:
+// NOTE: 4-space indent is accepted convention:
+// https://docs.openstack.org/doc-contrib-guide/json-conv.html
+const (
+	TRIM_OUTPUT_PREFIX = ""
+	TRIM_OUTPUT_INDENT = "    "
+)
+
 // flag help (translate)
 const (
 	FLAG_TRIM_OUTPUT_FORMAT_HELP = "format output using the specified type"
@@ -207,12 +215,12 @@ func Trim(writer io.Writer, persistentFlags utils.PersistentCommandFlags, trimFl
 	getLogger().Infof("Outputting listing (`%s` format)...", format)
 	switch format {
 	case FORMAT_JSON:
-		err = document.EncodeAsFormattedJSON(writer, "", "  ")
+		err = document.EncodeAsFormattedJSON(writer, TRIM_OUTPUT_PREFIX, TRIM_OUTPUT_INDENT)
 	default:
 		// Default to Text output for anything else (set as flag default)
-		getLogger().Warningf("Stats not supported for `%s` format; defaulting to `%s` format...",
-			format, FORMAT_TEXT)
-		err = document.EncodeAsFormattedJSON(writer, "", "  ")
+		getLogger().Warningf("Trim not supported for `%s` format; defaulting to `%s` format...",
+			format, FORMAT_JSON)
+		err = document.EncodeAsFormattedJSON(writer, TRIM_OUTPUT_PREFIX, TRIM_OUTPUT_INDENT)
 	}
 
 	return

diff --git a/cmd/trim_test.go b/cmd/trim_test.go
@@ -37,6 +37,7 @@ const (
 	TEST_TRIM_CDX_1_4_ENCODED_CHARS           = "test/trim/trim-cdx-1-4-sample-encoded-chars.sbom.json"
 	TEST_TRIM_CDX_1_4_SAMPLE_XXL_1            = "test/trim/trim-cdx-1-4-sample-xxl-1.sbom.json"
 	TEST_TRIM_CDX_1_5_SAMPLE_SMALL_COMPS_ONLY = "test/trim/trim-cdx-1-5-sample-small-components-only.sbom.json"
+	TEST_TRIM_CDX_1_4_SAMPLE_VEX              = "test/trim/trim-cdx-1-4-sample-vex.json"
 	TEST_TRIM_CDX_1_5_SAMPLE_MEDIUM_1         = "test/trim/trim-cdx-1-5-sample-medium-1.sbom.json"
 )
 
@@ -352,3 +353,24 @@ func TestTrimCdx15FooFromTools(t *testing.T) {
 		t.Error(fmt.Errorf("invalid trim result: string not found: %s", TEST_STRING_1))
 	}
 }
+
+func TestTrimCdx14SourceFromVulnerabilities(t *testing.T) {
+	ti := NewTrimTestInfoBasic(TEST_TRIM_CDX_1_4_SAMPLE_VEX, nil)
+	ti.Keys = append(ti.Keys, "source")
+	ti.FromPaths = []string{"vulnerabilities"}
+	ti.TestOutputVariantName = utils.GetCallerFunctionName(2)
+	ti.OutputFile = ti.CreateTemporaryFilename(TEST_TRIM_CDX_1_4_SAMPLE_VEX)
+
+	buffer, _, err := innerTestTrim(t, ti)
+	s := buffer.String()
+	if err != nil {
+		getLogger().Debugf("result: %s", s)
+		t.Error(err)
+	}
+
+	// Assure JSON map does not contain the trimmed key(s)
+	err = VerifyTrimOutputFileResult(t, ti, ti.Keys, ti.FromPaths[0])
+	if err != nil {
+		t.Error(err)
+	}
+}
diff --git a/cmd/vulnerability.go b/cmd/vulnerability.go
@@ -22,7 +22,6 @@ import (
 	"encoding/csv"
 	"fmt"
 	"io"
-	"reflect"
 	"sort"
 	"strings"
 	"text/tabwriter"
@@ -260,156 +259,14 @@ func loadDocumentVulnerabilities(document *schema.BOM, whereFilters []common.Whe
 	// Hash all components found in the (root).components[] (+ "nested" components)
 	pVulnerabilities := document.GetCdxVulnerabilities()
 	if pVulnerabilities != nil && len(*pVulnerabilities) > 0 {
-		if err = hashVulnerabilities(document, *pVulnerabilities, whereFilters); err != nil {
+		if err = document.HashVulnerabilities(*pVulnerabilities, whereFilters); err != nil {
 			return
 		}
 	}
 
 	return
 }
 
-// We need to hash our own informational structure around the CDX data in order
-// to simplify --where queries to command line users
-func hashVulnerabilities(bom *schema.BOM, vulnerabilities []schema.CDXVulnerability, whereFilters []common.WhereFilter) (err error) {
-	getLogger().Enter()
-	defer getLogger().Exit(err)
-
-	for _, cdxVulnerability := range vulnerabilities {
-		_, err = hashVulnerability(bom, cdxVulnerability, whereFilters)
-		if err != nil {
-			return
-		}
-	}
-	return
-}
-
-// Hash a CDX Component and recursively those of any "nested" components
-// TODO we should WARN if version is not a valid semver (e.g., examples/cyclonedx/BOM/laravel-7.12.0/bom.1.3.json)
-func hashVulnerability(bom *schema.BOM, cdxVulnerability schema.CDXVulnerability, whereFilters []common.WhereFilter) (vi *schema.VulnerabilityInfo, err error) {
-	getLogger().Enter()
-	defer getLogger().Exit(err)
-	var vulnInfo schema.VulnerabilityInfo
-	vi = &vulnInfo
-
-	if reflect.DeepEqual(cdxVulnerability, schema.CDXVulnerability{}) {
-		err = getLogger().Errorf("invalid vulnerability info: missing or empty : %v ", cdxVulnerability)
-		return
-	}
-
-	if cdxVulnerability.Id == "" {
-		getLogger().Warningf("vulnerability missing required value `id` : %v ", cdxVulnerability)
-	}
-
-	if cdxVulnerability.Published == "" {
-		getLogger().Warningf("vulnerability (`%s`) missing `published` date", cdxVulnerability.Id)
-	}
-
-	if cdxVulnerability.Created == "" {
-		getLogger().Warningf("vulnerability (`%s`) missing `created` date", cdxVulnerability.Id)
-	}
-
-	if len(cdxVulnerability.Ratings) == 0 {
-		getLogger().Warningf("vulnerability (`%s`) missing `ratings`", cdxVulnerability.Id)
-	}
-
-	// hash any component w/o a license using special key name
-	vulnInfo.Vulnerability = cdxVulnerability
-	if cdxVulnerability.BOMRef != nil {
-		vulnInfo.BOMRef = cdxVulnerability.BOMRef.String()
-	}
-	vulnInfo.Id = cdxVulnerability.Id
-
-	// Truncate dates from 2023-02-02T00:00:00.000Z to 2023-02-02
-	// Note: if validation errors are found by the "truncate" function,
-	// it will emit an error and return the original (failing) value
-	dateTime, _ := utils.TruncateTimeStampISO8601Date(cdxVulnerability.Created)
-	vulnInfo.Created = dateTime
-
-	dateTime, _ = utils.TruncateTimeStampISO8601Date(cdxVulnerability.Published)
-	vulnInfo.Published = dateTime
-
-	dateTime, _ = utils.TruncateTimeStampISO8601Date(cdxVulnerability.Updated)
-	vulnInfo.Updated = dateTime
-
-	dateTime, _ = utils.TruncateTimeStampISO8601Date(cdxVulnerability.Rejected)
-	vulnInfo.Rejected = dateTime
-
-	vulnInfo.Description = cdxVulnerability.Description
-
-	// Source object: retrieve report fields from nested objects
-	if cdxVulnerability.Source != nil {
-		source := *cdxVulnerability.Source
-		vulnInfo.Source = source
-		vulnInfo.SourceName = source.Name
-		vulnInfo.SourceUrl = source.Url
-	}
-
-	// TODO: replace empty Analysis values with "UNDEFINED"
-	vulnInfo.AnalysisState = cdxVulnerability.Analysis.State
-	if vulnInfo.AnalysisState == "" {
-		vulnInfo.AnalysisState = schema.VULN_ANALYSIS_STATE_EMPTY
-	}
-
-	vulnInfo.AnalysisJustification = cdxVulnerability.Analysis.Justification
-	if vulnInfo.AnalysisJustification == "" {
-		vulnInfo.AnalysisJustification = schema.VULN_ANALYSIS_STATE_EMPTY
-	}
-	vulnInfo.AnalysisResponse = cdxVulnerability.Analysis.Response
-	if len(vulnInfo.AnalysisResponse) == 0 {
-		vulnInfo.AnalysisResponse = []string{schema.VULN_ANALYSIS_STATE_EMPTY}
-	}
-
-	// Convert []int to []string for --where filter
-	// TODO see if we can eliminate this conversion and handle while preparing report data
-	// as this SHOULD appear there as []interface{}
-	if len(cdxVulnerability.Cwes) > 0 {
-		vulnInfo.CweIds = strings.Fields(strings.Trim(fmt.Sprint(cdxVulnerability.Cwes), "[]"))
-	}
-
-	// CVSS Score 	Qualitative Rating
-	// 0.0 	        None
-	// 0.1 – 3.9 	Low
-	// 4.0 – 6.9 	Medium
-	// 7.0 – 8.9 	High
-	// 9.0 – 10.0 	Critical
-
-	// TODO: if summary report, see if more than one severity can be shown without clogging up column data
-	numRatings := len(cdxVulnerability.Ratings)
-	if numRatings > 0 {
-		//var sourceMatch int
-		for _, rating := range cdxVulnerability.Ratings {
-			// defer to same source as the top-level vuln. declares
-			fSeverity := fmt.Sprintf("%s: %v (%s)", rating.Method, rating.Score, rating.Severity)
-			// give listing priority to ratings that matches top-level vuln. reporting source
-			if rating.Source.Name == cdxVulnerability.Source.Name {
-				// prepend to slice
-				vulnInfo.CvssSeverity = append([]string{fSeverity}, vulnInfo.CvssSeverity...)
-				continue
-			}
-			vulnInfo.CvssSeverity = append(vulnInfo.CvssSeverity, fSeverity)
-		}
-
-	} else {
-		// Set first entry to empty value (i.e., "none")
-		vulnInfo.CvssSeverity = append(vulnInfo.CvssSeverity, schema.VULN_RATING_EMPTY)
-	}
-
-	var match bool = true
-	if len(whereFilters) > 0 {
-		mapVulnInfo, _ := utils.MarshalStructToJsonMap(vulnInfo)
-		match, _ = whereFilterMatch(mapVulnInfo, whereFilters)
-	}
-
-	if match {
-		bom.VulnerabilityMap.Put(vulnInfo.Id, vulnInfo)
-
-		getLogger().Tracef("Put: %s (`%s`), `%s`)",
-			vulnInfo.Id, vulnInfo.Description, vulnInfo.BOMRef)
-	}
-
-	return
-}
-
 // NOTE: This list is NOT de-duplicated
 // TODO: Add a --no-title flag to skip title output
 func DisplayVulnListText(bom *schema.BOM, output io.Writer, flags utils.VulnerabilityCommandFlags) {

diff --git a/cmd/vulnerability_test.go b/cmd/vulnerability_test.go
@@ -232,7 +232,6 @@ func TestVulnListCdx13JSON(t *testing.T) {
 	testInfo.ResultExpectedLineCount = 185
 	result, _, _ := innerTestVulnList(t, testInfo, VULN_TEST_DEFAULT_FLAGS)
 	getLogger().Debugf("result:\n%s", result.String())
-	//fmt.Printf("result:\n%s", result.String())
 }
 
 // -------------------------------------------
@@ -254,7 +253,8 @@ func TestVulnListTextCdx14WhereClauseAndResultsByIdStartsWith(t *testing.T) {
 		nil)
 	testInfo.ResultLineContainsValues = TEST_OUTPUT_CONTAINS
 	testInfo.ResultLineContainsValuesAtLineNum = 2
-	innerTestVulnList(t, testInfo, VULN_TEST_DEFAULT_FLAGS)
+	result, _, _ := innerTestVulnList(t, testInfo, VULN_TEST_DEFAULT_FLAGS)
+	getLogger().Debugf("result:\n%s", result.String())
 }
 
 func TestVulnListTextCdx14WhereClauseDescContains(t *testing.T) {

diff --git a/schema/bom_hash.go b/schema/bom_hash.go
@@ -346,7 +346,7 @@ func (bom *BOM) HashVulnerability(cdxVulnerability CDXVulnerability, whereFilter
 		getLogger().Warningf("vulnerability (`%s`) missing `created` date", cdxVulnerability.Id)
 	}
 
-	if len(cdxVulnerability.Ratings) == 0 {
+	if cdxVulnerability.Ratings == nil || len(*cdxVulnerability.Ratings) == 0 {
 		getLogger().Warningf("vulnerability (`%s`) missing `ratings`", cdxVulnerability.Id)
 	}
 
@@ -383,24 +383,32 @@ func (bom *BOM) HashVulnerability(cdxVulnerability CDXVulnerability, whereFilter
 	}
 
 	// TODO: replace empty Analysis values with "UNDEFINED"
-	vulnInfo.AnalysisState = cdxVulnerability.Analysis.State
-	if vulnInfo.AnalysisState == "" {
-		vulnInfo.AnalysisState = VULN_ANALYSIS_STATE_EMPTY
-	}
+	if cdxVulnerability.Analysis != nil {
+		vulnInfo.AnalysisState = cdxVulnerability.Analysis.State
+		if vulnInfo.AnalysisState == "" {
+			vulnInfo.AnalysisState = VULN_ANALYSIS_STATE_EMPTY
+		}
+
+		vulnInfo.AnalysisJustification = cdxVulnerability.Analysis.Justification
+		if vulnInfo.AnalysisJustification == "" {
+			vulnInfo.AnalysisJustification = VULN_ANALYSIS_STATE_EMPTY
+		}
 
-	vulnInfo.AnalysisJustification = cdxVulnerability.Analysis.Justification
-	if vulnInfo.AnalysisJustification == "" {
+		vulnInfo.AnalysisResponse = *cdxVulnerability.Analysis.Response
+		if len(vulnInfo.AnalysisResponse) == 0 {
+			vulnInfo.AnalysisResponse = []string{VULN_ANALYSIS_STATE_EMPTY}
+		}
+	} else {
+		vulnInfo.AnalysisState = VULN_ANALYSIS_STATE_EMPTY
 		vulnInfo.AnalysisJustification = VULN_ANALYSIS_STATE_EMPTY
-	}
-	vulnInfo.AnalysisResponse = cdxVulnerability.Analysis.Response
-	if len(vulnInfo.AnalysisResponse) == 0 {
 		vulnInfo.AnalysisResponse = []string{VULN_ANALYSIS_STATE_EMPTY}
 	}
 
 	// Convert []int to []string for --where filter
 	// TODO see if we can eliminate this conversion and handle while preparing report data
 	// as this SHOULD appear there as []interface{}
-	if len(cdxVulnerability.Cwes) > 0 {
+	if cdxVulnerability.Cwes != nil && len(*cdxVulnerability.Cwes) > 0 {
+		// strip off slice/array brackets
 		vulnInfo.CweIds = strings.Fields(strings.Trim(fmt.Sprint(cdxVulnerability.Cwes), "[]"))
 	}
 
@@ -412,10 +420,9 @@ func (bom *BOM) HashVulnerability(cdxVulnerability CDXVulnerability, whereFilter
 	// 9.0 – 10.0 	Critical
 
 	// TODO: if summary report, see if more than one severity can be shown without clogging up column data
-	numRatings := len(cdxVulnerability.Ratings)
-	if numRatings > 0 {
+	if cdxVulnerability.Ratings != nil && len(*cdxVulnerability.Ratings) > 0 {
 		//var sourceMatch int
-		for _, rating := range cdxVulnerability.Ratings {
+		for _, rating := range *cdxVulnerability.Ratings {
 			// defer to same source as the top-level vuln. declares
 			fSeverity := fmt.Sprintf("%s: %v (%s)", rating.Method, rating.Score, rating.Severity)
 			// give listing priority to ratings that matches top-level vuln. reporting source
@@ -426,7 +433,6 @@ func (bom *BOM) HashVulnerability(cdxVulnerability CDXVulnerability, whereFilter
 			}
 			vulnInfo.CvssSeverity = append(vulnInfo.CvssSeverity, fSeverity)
 		}
-
 	} else {
 		// Set first entry to empty value (i.e., "none")
 		vulnInfo.CvssSeverity = append(vulnInfo.CvssSeverity, VULN_RATING_EMPTY)

diff --git a/schema/cyclonedx.go b/schema/cyclonedx.go
@@ -59,8 +59,8 @@ type CDXBom struct {
 // v1.3: added "licenses", "properties"
 // v1.5: added "lifecycles"
 type CDXMetadata struct {
-	Timestamp    string                      `json:"timestamp,omitempty"`
-	Tools        interface{}                 `json:"tools,omitempty"` // v1.2: added.v1.5: "tools" is now an interface{}
+	Timestamp    string                      `json:"timestamp,omitempty" scvs:"bom:core:timestamp"` // urn:owasp:scvs:bom:core:timestamp
+	Tools        interface{}                 `json:"tools,omitempty"`                               // v1.2: added.v1.5: "tools" is now an interface{}
 	Authors      *[]CDXOrganizationalContact `json:"authors,omitempty"`
 	Component    *CDXComponent               `json:"component,omitempty"`
 	Manufacturer *CDXOrganizationalEntity    `json:"manufacturer,omitempty"`
@@ -96,10 +96,10 @@ type CDXComponent struct {
 	Hashes             *[]CDXHash               `json:"hashes,omitempty"`
 	Licenses           *[]CDXLicenseChoice      `json:"licenses,omitempty"`
 	Copyright          string                   `json:"copyright,omitempty"`
-	Cpe                string                   `json:"cpe,omitempty"`      // See: https://nvd.nist.gov/products/cpe
-	Purl               string                   `json:"purl,omitempty"`     // See: https://github.com/package-url/purl-spec
-	Swid               *CDXSwid                 `json:"swid,omitempty"`     // See: https://www.iso.org/standard/65666.html
-	Pedigree           *CDXPedigree             `json:"pedigree,omitempty"` // anon. type
+	Cpe                string                   `json:"cpe,omitempty"`                                       // See: https://nvd.nist.gov/products/cpe
+	Purl               string                   `json:"purl,omitempty" scvs:"bom:resource:identifiers:purl"` // See: https://github.com/package-url/purl-spec
+	Swid               *CDXSwid                 `json:"swid,omitempty"`                                      // See: https://www.iso.org/standard/65666.html
+	Pedigree           *CDXPedigree             `json:"pedigree,omitempty"`                                  // anon. type
 	ExternalReferences *[]CDXExternalReference  `json:"externalReferences,omitempty"`
 	Components         *[]CDXComponent          `json:"components,omitempty"`
 	Evidence           *CDXComponentEvidence    `json:"evidence,omitempty"`                  // v1.3: added