From 3dc91843550dfc13f9f31558bbb3cf0e645d8c17 Mon Sep 17 00:00:00 2001 From: Vivek Kumar Sahu Date: Tue, 1 Oct 2024 23:50:38 +0530 Subject: [PATCH] added scvs tests Signed-off-by: Vivek Kumar Sahu --- pkg/sbom/cdx.go | 3 +- pkg/scvs/features.go | 4 +- pkg/scvs/features_test.go | 379 ++++++++++++++++++++++++++++++++++++++ pkg/scvs/scvs.go | 14 +- pkg/scvs/scvsCriteria.go | 2 +- 5 files changed, 390 insertions(+), 12 deletions(-) create mode 100644 pkg/scvs/features_test.go diff --git a/pkg/sbom/cdx.go b/pkg/sbom/cdx.go index 506a3952..b9694609 100644 --- a/pkg/sbom/cdx.go +++ b/pkg/sbom/cdx.go @@ -215,7 +215,7 @@ func (c *CdxDoc) parseSpec() { func (c *CdxDoc) parseSignature() { if c.doc.Declarations == nil { - fmt.Println("Declaratic.doc.Declarationsons field is nil ") + fmt.Println("c.doc.Declarations field is nil ") return } @@ -295,6 +295,7 @@ func copyC(cdxc *cydx.Component, c *CdxDoc) *Component { nc.Name = cdxc.Name nc.purpose = string(cdxc.Type) nc.isReqFieldsPresent = c.pkgRequiredFields(cdxc) + nc.CopyRight = cdxc.Copyright ncpe := cpe.NewCPE(cdxc.CPE) if ncpe.Valid() { diff --git a/pkg/scvs/features.go b/pkg/scvs/features.go index 5cc858f6..e443517f 100644 --- a/pkg/scvs/features.go +++ b/pkg/scvs/features.go @@ -59,7 +59,7 @@ func scvsSBOMAutomationCreationCheck(d sbom.Document, c *scvsCheck) scvsScore { func scvsSBOMUniqIDCheck(d sbom.Document, c *scvsCheck) scvsScore { s := newScoreFromScvsCheck(c) - if IsSBOMHasUniqID(d, s) { + if IsSBOMHasGlobalUniqID(d, s) { s.setL3Score(green + bold + "✓" + reset) s.setL2Score(green + bold + "✓" + reset) s.setL1Score(green + bold + "✓" + reset) @@ -115,7 +115,7 @@ func scvsSBOMSigVerified(d sbom.Document, c *scvsCheck) scvsScore { func scvsSBOMTimestampCheck(d sbom.Document, c *scvsCheck) scvsScore { s := newScoreFromScvsCheck(c) - if IsSBOMTimestamped(d, s) { + if DoesSBOMHasTimestamp(d, s) { s.setL3Score(green + bold + "✓" + reset) s.setL2Score(green + bold + "✓" + reset) s.setL1Score(green + bold + "✓" + reset) diff --git a/pkg/scvs/features_test.go b/pkg/scvs/features_test.go new file mode 100644 index 00000000..4b293a36 --- /dev/null +++ b/pkg/scvs/features_test.go @@ -0,0 +1,379 @@ +package scvs + +import ( + "testing" + + "github.com/interlynk-io/sbomqs/pkg/cpe" + "github.com/interlynk-io/sbomqs/pkg/purl" + "github.com/interlynk-io/sbomqs/pkg/sbom" + "gotest.tools/assert" +) + +func cdxDocWithTool() (d sbom.Document, c *scvsCheck) { + p := scvsCheck{ + Key: "SBOM creation is automated and reproducible", + } + tools := []sbom.GetTool{} + tool := sbom.Tool{} + tool.Name = "sbom-tool" + tool.Version = "9.1.2" + tools = append(tools, tool) + + doc := sbom.CdxDoc{ + CdxTools: tools, + } + return doc, &p +} + +func cdxToolWithoutVersion() (d sbom.Document, c *scvsCheck) { + p := scvsCheck{ + Key: "SBOM creation is automated and reproducible", + } + tools := []sbom.GetTool{} + tool := sbom.Tool{} + tool.Name = "sbom-tool" + tools = append(tools, tool) + + doc := sbom.CdxDoc{ + CdxTools: tools, + } + return doc, &p +} + +func cdxToolWithoutName() (d sbom.Document, c *scvsCheck) { + p := scvsCheck{ + Key: "SBOM creation is automated and reproducible", + } + tools := []sbom.GetTool{} + tool := sbom.Tool{} + tool.Name = "sbom-tool" + tools = append(tools, tool) + + doc := sbom.CdxDoc{ + CdxTools: tools, + } + return doc, &p +} + +type desired struct { + feature string + l1score string + l2score string + l3score string + desc string +} + +func TestSBOMAutomationCreation(t *testing.T) { + testCases := []struct { + name string + actual scvsScore + expected desired + }{ + { + name: "cdxSBOMWithToolNameAndVersion", + actual: scvsSBOMAutomationCreationCheck(cdxDocWithTool()), + expected: desired{ + feature: "SBOM creation is automated and reproducible", + l2score: green + bold + "✓" + reset, + l3score: green + bold + "✓" + reset, + desc: "SBOM creation is automated", + }, + }, + { + name: "cdxSBOMWithToolWithoutVersion", + actual: scvsSBOMAutomationCreationCheck(cdxToolWithoutVersion()), + expected: desired{ + feature: "SBOM creation is automated and reproducible", + // l1score: 10.0, + l2score: red + bold + "✗" + reset, + l3score: red + bold + "✗" + reset, + desc: "SBOM creation is non-automated", + }, + }, + { + name: "cdxSBOMWithToolWithoutName", + actual: scvsSBOMAutomationCreationCheck(cdxToolWithoutName()), + expected: desired{ + feature: "SBOM creation is automated and reproducible", + l2score: red + bold + "✗" + reset, + l3score: red + bold + "✗" + reset, + desc: "SBOM creation is non-automated", + }, + }, + } + for _, test := range testCases { + assert.Equal(t, test.expected.feature, test.actual.feature, "Score mismatch for %s", test.name) + assert.Equal(t, test.expected.l1score, test.actual.l1Score, "Key mismatch for %s", test.name) + assert.Equal(t, test.expected.l2score, test.actual.l2Score, "ID mismatch for %s", test.name) + assert.Equal(t, test.expected.l3score, test.actual.l3Score, "Result mismatch for %s", test.name) + assert.Equal(t, test.expected.desc, test.actual.descr, "Maturity mismatch for %s", test.name) + } +} + +func spdxSbomWithGlobalUniqID() (d sbom.Document, c *scvsCheck) { + p := scvsCheck{ + Key: "Each SBOM has global unique ID", + } + namespace := "https://anchore.com/syft/file/sbomqs-linux-amd64-ef8c4621-f421-44cd-8267-749e6cf75626" + + spec := sbom.NewSpec() + spec.UniqID = namespace + + doc := sbom.SpdxDoc{ + SpdxSpec: spec, + } + return doc, &p +} + +func cdxSbomWithGlobalUniqID() (d sbom.Document, c *scvsCheck) { + p := scvsCheck{ + Key: "Each SBOM has global unique ID", + } + serialNumber := "urn:uuid:59449365-e065-4fbe-aec6-6b2f852e8147" + spec := sbom.NewSpec() + spec.UniqID = serialNumber + + doc := sbom.CdxDoc{ + CdxSpec: spec, + } + return doc, &p +} + +func TestSBOMWithGlobalUniqIDs(t *testing.T) { + testCases := []struct { + name string + actual scvsScore + expected desired + }{ + { + name: "spdxSBOMWithGlobalUniqID", + actual: scvsSBOMUniqIDCheck(spdxSbomWithGlobalUniqID()), + expected: desired{ + feature: "Each SBOM has global unique ID", + l1score: green + bold + "✓" + reset, + l2score: green + bold + "✓" + reset, + l3score: green + bold + "✓" + reset, + desc: "SBOM have global uniq ID", + }, + }, + { + name: "cdxSBOMWithGlobalUniqID", + actual: scvsSBOMUniqIDCheck(cdxSbomWithGlobalUniqID()), + expected: desired{ + feature: "Each SBOM has global unique ID", + l1score: green + bold + "✓" + reset, + l2score: green + bold + "✓" + reset, + l3score: green + bold + "✓" + reset, + desc: "SBOM have global uniq ID", + }, + }, + } + for _, test := range testCases { + assert.Equal(t, test.expected.feature, test.actual.feature, "Score mismatch for %s", test.name) + assert.Equal(t, test.expected.l1score, test.actual.l1Score, "Key mismatch for %s", test.name) + assert.Equal(t, test.expected.l2score, test.actual.l2Score, "ID mismatch for %s", test.name) + assert.Equal(t, test.expected.l3score, test.actual.l3Score, "Result mismatch for %s", test.name) + assert.Equal(t, test.expected.desc, test.actual.descr, "Maturity mismatch for %s", test.name) + } +} + +func sbomWithTimestamp() (d sbom.Document, c *scvsCheck) { + p := scvsCheck{ + Key: "SBOM is timestamped", + } + + s := sbom.NewSpec() + s.CreationTimestamp = "2020-04-13T20:20:39+00:00" + doc := sbom.CdxDoc{ + CdxSpec: s, + } + return doc, &p +} + +func TestSBOMHasTimestamp(t *testing.T) { + testCases := []struct { + name string + actual scvsScore + expected desired + }{ + { + name: "sbomWithTimestamp", + actual: scvsSBOMTimestampCheck(sbomWithTimestamp()), + expected: desired{ + feature: "SBOM is timestamped", + l1score: green + bold + "✓" + reset, + l2score: green + bold + "✓" + reset, + l3score: green + bold + "✓" + reset, + desc: "SBOM is timestamped", + }, + }, + } + for _, test := range testCases { + assert.Equal(t, test.expected.feature, test.actual.feature, "Score mismatch for %s", test.name) + assert.Equal(t, test.expected.l1score, test.actual.l1Score, "Key mismatch for %s", test.name) + assert.Equal(t, test.expected.l2score, test.actual.l2Score, "ID mismatch for %s", test.name) + assert.Equal(t, test.expected.l3score, test.actual.l3Score, "Result mismatch for %s", test.name) + assert.Equal(t, test.expected.desc, test.actual.descr, "Maturity mismatch for %s", test.name) + } +} + +// func sbomWithTimestamp() (d sbom.Document, c *scvsCheck) { +// p := scvsCheck{ +// Key: "SBOM is timestamped", +// } + +// s := sbom.NewSpec() +// s.CreationTimestamp = "2020-04-13T20:20:39+00:00" +// doc := sbom.CdxDoc{ +// CdxSpec: s, +// } +// return doc, &p +// } + +func docWithPrimaryComponent() (d sbom.Document, c *scvsCheck) { + p := scvsCheck{ + Key: "SBOM contains metadata about the asset or software the SBOM describes", + } + + primary := sbom.PrimaryComp{} + primary.Present = true + primary.ID = "git@github.com:interlynk/sbomqs.git" + + doc := sbom.CdxDoc{ + PrimaryComponent: primary, + } + return doc, &p +} + +func TestSBOMWithPrimaryComp(t *testing.T) { + testCases := []struct { + name string + actual scvsScore + expected desired + }{ + { + name: "sbomWithPrimaryComp", + actual: scvsSBOMPrimaryCompCheck(docWithPrimaryComponent()), + expected: desired{ + feature: "SBOM contains metadata about the asset or software the SBOM describes", + l2score: green + bold + "✓" + reset, + l3score: green + bold + "✓" + reset, + desc: "SBOM have primary comp", + }, + }, + } + for _, test := range testCases { + assert.Equal(t, test.expected.feature, test.actual.feature, "Score mismatch for %s", test.name) + // assert.Equal(t, test.expected.l1score, test.actual.l1Score, "Key mismatch for %s", test.name) + assert.Equal(t, test.expected.l2score, test.actual.l2Score, "ID mismatch for %s", test.name) + assert.Equal(t, test.expected.l3score, test.actual.l3Score, "Result mismatch for %s", test.name) + assert.Equal(t, test.expected.desc, test.actual.descr, "Maturity mismatch for %s", test.name) + } +} + +func docWithIdentityCheck() (d sbom.Document, c *scvsCheck) { + p := scvsCheck{ + Key: "SBOM contains metadata about the asset or software the SBOM describes", + } + + primary := sbom.PrimaryComp{} + primary.Present = true + primary.ID = "git@github.com:interlynk/sbomqs.git" + + doc := sbom.CdxDoc{ + PrimaryComponent: primary, + } + return doc, &p +} + +type externalRef struct { + refCategory string + refType string + refLocator string +} + +func docWithCPE() (d sbom.Document, c *scvsCheck) { + p := scvsCheck{ + Key: "Component identifiers are derived from their native ecosystems (if applicable)", + } + + urls := []cpe.CPE{} + comps := []sbom.GetComponent{} + comp := sbom.NewComponent() + + comp.Name = "glibc" + comp.Spdxid = "SPDXRef-git-github.com-glibc-afb1ddc0824ce0052d72ac0d6917f144a1207424" + + ext := externalRef{ + // refCategory: "SECURITY", + // refType: "cpe23Type", + refLocator: "cpe:2.3:a:pivotal_software:spring_framework:4.1.0:*:*:*:*:*:*:*", + } + + prl := cpe.NewCPE(ext.refLocator) + urls = append(urls, prl) + comp.Cpes = urls + + comps = append(comps, comp) + + doc := sbom.SpdxDoc{ + Comps: comps, + } + return doc, &p +} + +func docWithPurl() (d sbom.Document, c *scvsCheck) { + p := scvsCheck{ + Key: "Component point of origin is identified in a consistent, machine readable format (e.g. PURL)", + } + comps := []sbom.GetComponent{} + + comp := sbom.NewComponent() + comp.Name = "acme" + PackageURL := "pkg:npm/acme/component@1.0.0" + + prl := purl.NewPURL(PackageURL) + comp.Purls = []purl.PURL{prl} + comps = append(comps, comp) + + doc := sbom.CdxDoc{ + Comps: comps, + } + return doc, &p +} + +func TestComponentWithID(t *testing.T) { + testCases := []struct { + name string + actual scvsScore + expected desired + }{ + { + name: "sbomWithCPE", + actual: scvsCompHasIdentityIDCheck(docWithCPE()), + expected: desired{ + feature: "Component identifiers are derived from their native ecosystems (if applicable)", + l1score: green + bold + "✓" + reset, + l2score: green + bold + "✓" + reset, + l3score: green + bold + "✓" + reset, + desc: "1/1 comp have Identity ID's", + }, + }, + { + name: "sbomWithPurl", + actual: scvsCompHasOriginIDCheck(docWithPurl()), + expected: desired{ + feature: "Component point of origin is identified in a consistent, machine readable format (e.g. PURL)", + l3score: green + bold + "✓" + reset, + desc: "1/1 comp have Origin ID's", + }, + }, + } + for _, test := range testCases { + assert.Equal(t, test.expected.feature, test.actual.feature, "Score mismatch for %s", test.name) + assert.Equal(t, test.expected.l1score, test.actual.l1Score, "Key mismatch for %s", test.name) + assert.Equal(t, test.expected.l2score, test.actual.l2Score, "ID mismatch for %s", test.name) + assert.Equal(t, test.expected.l3score, test.actual.l3Score, "Result mismatch for %s", test.name) + assert.Equal(t, test.expected.desc, test.actual.descr, "Maturity mismatch for %s", test.name) + } +} diff --git a/pkg/scvs/scvs.go b/pkg/scvs/scvs.go index 9ade77f2..84100181 100644 --- a/pkg/scvs/scvs.go +++ b/pkg/scvs/scvs.go @@ -43,30 +43,28 @@ func IsSBOMMachineReadable(d sbom.Document, s *scvsScore) bool { // 2.3 SBOM creation is automated and reproducible(L2, L3) func IsSBOMCreationAutomated(d sbom.Document, s *scvsScore) bool { - noOfTools := len(d.Tools()) if tools := d.Tools(); tools != nil { for _, tool := range tools { name := tool.GetName() version := tool.GetVersion() if name != "" && version != "" { - s.setDesc(fmt.Sprintf("SBOM has %d authors", noOfTools)) + s.setDesc(fmt.Sprintf("SBOM creation is automated")) return true } } } - - s.setDesc(fmt.Sprintf("SBOM has %d authors", noOfTools)) + s.setDesc(fmt.Sprintf("SBOM creation is non-automated")) return false } // 2.3 Each SBOM has a unique identifier(L1, L2, L3) -func IsSBOMHasUniqID(d sbom.Document, s *scvsScore) bool { +func IsSBOMHasGlobalUniqID(d sbom.Document, s *scvsScore) bool { if ns := d.Spec().GetUniqID(); ns != "" { - s.setDesc("SBOM has uniq ID") + s.setDesc("SBOM have global uniq ID") return true } - s.setDesc("SBOM doesn't has uniq ID") + s.setDesc("SBOM doesn't have global uniq ID") return false } @@ -149,7 +147,7 @@ func IsSBOMSignatureVerified(d sbom.Document, s *scvsScore) bool { } // 2.7 SBOM is timestamped(L1, L2, L3) -func IsSBOMTimestamped(d sbom.Document, s *scvsScore) bool { +func DoesSBOMHasTimestamp(d sbom.Document, s *scvsScore) bool { if result := d.Spec().GetCreationTimestamp(); result != "" { _, err := time.Parse(time.RFC3339, result) if err != nil { diff --git a/pkg/scvs/scvsCriteria.go b/pkg/scvs/scvsCriteria.go index 26fc4d29..9fa6a74d 100644 --- a/pkg/scvs/scvsCriteria.go +++ b/pkg/scvs/scvsCriteria.go @@ -25,7 +25,7 @@ var scvsChecks = []scvsCheck{ // scvs {"A structured, machine readable software bill of materials (SBOM) format is present", scvsSBOMMachineReadableCheck}, {"SBOM creation is automated and reproducible", scvsSBOMAutomationCreationCheck}, - {"Each SBOM has a unique identifier", scvsSBOMUniqIDCheck}, + {"Each SBOM has global unique ID", scvsSBOMUniqIDCheck}, {"SBOM has been signed by publisher, supplier, or certifying authority", scvsSBOMSigcheck}, {"SBOM signature verification exists", scvsSBOMSigCorrectnessCheck}, {"SBOM signature verification is performed", scvsSBOMSigVerified},