From 6e05de38253bb861e4ecdf7345112959a81e2f41 Mon Sep 17 00:00:00 2001 From: Paul Rosca Date: Mon, 4 Mar 2024 16:31:48 +0200 Subject: [PATCH] feat: snyk enrich external refs --- go.mod | 2 +- go.sum | 4 +- lib/ecosystems/enrich_cyclonedx_test.go | 6 +- lib/ecosystems/enrich_spdx_test.go | 2 - lib/snyk/enrich_cyclonedx.go | 43 ++++++++- lib/snyk/enrich_spdx.go | 46 +++++++++- lib/snyk/enrich_test.go | 116 +++++++++++++++++++++++- lib/snyk/package.go | 60 +++++++++++- 8 files changed, 261 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index ca9edd7..9d69184 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/edoardottt/depsdev v0.0.3 github.com/google/uuid v1.3.0 github.com/jarcoal/httpmock v1.3.0 - github.com/package-url/packageurl-go v0.1.2-0.20230717211154-3587d8c2829e + github.com/package-url/packageurl-go v0.1.2 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/zerolog v1.29.1 github.com/spdx/tools-golang v0.5.2 diff --git a/go.sum b/go.sum index 2c7bfc7..73aac61 100644 --- a/go.sum +++ b/go.sum @@ -168,8 +168,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/package-url/packageurl-go v0.1.2-0.20230717211154-3587d8c2829e h1:h/TWC+mVfoco4qhPEsaxkdwymlTaDe/BGnzljU8SIPw= -github.com/package-url/packageurl-go v0.1.2-0.20230717211154-3587d8c2829e/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c= +github.com/package-url/packageurl-go v0.1.2 h1:0H2DQt6DHd/NeRlVwW4EZ4oEI6Bn40XlNPRqegcxuo4= +github.com/package-url/packageurl-go v0.1.2/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/lib/ecosystems/enrich_cyclonedx_test.go b/lib/ecosystems/enrich_cyclonedx_test.go index 797f844..000a07e 100644 --- a/lib/ecosystems/enrich_cyclonedx_test.go +++ b/lib/ecosystems/enrich_cyclonedx_test.go @@ -30,6 +30,8 @@ import ( "github.com/snyk/parlay/lib/sbom" ) +var logger = zerolog.Nop() + func TestEnrichSBOM_CycloneDX(t *testing.T) { httpmock.Activate() defer httpmock.DeactivateAndReset() @@ -65,7 +67,6 @@ func TestEnrichSBOM_CycloneDX(t *testing.T) { }, } doc := &sbom.SBOMDocument{BOM: bom} - logger := zerolog.Nop() EnrichSBOM(doc, &logger) @@ -113,7 +114,6 @@ func TestEnrichSBOM_CycloneDX_NestedComps(t *testing.T) { }, } doc := &sbom.SBOMDocument{BOM: bom} - logger := zerolog.Nop() EnrichSBOM(doc, &logger) @@ -146,7 +146,6 @@ func TestEnrichSBOMWithoutLicense(t *testing.T) { }, } doc := &sbom.SBOMDocument{BOM: bom} - logger := zerolog.Nop() EnrichSBOM(doc, &logger) @@ -192,7 +191,6 @@ func TestEnrichLicense(t *testing.T) { func TestEnrichBlankSBOM(t *testing.T) { bom := new(cdx.BOM) doc := &sbom.SBOMDocument{BOM: bom} - logger := zerolog.Nop() EnrichSBOM(doc, &logger) diff --git a/lib/ecosystems/enrich_spdx_test.go b/lib/ecosystems/enrich_spdx_test.go index 76f7497..114d4fc 100644 --- a/lib/ecosystems/enrich_spdx_test.go +++ b/lib/ecosystems/enrich_spdx_test.go @@ -21,7 +21,6 @@ import ( "testing" "github.com/jarcoal/httpmock" - "github.com/rs/zerolog" "github.com/spdx/tools-golang/spdx/v2/common" "github.com/spdx/tools-golang/spdx/v2/v2_3" "github.com/stretchr/testify/assert" @@ -61,7 +60,6 @@ func TestEnrichSBOM_SPDX(t *testing.T) { }, } doc := &sbom.SBOMDocument{BOM: bom} - logger := zerolog.Nop() EnrichSBOM(doc, &logger) diff --git a/lib/snyk/enrich_cyclonedx.go b/lib/snyk/enrich_cyclonedx.go index 3298c2f..ab6c02e 100644 --- a/lib/snyk/enrich_cyclonedx.go +++ b/lib/snyk/enrich_cyclonedx.go @@ -31,6 +31,45 @@ import ( "github.com/snyk/parlay/snyk/issues" ) +type cdxEnricher = func(*cdx.Component, *packageurl.PackageURL) + +var cdxEnrichers = []cdxEnricher{ + enrichCDXSnykAdvisorData, + enrichCDXSnykVulnerabilityDBData, +} + +func enrichCDXSnykVulnerabilityDBData(component *cdx.Component, purl *packageurl.PackageURL) { + url := SnykVulnURL(purl) + if url != "" { + ext := cdx.ExternalReference{ + URL: url, + Comment: "Snyk Vulnerability DB", + Type: "Other", + } + if component.ExternalReferences == nil { + component.ExternalReferences = &[]cdx.ExternalReference{ext} + } else { + *component.ExternalReferences = append(*component.ExternalReferences, ext) + } + } +} + +func enrichCDXSnykAdvisorData(component *cdx.Component, purl *packageurl.PackageURL) { + url := SnykAdvisorURL(purl) + if url != "" { + ext := cdx.ExternalReference{ + URL: url, + Comment: "Snyk Advisor", + Type: "Other", + } + if component.ExternalReferences == nil { + component.ExternalReferences = &[]cdx.ExternalReference{ext} + } else { + *component.ExternalReferences = append(*component.ExternalReferences, ext) + } + } +} + func enrichCycloneDX(bom *cdx.BOM, logger *zerolog.Logger) *cdx.BOM { auth, err := AuthFromToken(APIToken()) if err != nil { @@ -65,7 +104,9 @@ func enrichCycloneDX(bom *cdx.BOM, logger *zerolog.Logger) *cdx.BOM { Msg("Could not identify package") return } - + for _, enrichFunc := range cdxEnrichers { + enrichFunc(component, &purl) + } resp, err := GetPackageVulnerabilities(&purl, auth, orgID) if err != nil { l.Err(err). diff --git a/lib/snyk/enrich_spdx.go b/lib/snyk/enrich_spdx.go index 39eecd3..8b746f5 100644 --- a/lib/snyk/enrich_spdx.go +++ b/lib/snyk/enrich_spdx.go @@ -22,6 +22,7 @@ import ( "net/url" "sync" + "github.com/package-url/packageurl-go" "github.com/remeh/sizedwaitgroup" "github.com/rs/zerolog" "github.com/spdx/tools-golang/spdx" @@ -35,6 +36,47 @@ const ( snykVulnerabilityDB_URI = "https://security.snyk.io" ) +type spdxEnricher = func(*spdx_2_3.Package, *packageurl.PackageURL) + +var spdxEnrichers = []spdxEnricher{ + enrichSPDXSnykAdvisorData, + enrichSPDXSnykVulnerabilityDBData, +} + +func enrichSPDXSnykAdvisorData(component *spdx_2_3.Package, purl *packageurl.PackageURL) { + url := SnykAdvisorURL(purl) + if url != "" { + ext := &spdx_2_3.PackageExternalReference{ + Locator: url, + RefType: "advisory", + Category: "Other", + ExternalRefComment: "Snyk Advisor", + } + if component.PackageExternalReferences == nil { + component.PackageExternalReferences = []*spdx_2_3.PackageExternalReference{ext} + } else { + component.PackageExternalReferences = append(component.PackageExternalReferences, ext) + } + } +} + +func enrichSPDXSnykVulnerabilityDBData(component *spdx_2_3.Package, purl *packageurl.PackageURL) { + url := SnykVulnURL(purl) + if url != "" { + ext := &spdx_2_3.PackageExternalReference{ + Locator: url, + RefType: "url", + Category: "Other", + ExternalRefComment: "Snyk Vulnerability DB", + } + if component.PackageExternalReferences == nil { + component.PackageExternalReferences = []*spdx_2_3.PackageExternalReference{ext} + } else { + component.PackageExternalReferences = append(component.PackageExternalReferences, ext) + } + } +} + func enrichSPDX(bom *spdx.Document, logger *zerolog.Logger) *spdx.Document { auth, err := AuthFromToken(APIToken()) if err != nil { @@ -71,7 +113,9 @@ func enrichSPDX(bom *spdx.Document, logger *zerolog.Logger) *spdx.Document { l.Debug().Msg("Could not identify package") return } - + for _, enrichFn := range spdxEnrichers { + enrichFn(pkg, purl) + } resp, err := GetPackageVulnerabilities(purl, auth, orgID) if err != nil { l.Err(err). diff --git a/lib/snyk/enrich_test.go b/lib/snyk/enrich_test.go index 19a13a4..2b4ad72 100644 --- a/lib/snyk/enrich_test.go +++ b/lib/snyk/enrich_test.go @@ -13,6 +13,8 @@ import ( "github.com/snyk/parlay/lib/sbom" ) +var logger = zerolog.Nop() + func TestEnrichSBOM_CycloneDXWithVulnerabilities(t *testing.T) { teardown := setupTestEnv(t) defer teardown() @@ -28,7 +30,6 @@ func TestEnrichSBOM_CycloneDXWithVulnerabilities(t *testing.T) { }, } doc := &sbom.SBOMDocument{BOM: bom} - logger := zerolog.Nop() EnrichSBOM(doc, &logger) @@ -39,6 +40,72 @@ func TestEnrichSBOM_CycloneDXWithVulnerabilities(t *testing.T) { assert.Equal(t, "SNYK-PYTHON-NUMPY-73513", vuln.ID) } +func TestEnrichSBOM_CycloneDXExternalRefs(t *testing.T) { + teardown := setupTestEnv(t) + defer teardown() + + bom := &cdx.BOM{ + Components: &[]cdx.Component{ + { + BOMRef: "pkg:pypi/numpy@1.16.0", + Name: "numpy", + Version: "1.16.0", + PackageURL: "pkg:pypi/numpy@1.16.0", + }, + }, + } + doc := &sbom.SBOMDocument{BOM: bom} + + EnrichSBOM(doc, &logger) + + assert.NotNil(t, bom.Components) + refs := (*bom.Components)[0].ExternalReferences + assert.Len(t, *refs, 2) + + ref1 := (*refs)[0] + assert.Equal(t, "https://snyk.io/advisor/python/numpy", ref1.URL) + assert.Equal(t, "Snyk Advisor", ref1.Comment) + assert.Equal(t, cdx.ExternalReferenceType("Other"), ref1.Type) + + ref2 := (*refs)[1] + assert.Equal(t, "https://security.snyk.io/package/pip/numpy", ref2.URL) + assert.Equal(t, "Snyk Vulnerability DB", ref2.Comment) + assert.Equal(t, cdx.ExternalReferenceType("Other"), ref2.Type) +} + +func TestEnrichSBOM_CycloneDXExternalRefs_WithNamespace(t *testing.T) { + teardown := setupTestEnv(t) + defer teardown() + + bom := &cdx.BOM{ + Components: &[]cdx.Component{ + { + BOMRef: "@emotion/react@11.11.3", + Name: "react", + Version: "11.11.3", + PackageURL: "pkg:npm/%40emotion/react@11.11.3", + }, + }, + } + doc := &sbom.SBOMDocument{BOM: bom} + + EnrichSBOM(doc, &logger) + + assert.NotNil(t, bom.Components) + refs := (*bom.Components)[0].ExternalReferences + assert.Len(t, *refs, 2) + + ref1 := (*refs)[0] + assert.Equal(t, "https://snyk.io/advisor/npm-package/@emotion/react", ref1.URL) + assert.Equal(t, "Snyk Advisor", ref1.Comment) + assert.Equal(t, cdx.ExternalReferenceType("Other"), ref1.Type) + + ref2 := (*refs)[1] + assert.Equal(t, "https://security.snyk.io/package/npm/@emotion%2Freact", ref2.URL) + assert.Equal(t, "Snyk Vulnerability DB", ref2.Comment) + assert.Equal(t, cdx.ExternalReferenceType("Other"), ref2.Type) +} + func TestEnrichSBOM_CycloneDXWithVulnerabilities_NestedComponents(t *testing.T) { teardown := setupTestEnv(t) defer teardown() @@ -62,7 +129,6 @@ func TestEnrichSBOM_CycloneDXWithVulnerabilities_NestedComponents(t *testing.T) }, } doc := &sbom.SBOMDocument{BOM: bom} - logger := zerolog.Nop() EnrichSBOM(doc, &logger) @@ -85,7 +151,6 @@ func TestEnrichSBOM_CycloneDXWithoutVulnerabilities(t *testing.T) { }, } doc := &sbom.SBOMDocument{BOM: bom} - logger := zerolog.Nop() EnrichSBOM(doc, &logger) @@ -113,17 +178,58 @@ func TestEnrichSBOM_SPDXWithVulnerabilities(t *testing.T) { }, } doc := &sbom.SBOMDocument{BOM: bom} - logger := zerolog.Nop() EnrichSBOM(doc, &logger) - vulnRef := bom.Packages[0].PackageExternalReferences[1] + vulnRef := bom.Packages[0].PackageExternalReferences[3] assert.Equal(t, "SECURITY", vulnRef.Category) assert.Equal(t, "advisory", vulnRef.RefType) assert.Equal(t, "https://security.snyk.io/vuln/SNYK-PYTHON-NUMPY-73513", vulnRef.Locator) assert.Equal(t, "Arbitrary Code Execution", vulnRef.ExternalRefComment) } +func TestEnrichSBOM_SPDXExternalRefs(t *testing.T) { + teardown := setupTestEnv(t) + defer teardown() + + bom := &spdx_2_3.Document{ + Packages: []*spdx_2_3.Package{ + { + PackageSPDXIdentifier: "pkg:pypi/numpy@1.16.0", + PackageName: "numpy", + PackageVersion: "1.16.0", + PackageExternalReferences: []*spdx_2_3.PackageExternalReference{ + { + Category: spdx.CategoryPackageManager, + RefType: "purl", + Locator: "pkg:pypi/numpy@1.16.0", + }, + }, + }, + }, + } + + doc := &sbom.SBOMDocument{BOM: bom} + + EnrichSBOM(doc, &logger) + + assert.NotNil(t, bom.Packages) + refs := (*bom.Packages[0]).PackageExternalReferences + assert.Len(t, refs, 4) + + ref1 := refs[1] + assert.Equal(t, "https://snyk.io/advisor/python/numpy", ref1.Locator) + assert.Equal(t, "Snyk Advisor", ref1.ExternalRefComment) + assert.Equal(t, "advisory", ref1.RefType) + assert.Equal(t, "Other", ref1.Category) + + ref2 := refs[2] + assert.Equal(t, "https://security.snyk.io/package/pip/numpy", ref2.Locator) + assert.Equal(t, "Snyk Vulnerability DB", ref2.ExternalRefComment) + assert.Equal(t, "url", ref2.RefType) + assert.Equal(t, "Other", ref2.Category) +} + func setupTestEnv(t *testing.T) func() { t.Helper() diff --git a/lib/snyk/package.go b/lib/snyk/package.go index ea10e0f..2f3403b 100644 --- a/lib/snyk/package.go +++ b/lib/snyk/package.go @@ -26,8 +26,64 @@ import ( "github.com/snyk/parlay/snyk/issues" ) -const snykServer = "https://api.snyk.io/rest" -const version = "2023-04-28" +const ( + snykServer = "https://api.snyk.io/rest" + version = "2023-04-28" + snykAdvisorServer = "https://snyk.io/advisor" + snykVulnDBServer = "https://security.snyk.io/package" +) + +func purlToSnykAdvisor(purl *packageurl.PackageURL) string { + return map[string]string{ + packageurl.TypeNPM: "npm-package", + packageurl.TypePyPi: "python", + packageurl.TypeGolang: "golang", + packageurl.TypeDocker: "docker", + }[purl.Type] +} + +func SnykAdvisorURL(purl *packageurl.PackageURL) string { + ecosystem := purlToSnykAdvisor(purl) + if ecosystem == "" { + return "" + } + url := snykAdvisorServer + "/" + ecosystem + "/" + if purl.Namespace != "" { + url += purl.Namespace + "/" + } + url += purl.Name + return url +} + +func purlToSnykVulnDB(purl *packageurl.PackageURL) string { + return map[string]string{ + packageurl.TypeCargo: "cargo", + packageurl.TypeCocoapods: "cocoapods", + packageurl.TypeComposer: "composer", + packageurl.TypeGolang: "golang", + packageurl.TypeHex: "hex", + packageurl.TypeMaven: "maven", + packageurl.TypeNPM: "npm", + packageurl.TypeNuget: "nuget", + packageurl.TypePyPi: "pip", + packageurl.TypePub: "pub", + packageurl.TypeGem: "rubygems", + packageurl.TypeSwift: "swift", + }[purl.Type] +} + +func SnykVulnURL(purl *packageurl.PackageURL) string { + ecosystem := purlToSnykVulnDB(purl) + if ecosystem == "" { + return "" + } + url := snykVulnDBServer + "/" + ecosystem + "/" + if purl.Namespace != "" { + url += purl.Namespace + "%2F" + } + url += purl.Name + return url +} func GetPackageVulnerabilities(purl *packageurl.PackageURL, auth *securityprovider.SecurityProviderApiKey, orgID *uuid.UUID) (*issues.FetchIssuesPerPurlResponse, error) { client, err := issues.NewClientWithResponses(snykServer, issues.WithRequestEditorFn(auth.Intercept))