diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..f6c104e --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,46 @@ +name: Unit Tests and Fuzzing Tests + +on: + pull_request: + branches: + - main + + push: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.21.0 + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: 16.x + + - name: Build bootstrap + run: make bootstrap + + - name: Build upstream-protobom + run: make upstream-protobom + + - name: Copy goreleaser.yaml + run: cp .goreleaser.yaml ./.tmp/goreleaser.yaml + + - name: Build binary + run: make binary + + - name: Run unit tests + run: make unittest + + # - name: Run fuzzing tests + # run: make fuzztest \ No newline at end of file diff --git a/Makefile b/Makefile index fa4f7dd..80cecec 100644 --- a/Makefile +++ b/Makefile @@ -45,5 +45,8 @@ upstream-protobom: ## Upstream protobom library .PHONY: unittest unittest: ## Run unittests - go test -count=1 -v ./... - \ No newline at end of file + go test -count=1 -v ./... -run='^(Test[^F])' -args -max_test_count=100 -test_spdx_keyvalue=false -test_spdx_json=true -test_cdx_json=true + +.PHONY: fuzztest +fuzztest: ## Run fuzzing tests + go test --fuzz=Fuzz pkg/convert/convert_fuzzing_test.go -args -seed_input=fuzz_seed_spdx.json \ No newline at end of file diff --git a/pkg/convert/README.md b/pkg/convert/README.md new file mode 100644 index 0000000..ec89387 --- /dev/null +++ b/pkg/convert/README.md @@ -0,0 +1,38 @@ +# Unit and Fuzz Testing for SBOM Conversion + +This repository contains a Go unit test file aimed at evaluating the accuracy and integrity of the Software Bill of Materials (SBOM) conversion process. It ensures that the SBOM conversion maintains specific properties and is error-free. + +## How it Functions + +This unit test file operates through the following steps to validate SBOM conversion: + +1. **Unit Test**: + - Automatically Download our shared [SBOM dataset](https://drive.google.com/file/d/1LgGlq3g_H02mhzkc94cUd0zzxy0JhFim/view?usp=sharing), comprising 10,494 SBOM files in both SPDX and CycloneDX formats. + - Utilize the sbom-convert tool for converting SBOMs from one format to another. + - Verify that the sbom-convert tool does not encounter any errors. + - Compare the counts of PURLs in the original and converted SBOMs. + - Compare the counts of licenses in the original and converted SBOMs. + +2. **Fuzzing Test**: + - The fuzzer takes seed inputs and generates new inputs through mutations. + - It then checks if the binary fails when exposed to certain corner cases. + +## Getting Started + +### Running the Unit Test + +1. **Run the Unit Test**: Execute the following command in your project's root directory: + + ```bash + make unittest + ``` + +2. **Run the Fuzzing Test**: Execute the following command in your project's root directory: + + ```bash + make fuzztest + ``` + +## Contribution + +The current unit tests primarily focus on conversion, PURL counts, and license counts. Additional unit tests will be added in the future. If you have suggestions, improvements, or bug fixes, please don't hesitate to reach out and contribute to the project. Your input is highly valued. \ No newline at end of file diff --git a/pkg/convert/convert_fuzzing_test.go b/pkg/convert/convert_fuzzing_test.go new file mode 100644 index 0000000..00327f0 --- /dev/null +++ b/pkg/convert/convert_fuzzing_test.go @@ -0,0 +1,80 @@ +package convert + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/bom-squad/protobom/pkg/reader" +) + +var ( + seed_input string +) + +func init() { + flag.StringVar(&seed_input, "seed_input", "fuzz_seed_spdx.json", "Seed input file path") +} + +type ReadWriteSeeker struct { + *os.File +} + +func (rws *ReadWriteSeeker) Close() error { + return rws.File.Close() +} + +func WriteStringToTempFile(content string) (io.ReadSeekCloser, error) { + // Create a temporary file + tempFile, err := os.CreateTemp("", "tempfile") + if err != nil { + return nil, err + } + + // Write the content to the temporary file + _, err = tempFile.WriteString(content) + if err != nil { + return nil, err + } + + // Close the file to make sure all data is flushed to disk + err = tempFile.Close() + if err != nil { + return nil, err + } + + // Reopen the temporary file for reading and seeking + file, err := os.OpenFile(tempFile.Name(), os.O_RDWR, 0644) + if err != nil { + return nil, err + } + + return &ReadWriteSeeker{file}, nil +} + +func ParseStreamWrapper(content string) { + t := io.NopCloser(strings.NewReader(content)) + r := reader.New() + t, _ = WriteStringToTempFile(content) + t2 := t.(io.ReadSeekCloser) + r.ParseStream(t2) +} + +func FuzzParseStream(f *testing.F) { + absPath, _ := filepath.Abs(seed_input) + fmt.Println(absPath) + content, err := os.ReadFile(absPath) + if err != nil { + log.Fatal(err) + } + f.Add(string(content)) + + f.Fuzz(func(t *testing.T, orig string) { + ParseStreamWrapper(orig) + }) +} \ No newline at end of file diff --git a/pkg/convert/convert_unit_test.go b/pkg/convert/convert_unit_test.go new file mode 100644 index 0000000..595dc12 --- /dev/null +++ b/pkg/convert/convert_unit_test.go @@ -0,0 +1,312 @@ +package convert + +import ( + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" +) + +var ( + max_test_count int + test_spdx_json bool + test_spdx_keyvalue bool + test_cdx_json bool +) + +func init() { + flag.IntVar(&max_test_count, "max_test_count", 100, "Maximum number of test files") + flag.BoolVar(&test_spdx_json, "test_spdx_json", true, "Test SPDX json files") + flag.BoolVar(&test_spdx_keyvalue, "test_spdx_keyvalue", false, "Test SPDX key-value files") + flag.BoolVar(&test_cdx_json, "test_cdx_json", true, "Test CycloneDX json files") +} + +func ExtractLicenses(input string) []string { + input = strings.NewReplacer("(", "", ")", "").Replace(input) + licenses := regexp.MustCompile(`\s*(OR|AND)\s*`).Split(input, -1) + var result []string + for _, license := range licenses { + if license != "" { + result = append(result, license) + } + } + return result +} + +func FilterLicenses(licenses []string) []string { + var result []string + for _, license := range licenses { + switch license { + case "", "None", "NONE", "NOASSERTION": + continue + default: + result = append(result, license) + } + } + return result +} + +func GetPurlsCount(json_str string) int { + var data map[string]interface{} + err := json.Unmarshal([]byte(json_str), &data) + if err != nil { + fmt.Println("Error:", err) + } + + purls := []string{} + + if _, exists := data["spdxVersion"]; exists { + packages := data["packages"].([]interface{}) + for _, pkg := range packages { + packageMap := pkg.(map[string]interface{}) + if externalRefs, ok := packageMap["externalRefs"].([]interface{}); ok { + for _, ref := range externalRefs { + refMap := ref.(map[string]interface{}) + if refType, ok := refMap["referenceType"].(string); ok && refType == "purl" { + if refLocator, ok := refMap["referenceLocator"].(string); ok { + purls = append(purls, refLocator) + } + } + } + } + } + } else { + components := data["components"].([]interface{}) + for _, component := range components { + componentMap := component.(map[string]interface{}) + if purl, ok := componentMap["purl"].(string); ok { + purls = append(purls, purl) + } + + if subcomponents, ok := componentMap["components"].([]interface{}); ok { + for _, subcomponent := range subcomponents { + subcomponentMap := subcomponent.(map[string]interface{}) + if purl, ok := subcomponentMap["purl"].(string); ok { + purls = append(purls, purl) + } + } + } + } + } + return len(purls) +} + +func GetLicensesCount(json_str string) int { + var data map[string]interface{} + err := json.Unmarshal([]byte(json_str), &data) + if err != nil { + fmt.Println("Error:", err) + } + + licenses := map[string]struct{}{} + + if _, exists := data["spdxVersion"]; exists { + packages, _ := data["packages"].([]interface{}) + for _, pkg := range packages { + packageMap, _ := pkg.(map[string]interface{}) + + if licenseConcluded, ok := packageMap["licenseConcluded"].(string); ok { + concludedLicenses := ExtractLicenses(licenseConcluded) + for _, lic := range FilterLicenses(concludedLicenses) { + licenses[lic] = struct{}{} + } + } + + // if licenseDeclared, ok := packageMap["licenseDeclared"].(string); ok { + // declaredLicenses := ExtractLicenses(licenseDeclared) + // for _, lic := range FilterLicenses(declaredLicenses) { + // licenses[lic] = struct{}{} + // } + // } + } + } else { + components, _ := data["components"].([]interface{}) + for _, component := range components { + componentMap, _ := component.(map[string]interface{}) + if licensesList, ok := componentMap["licenses"].([]interface{}); ok { + for _, license := range licensesList { + if licenseMap, ok := license.(map[string]interface{}); ok { + if licenseIDMap, ok := licenseMap["license"].(map[string]interface{}); ok { + if licenseID, ok := licenseIDMap["id"].(string); ok { + licenses[licenseID] = struct{}{} + } + } + } + } + } + } + } + return len(licenses) +} + +func DownloadSBOMs() { + fileURL := "https://drive.usercontent.google.com/download?id=1LgGlq3g_H02mhzkc94cUd0zzxy0JhFim&export=download&authuser=0&confirm=t&uuid=483eac07-f1af-4356-abeb-4ba254e32b86&at=APZUnTWjSNLUgCQ8wwFZjsLS7Y36:1694113089657" + tarPath := "./SBOM.tar.xz" + + resp, err := http.Get(fileURL) + if err != nil { + log.Fatalf("Error making GET request: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + log.Fatalf("Error: HTTP status code %d", resp.StatusCode) + } + + outputFile, err := os.Create(tarPath) + if err != nil { + log.Fatalf("Error creating output file: %v", err) + } + defer outputFile.Close() + + _, err = io.Copy(outputFile, resp.Body) + if err != nil { + log.Fatalf("Error copying content to output file: %v", err) + } + + fmt.Println("SBOMs downloaded successfully.") + + cmd := exec.Command("tar", "-xJf", tarPath) + cmd.Dir = "./" + + err = cmd.Run() + if err != nil { + log.Fatalf("Error extracting file: %v", err) + } + + fmt.Println("SBOMs extracted successfully.") +} + +func readJSONFile(path string, prefix string) (string, error) { + var result string + files, err := filepath.Glob(path + prefix + ".*.json") + if err != nil { + return "", err + } + for _, file := range files { + data, err := os.ReadFile(file) + if err != nil { + return "", err + } + result = string(data) + } + return result, nil +} + +func TestCount(t *testing.T) { + sbomFolder := "./SBOM/" + _, err := os.Stat(sbomFolder) + + if os.IsNotExist(err) { + // sbomFolder does not exist, download it. + fmt.Println("Downloading SBOMs from Google Drive...") + DownloadSBOMs() + } else if err != nil { + fmt.Printf("Error checking SBOM folder: %v\n", err) + return + } + + SBOM_CONVERT_PATH := "../../dist/sbom-convert_linux_amd64_v1/sbom-convert" + + SBOM_CONVERT_ABSPATH, _ := filepath.Abs(SBOM_CONVERT_PATH) + _, err2 := os.Stat(SBOM_CONVERT_ABSPATH) + if os.IsNotExist(err2) { + fmt.Printf("Binary does not exist: %s.\nRun make binary first!\nExiting...\n", SBOM_CONVERT_ABSPATH) + return + } + + fileInfos, err := os.ReadDir(sbomFolder) + if err != nil { + fmt.Println("Error reading directory:", err) + return + } + tested_count := 0 + for _, fileInfo := range fileInfos { + filename := fileInfo.Name() + + if test_spdx_json == false && strings.Contains(filename, "_spdx.json") { + continue + } + + if test_spdx_keyvalue == false && strings.Contains(filename, "_spdx.txt") { + continue + } + + if test_cdx_json == false && strings.Contains(filename, "_cyclonedx.json") { + continue + } + + filePath := filepath.Join(sbomFolder, filename) + + tested_count++ + if tested_count > max_test_count { + fmt.Printf("Reached max_test_count: %d\n", max_test_count) + break + } + fmt.Printf("=> (%d/%d) Testing %s\n", tested_count, max_test_count, filePath) + + data, err := os.ReadFile(filePath) + ori_json := string(data) + if err != nil { + fmt.Println("Error reading file:", err, "Skipping...") + continue + } + + prefix := filename[:len(filename)-len(filepath.Ext(filename))] + + cmd := exec.Command(SBOM_CONVERT_ABSPATH, filePath, "-o", prefix) + output, err := cmd.CombinedOutput() + if err != nil { + t.Errorf("Convert Check failed: %s\n %s", err, string(output)) + continue + } + fmt.Printf("convert successfully\n") + + converted_json, err := readJSONFile("./", prefix) + // fmt.Println(converted_json) + + ori_purls_count := GetPurlsCount(ori_json) + fmt.Printf("ori_purls_count: %d\n", ori_purls_count) + + converted_purls_count := GetPurlsCount(converted_json) + fmt.Printf("converted_purls_count: %d\n", converted_purls_count) + // continue + + if ori_purls_count > 0 && ori_purls_count > converted_purls_count { + t.Errorf("PURL Check failed. 'Original PURL Count:', %d, 'Converted PURL Count:' %d", ori_purls_count, converted_purls_count) + } + + ori_licenses_count := GetLicensesCount(ori_json) + fmt.Printf("ori_licenses_count: %d\n", ori_licenses_count) + converted_licenses_count := GetLicensesCount(converted_json) + fmt.Printf("converted_licenses_count: %d\n", converted_licenses_count) + + if ori_licenses_count > 0 && ori_licenses_count != converted_licenses_count { + t.Errorf("License Check failed. 'Original License Count:', %d, 'Converted License Count:' %d", ori_licenses_count, converted_licenses_count) + } + + files, err := filepath.Glob(prefix + "*.json") + if err != nil { + panic(err) + } + for _, file := range files { + err := os.Remove(file) + if err != nil { + panic(err) + } + fmt.Printf("Removed file: %s\n", file) + } + } + + // delete downloaded SBOM files and the SBOM.tar.xz file + os.RemoveAll(sbomFolder) + os.Remove("SBOM.tar.xz") +} diff --git a/pkg/convert/fuzz_seed_cyclonedx.json b/pkg/convert/fuzz_seed_cyclonedx.json new file mode 100644 index 0000000..5fdc95b --- /dev/null +++ b/pkg/convert/fuzz_seed_cyclonedx.json @@ -0,0 +1,94 @@ +{ + "$schema": "http://cyclonedx.org/schema/bom-1.4.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "serialNumber": "urn:uuid:fcd73c24-a23c-4c9e-b09c-1e8d068fde44", + "version": 1, + "metadata": { + "timestamp": "2023-07-16T18:18:17-07:00", + "tools": [ + { + "vendor": "anchore", + "name": "syft", + "version": "0.85.0" + } + ], + "component": { + "bom-ref": "cc3ab01ab0c1ab1e", + "type": "file", + "name": "/home/wei/code/repos/python/abhiTronix/vidgear" + } + }, + "components": [ + { + "bom-ref": "pkg:pypi/pyzmq@24.0.1?package-id=d345c09cbb0c1e23", + "type": "library", + "name": "pyzmq", + "version": "24.0.1", + "cpe": "cpe:2.3:a:python-pyzmq:python-pyzmq:24.0.1:*:*:*:*:*:*:*", + "purl": "pkg:pypi/pyzmq@24.0.1", + "properties": [ + { + "name": "syft:package:foundBy", + "value": "python-index-cataloger" + }, + { + "name": "syft:package:language", + "value": "python" + }, + { + "name": "syft:package:type", + "value": "python" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python-pyzmq:python_pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_pyzmq:python-pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_pyzmq:python_pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:python-pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:python_pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python-pyzmq:pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python_pyzmq:pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:pyzmq:python-pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:pyzmq:python_pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:python:pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:pyzmq:pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "name": "syft:location:0:path", + "value": "/setup.py" + } + ] + } + ] +} diff --git a/pkg/convert/fuzz_seed_spdx.json b/pkg/convert/fuzz_seed_spdx.json new file mode 100644 index 0000000..b400d3f --- /dev/null +++ b/pkg/convert/fuzz_seed_spdx.json @@ -0,0 +1,122 @@ +{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "/home/wei/code/repos/python/abhiTronix/vidgear", + "documentNamespace": "https://anchore.com/syft/dir/home/wei/code/repos/python/abhiTronix/vidgear-710e3edc-b86e-4c5a-b71a-2f81c876b318", + "creationInfo": { + "licenseListVersion": "3.21", + "creators": [ + "Organization: Anchore, Inc", + "Tool: syft-0.85.0" + ], + "created": "2023-07-19T15:20:12Z" + }, + "packages": [ + { + "name": "pyzmq", + "SPDXID": "SPDXRef-Package-python-pyzmq-d345c09cbb0c1e23", + "versionInfo": "24.0.1", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": false, + "sourceInfo": "acquired package info from installed python package manifest file: /setup.py", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + "externalRefs": [ + { + "referenceCategory": "SECURITY", + "referenceType": "cpe23Type", + "referenceLocator": "cpe:2.3:a:python-pyzmq:python-pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "referenceCategory": "SECURITY", + "referenceType": "cpe23Type", + "referenceLocator": "cpe:2.3:a:python-pyzmq:python_pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "referenceCategory": "SECURITY", + "referenceType": "cpe23Type", + "referenceLocator": "cpe:2.3:a:python_pyzmq:python-pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "referenceCategory": "SECURITY", + "referenceType": "cpe23Type", + "referenceLocator": "cpe:2.3:a:python_pyzmq:python_pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "referenceCategory": "SECURITY", + "referenceType": "cpe23Type", + "referenceLocator": "cpe:2.3:a:python:python-pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "referenceCategory": "SECURITY", + "referenceType": "cpe23Type", + "referenceLocator": "cpe:2.3:a:python:python_pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "referenceCategory": "SECURITY", + "referenceType": "cpe23Type", + "referenceLocator": "cpe:2.3:a:python-pyzmq:pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "referenceCategory": "SECURITY", + "referenceType": "cpe23Type", + "referenceLocator": "cpe:2.3:a:python_pyzmq:pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "referenceCategory": "SECURITY", + "referenceType": "cpe23Type", + "referenceLocator": "cpe:2.3:a:pyzmq:python-pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "referenceCategory": "SECURITY", + "referenceType": "cpe23Type", + "referenceLocator": "cpe:2.3:a:pyzmq:python_pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "referenceCategory": "SECURITY", + "referenceType": "cpe23Type", + "referenceLocator": "cpe:2.3:a:python:pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "referenceCategory": "SECURITY", + "referenceType": "cpe23Type", + "referenceLocator": "cpe:2.3:a:pyzmq:pyzmq:24.0.1:*:*:*:*:*:*:*" + }, + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:pypi/pyzmq@24.0.1" + } + ] + } + ], + "files": [ + { + "fileName": "/setup.py", + "SPDXID": "SPDXRef-File-setup.py-c829033125510d5a", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "0000000000000000000000000000000000000000" + } + ], + "licenseConcluded": "NOASSERTION", + "copyrightText": "" + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-Package-python-pyzmq-d345c09cbb0c1e23", + "relatedSpdxElement": "SPDXRef-File-setup.py-c829033125510d5a", + "relationshipType": "OTHER", + "comment": "evident-by: indicates the package's existence is evident by the given file" + }, + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relatedSpdxElement": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES" + } + ] +}