From 42e46f4e4775f44c085dd033c040c49b8dc83033 Mon Sep 17 00:00:00 2001 From: williamfzc <178894043@qq.com> Date: Tue, 22 Aug 2023 23:03:00 +0800 Subject: [PATCH] feat: pnpm support Signed-off-by: williamfzc --- .github/workflows/test.yml | 13 +- go.mod | 2 +- pnpm/handler.go | 434 +++++++++++++++++++++++++++++++++++++ pnpm/handler_test.go | 154 +++++++++++++ pnpm/models.go | 13 ++ pnpm/test/package.json | 31 +++ 6 files changed, 645 insertions(+), 2 deletions(-) create mode 100644 pnpm/handler.go create mode 100644 pnpm/handler_test.go create mode 100644 pnpm/models.go create mode 100644 pnpm/test/package.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a443d4e..abb983e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ name: test on: push: - branches: ['main'] + branches: [ 'main' ] pull_request: jobs: @@ -46,6 +46,17 @@ jobs: with: cmd: install dir: 'yarn/test' + - name: Setup node + uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3 + with: + node-version: 16.14.0 + - name: Install dependencies with pnpm + uses: pnpm/action-setup@v2 + with: + version: 8 + run_install: | + - cwd: 'pnpm/test' + package_json_file: 'pnpm/test/package.json' - name: Run swift build working-directory: 'swift/test' run: swift build diff --git a/go.mod b/go.mod index 1b06c10..e7f5933 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/vifraa/gopom v0.2.1 golang.org/x/mod v0.11.0 + gopkg.in/yaml.v3 v3.0.1 sigs.k8s.io/release-utils v0.7.4 ) @@ -52,5 +53,4 @@ require ( gonum.org/v1/gonum v0.8.2 // indirect gopkg.in/neurosnap/sentences.v1 v1.0.7 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pnpm/handler.go b/pnpm/handler.go new file mode 100644 index 0000000..73d3109 --- /dev/null +++ b/pnpm/handler.go @@ -0,0 +1,434 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pnpm + +import ( + "crypto/sha256" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/opensbom-generator/parsers/internal/helper" + "github.com/opensbom-generator/parsers/meta" + "github.com/opensbom-generator/parsers/plugin" + "github.com/opensbom-generator/parsers/reader" +) + +type Pnpm struct { + metadata plugin.Metadata +} + +var ( + errDependenciesNotFound = errors.New("unable to generate SPDX file, no modules founded. Please install them before running spdx-sbom-generator, e.g.: `pnpm install`") + lockFile = "pnpm-lock.yaml" + rg = regexp.MustCompile(`^(((git|hg|svn|bzr)\+)?(http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/|ssh:\/\/|git:\/\/|svn:\/\/|sftp:\/\/|ftp:\/\/)?[a-z0-9]+([\-\.]{1}[a-z0-9]+){0,100}\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*))|(git\+git@[a-zA-Z0-9\.]+:[a-zA-Z0-9/\\.@]+)|(bzr\+lp:[a-zA-Z0-9\.]+)$`) +) + +// New creates a new pnpm instance +func New() *Pnpm { + return &Pnpm{ + metadata: plugin.Metadata{ + Name: "Performant Node Package Manager", + Slug: "pnpm", + Manifest: []string{"package.json", lockFile}, + ModulePath: []string{"node_modules"}, + }, + } +} + +// GetMetadata returns metadata descriptions Name, Slug, Manifest, ModulePath +func (m *Pnpm) GetMetadata() plugin.Metadata { + return m.metadata +} + +// IsValid checks if module has a valid Manifest file +// for pnpm manifest file is package.json +func (m *Pnpm) IsValid(path string) bool { + for _, p := range m.metadata.Manifest { + if !helper.Exists(filepath.Join(path, p)) { + return false + } + } + return true +} + +// HasModulesInstalled checks if modules of manifest file already installed +func (m *Pnpm) HasModulesInstalled(path string) error { + for _, p := range m.metadata.ModulePath { + if !helper.Exists(filepath.Join(path, p)) { + return errDependenciesNotFound + } + } + + for _, p := range m.metadata.Manifest { + if !helper.Exists(filepath.Join(path, p)) { + return errDependenciesNotFound + } + } + return nil +} + +// GetVersion returns pnpm version +func (m *Pnpm) GetVersion() (string, error) { + cmd := exec.Command("pnpm", "-v") + output, err := cmd.Output() + if err != nil { + return "", err + } + + if len(strings.Split(string(output), ".")) != 3 { + return "", fmt.Errorf("unexpected version format: %s", output) + } + + return string(output), nil +} + +// SetRootModule ... +func (m *Pnpm) SetRootModule(path string) error { + return nil +} + +// GetRootModule return +// root package information ex. Name, Version +func (m *Pnpm) GetRootModule(path string) (*meta.Package, error) { + r := reader.New(filepath.Join(path, m.metadata.Manifest[0])) + pkResult, err := r.ReadJSON() + if err != nil { + return &meta.Package{}, err + } + mod := &meta.Package{} + + if pkResult["name"] != nil { + mod.Name = pkResult["name"].(string) + } + if pkResult["author"] != nil { + mod.Supplier.Name = pkResult["author"].(string) + } + if pkResult["version"] != nil { + mod.Version = pkResult["version"].(string) + } + repository := pkResult["repository"] + if repository != nil { + if rep, ok := repository.(string); ok { + mod.PackageDownloadLocation = rep + } + if _, ok := repository.(map[string]interface{}); ok && repository.(map[string]interface{})["url"] != nil { + mod.PackageDownloadLocation = repository.(map[string]interface{})["url"].(string) + } + } + if pkResult["homepage"] != nil { + mod.PackageURL = helper.RemoveURLProtocol(pkResult["homepage"].(string)) + mod.PackageDownloadLocation = mod.PackageURL + } + if !rg.MatchString(mod.PackageDownloadLocation) { + mod.PackageDownloadLocation = "NONE" + } + mod.Packages = map[string]*meta.Package{} + mod.Copyright = getCopyright(path) + modLic, err := helper.GetLicenses(path) + if err != nil { + return mod, nil + } + mod.LicenseDeclared = helper.BuildLicenseDeclared(modLic.ID) + mod.LicenseConcluded = helper.BuildLicenseConcluded(modLic.ID) + mod.CommentsLicense = modLic.Comments + if !helper.LicenseSPDXExists(modLic.ID) { + mod.OtherLicense = append(mod.OtherLicense, *modLic) + } + return mod, nil +} + +// ListUsedModules return brief info of installed modules, Name and Version +func (m *Pnpm) ListUsedModules(path string) ([]meta.Package, error) { + r := reader.New(filepath.Join(path, m.metadata.Manifest[0])) + pkResult, err := r.ReadJSON() + if err != nil { + return []meta.Package{}, err + } + packages := make([]meta.Package, 0) + deps := pkResult["dependencies"].(map[string]interface{}) + + for k, v := range deps { + var mod meta.Package + mod.Name = k + mod.Version = strings.TrimPrefix(v.(string), "^") + packages = append(packages, mod) + } + + return packages, nil +} + +// ListModulesWithDeps return all info of installed modules +func (m *Pnpm) ListModulesWithDeps(path string, globalSettingFile string) ([]meta.Package, error) { + deps, err := readLockFile(filepath.Join(path, lockFile)) + if err != nil { + return nil, err + } + allDeps := appendNestedDependencies(deps) + return m.buildDependencies(path, allDeps) +} + +func (m *Pnpm) buildDependencies(path string, deps []dependency) ([]meta.Package, error) { + modules := make([]meta.Package, 0) + de, err := m.GetRootModule(path) + if err != nil { + return modules, err + } + h := fmt.Sprintf("%x", sha256.Sum256([]byte(fmt.Sprintf("%s-%s", de.Name, de.Version)))) + de.Checksum = meta.Checksum{ + Algorithm: "SHA256", + Value: h, + } + de.Supplier.Name = de.Name + if de.PackageDownloadLocation == "" { + de.PackageDownloadLocation = de.Name + } + modules = append(modules, *de) + for _, d := range deps { + var mod meta.Package + mod.Name = d.Name + mod.Version = extractVersion(d.Version) + modules[0].Packages[d.Name] = &meta.Package{ + Name: d.Name, + Version: mod.Version, + Checksum: meta.Checksum{Content: []byte(fmt.Sprintf("%s-%s", d.Name, mod.Version))}, + } + if len(d.Dependencies) != 0 { + mod.Packages = map[string]*meta.Package{} + for _, depD := range d.Dependencies { + ar := strings.Split(strings.TrimSpace(depD), " ") + name := strings.TrimPrefix(strings.TrimSuffix(strings.TrimPrefix(ar[0], "\""), "\""), "@") + if name == "optionalDependencies:" { + continue + } + + version := strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(ar[1]), "\""), "\"") + if extractVersion(version) == "*" { + continue + } + mod.Packages[name] = &meta.Package{ + Name: name, + Version: extractVersion(version), + Checksum: meta.Checksum{Content: []byte(fmt.Sprintf("%s-%s", name, version))}, + } + } + } + mod.PackageDownloadLocation = strings.TrimSuffix(strings.TrimPrefix(d.Resolved, "\""), "\"") + mod.Supplier.Name = mod.Name + + mod.PackageURL = getPackageHomepage(filepath.Join(path, m.metadata.ModulePath[0], d.PkPath, m.metadata.Manifest[0])) + h := fmt.Sprintf("%x", sha256.Sum256([]byte(mod.Name))) + mod.Checksum = meta.Checksum{ + Algorithm: "SHA256", + Value: h, + } + + licensePath := filepath.Join(path, m.metadata.ModulePath[0], d.PkPath, "LICENSE") + + libDirName := fmt.Sprintf("%s@%s", strings.ReplaceAll(d.PkPath, "/", "+"), d.Version) + if d.Belonging != "" { + libDirName += fmt.Sprintf("%s_%s", libDirName, d.Belonging) + } + licensePathInsidePnpm := filepath.Join( + path, + m.metadata.ModulePath[0], + ".pnpm", + libDirName, + m.metadata.ModulePath[0], + d.PkPath, + "LICENSE", + ) + + var validLicensePath string + switch { + case helper.Exists(licensePath): + validLicensePath = licensePath + case helper.Exists(licensePathInsidePnpm): + validLicensePath = licensePathInsidePnpm + default: + validLicensePath = "" + } + + r := reader.New(validLicensePath) + s := r.StringFromFile() + mod.Copyright = helper.GetCopyright(s) + + modLic, err := helper.GetLicenses(filepath.Join(path, m.metadata.ModulePath[0], d.PkPath)) + if err != nil { + modules = append(modules, mod) + continue + } + mod.LicenseDeclared = helper.BuildLicenseDeclared(modLic.ID) + mod.LicenseConcluded = helper.BuildLicenseConcluded(modLic.ID) + mod.CommentsLicense = modLic.Comments + if !helper.LicenseSPDXExists(modLic.ID) { + mod.OtherLicense = append(mod.OtherLicense, *modLic) + } + modules = append(modules, mod) + } + return modules, nil +} + +func readLockFile(path string) ([]dependency, error) { + fileContent, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var lockData map[string]interface{} + err = yaml.Unmarshal(fileContent, &lockData) + if err != nil { + return nil, err + } + + packages, ok := lockData["packages"].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid lock file format") + } + + dependencies := make([]dependency, 0) + + for pkgName, pkg := range packages { + pkgMap, ok := pkg.(map[string]interface{}) + if !ok { + continue + } + + dep := dependency{} + + name, version, belonging := splitPackageNameAndVersion(pkgName) + nameWithoutAt, pkPath, nameAndVersion := processName(name) + dep.Name = nameWithoutAt + dep.PkPath = pkPath + dep.Version = version + dep.Belonging = belonging + if resolution, ok := pkgMap["resolution"].(map[string]interface{}); ok { + if tarball, ok := resolution["tarball"].(string); ok { + dep.Resolved = tarball + } + if integrity, ok := resolution["integrity"].(string); ok { + dep.Integrity = integrity + } + } + if dep.Resolved == "" { + // .npmrc + registry := "https://registry.npmjs.org" + dep.Resolved = fmt.Sprintf("%s/%s/-/%s-%s.tgz", registry, name, nameAndVersion, dep.Version) + } + + dependenciesRaw, ok := pkgMap["dependencies"].(map[string]interface{}) + if ok { + for depName, ver := range dependenciesRaw { + depPath := fmt.Sprintf("%s %s", depName, ver) + dep.Dependencies = append(dep.Dependencies, strings.TrimSpace(depPath)) + } + } + + dependencies = append(dependencies, dep) + } + + return dependencies, nil +} + +func getCopyright(path string) string { + licensePath := filepath.Join(path, "LICENSE") + if helper.Exists(licensePath) { + r := reader.New(licensePath) + s := r.StringFromFile() + return helper.GetCopyright(s) + } + + licenseMDPath, err := filepath.Glob(filepath.Join(path, "LICENSE*")) + if err != nil { + return "" + } + if len(licenseMDPath) > 0 && helper.Exists(licenseMDPath[0]) { + r := reader.New(licenseMDPath[0]) + s := r.StringFromFile() + return helper.GetCopyright(s) + } + + return "" +} + +func getPackageHomepage(path string) string { + r := reader.New(path) + pkResult, err := r.ReadJSON() + if err != nil { + return "" + } + if pkResult["homepage"] != nil { + return helper.RemoveURLProtocol(pkResult["homepage"].(string)) + } + return "" +} + +func extractVersion(s string) string { + t := strings.TrimPrefix(s, "^") + t = strings.TrimPrefix(t, "~") + t = strings.TrimPrefix(t, ">") + t = strings.TrimPrefix(t, "=") + + t = strings.Split(t, " ")[0] + return t +} + +func splitPackageNameAndVersion(pkg string) (string, string, string) { + // Remove parentheses and content inside + parts := strings.Split(pkg, "(") + pkg = parts[0] + + atIndex := strings.LastIndex(pkg, "@") + if atIndex == -1 { + return "", "", "" + } + + name := strings.TrimLeft(pkg[:atIndex], "/") + version := pkg[atIndex+1:] + + // Extract extra content in parentheses + extra := "" + if len(parts) > 1 { + extra = strings.TrimSuffix(parts[1], ")") + } + + return name, version, extra +} + +func processName(name string) (string, string, string) { + nameWithoutAt := strings.TrimPrefix(name, "@") + pkPath := name + pkgNameParts := strings.Split(nameWithoutAt, "/") + version := pkgNameParts[len(pkgNameParts)-1] + + return nameWithoutAt, pkPath, version +} + +func appendNestedDependencies(deps []dependency) []dependency { + allDeps := make([]dependency, 0) + for _, d := range deps { + allDeps = append(allDeps, d) + if len(d.Dependencies) > 0 { + for _, depD := range d.Dependencies { + ar := strings.Split(strings.TrimSpace(depD), " ") + name := strings.TrimPrefix(strings.TrimSuffix(strings.TrimPrefix(ar[0], "\""), "\""), "@") + if name == "optionalDependencies:" { + continue + } + + version := strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(ar[1]), "\""), "\"") + if extractVersion(version) == "*" { + continue + } + allDeps = append(allDeps, dependency{Name: name, Version: extractVersion(version)}) + } + } + } + return allDeps +} diff --git a/pnpm/handler_test.go b/pnpm/handler_test.go new file mode 100644 index 0000000..b8bc08a --- /dev/null +++ b/pnpm/handler_test.go @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pnpm + +import ( + "crypto/sha256" + "fmt" + "os/exec" + "strings" + "testing" + + "github.com/opensbom-generator/parsers/meta" + + "github.com/stretchr/testify/assert" +) + +func TestPnpm(t *testing.T) { + t.Run("test is valid", TestIsValid) + t.Run("test has modules installed", TestHasModulesInstalled) + t.Run("test get module", TestGetModule) + t.Run("test list modules", TestListModules) + t.Run("test list all modules", TestListAllModules) +} + +func TestIsValid(t *testing.T) { + n := New() + path := fmt.Sprintf("%s/test", getPath()) + + valid := n.IsValid(path) + invalid := n.IsValid(getPath()) + + // Assert + assert.Equal(t, true, valid) + assert.Equal(t, false, invalid) +} + +func TestHasModulesInstalled(t *testing.T) { + n := New() + path := fmt.Sprintf("%s/test", getPath()) + + installed := n.HasModulesInstalled(path) + assert.NoError(t, installed) + uninstalled := n.HasModulesInstalled(getPath()) + assert.Error(t, uninstalled) +} + +func TestGetModule(t *testing.T) { + n := New() + path := fmt.Sprintf("%s/test", getPath()) + mod, err := n.GetRootModule(path) + + assert.NoError(t, err) + assert.Equal(t, "create-react-app-lambda", mod.Name) + assert.Equal(t, "", mod.Supplier.Name) + assert.Equal(t, "0.5.0", mod.Version) +} + +func TestListModules(t *testing.T) { + n := New() + path := fmt.Sprintf("%s/test", getPath()) + mods, err := n.ListUsedModules(path) + + assert.NoError(t, err) + + count := 0 + for _, mod := range mods { + if mod.Name == "axios" { + assert.Equal(t, "axios", mod.Name) + assert.Equal(t, "0.19.0", mod.Version) + count++ + continue + } + + if mod.Name == "react" { + assert.Equal(t, "react", mod.Name) + assert.Equal(t, "16.8.6", mod.Version) + count++ + continue + } + if mod.Name == "react-dom" { + assert.Equal(t, "react-dom", mod.Name) + assert.Equal(t, "16.8.6", mod.Version) + count++ + continue + } + } + + assert.Equal(t, 3, count) +} + +func TestListAllModules(t *testing.T) { + n := New() + path := fmt.Sprintf("%s/test", getPath()) + var globalSettingFile string + mods, err := n.ListModulesWithDeps(path, globalSettingFile) + + assert.NoError(t, err) + + count := 0 + for _, mod := range mods { + if mod.Name == "axios" { + h := fmt.Sprintf("%x", sha256.Sum256([]byte(mod.Name))) + assert.Equal(t, "0.19.0", mod.Version) + assert.Equal(t, "https://registry.npmjs.org/axios/-/axios-0.19.0.tgz", mod.PackageDownloadLocation) + assert.Equal(t, meta.HashAlgorithm("SHA256"), mod.Checksum.Algorithm) + assert.Equal(t, h, mod.Checksum.Value) + assert.Equal(t, "Copyright (c) 2014-present Matt Zabriskie", mod.Copyright) + assert.Equal(t, "MIT", mod.LicenseDeclared) + count++ + continue + } + if mod.Name == "react" { + // transitive dep if empty + if mod.Copyright == "" { + continue + } + h := fmt.Sprintf("%x", sha256.Sum256([]byte(mod.Name))) + + assert.Equal(t, "16.8.6", mod.Version) + assert.Equal(t, "https://registry.npmjs.org/react/-/react-16.8.6.tgz", mod.PackageDownloadLocation) + assert.Equal(t, meta.HashAlgorithm("SHA256"), mod.Checksum.Algorithm) + assert.Equal(t, h, mod.Checksum.Value) + assert.Equal(t, "Copyright (c) Facebook, Inc. and its affiliates.", mod.Copyright) + assert.Equal(t, "MIT", mod.LicenseDeclared) + count++ + continue + } + if mod.Name == "react-dom" { + h := fmt.Sprintf("%x", sha256.Sum256([]byte(mod.Name))) + + assert.Equal(t, "16.8.6", mod.Version) + assert.Equal(t, "https://registry.npmjs.org/react-dom/-/react-dom-16.8.6.tgz", mod.PackageDownloadLocation) + assert.Equal(t, meta.HashAlgorithm("SHA256"), mod.Checksum.Algorithm) + assert.Equal(t, h, mod.Checksum.Value) + assert.Equal(t, "Copyright (c) Facebook, Inc. and its affiliates.", mod.Copyright) + assert.Equal(t, "MIT", mod.LicenseDeclared) + count++ + continue + } + } + + assert.Equal(t, 3, count) +} + +func getPath() string { + cmd := exec.Command("pwd") + output, err := cmd.Output() + if err != nil { + return "" + } + path := strings.TrimSuffix(string(output), "\n") + + return path +} diff --git a/pnpm/models.go b/pnpm/models.go new file mode 100644 index 0000000..4ab742c --- /dev/null +++ b/pnpm/models.go @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pnpm + +type dependency struct { + Name string + PkPath string + Version string + Resolved string + Integrity string + Dependencies []string + Belonging string +} diff --git a/pnpm/test/package.json b/pnpm/test/package.json new file mode 100644 index 0000000..d9d19e3 --- /dev/null +++ b/pnpm/test/package.json @@ -0,0 +1,31 @@ +{ + "name": "create-react-app-lambda", + "version": "0.5.0", + "private": true, + "dependencies": { + "axios": "^0.19.0", + "react": "^16.8.6", + "react-dom": "^16.8.6" + }, + "scripts": { + "start": "react-scripts start", + "build": "run-p build:**", + "build:app": "react-scripts build", + "build:lambda": "netlify-lambda build src/lambda", + "test": "react-scripts test", + "eject": "react-scripts eject" + }, + "eslintConfig": { + "extends": "react-app" + }, + "browserslist": [ + ">0.2%", + "not dead", + "not ie <= 11", + "not op_mini all" + ], + "devDependencies": { + "netlify-lambda": "^1.4.5", + "npm-run-all": "^4.1.5" + } +}