Skip to content

Commit

Permalink
factor out package-lock.json struct
Browse files Browse the repository at this point in the history
  • Loading branch information
michaelkedar committed Feb 20, 2025
1 parent 64e12d8 commit fab3a03
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 110 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -28,43 +28,14 @@ import (
"github.com/google/osv-scalibr/extractor/filesystem"
"github.com/google/osv-scalibr/extractor/filesystem/language/javascript/internal/commitextractor"
"github.com/google/osv-scalibr/extractor/filesystem/osv"
"github.com/google/osv-scalibr/internal/dependencyfile/packagelockjson"
"github.com/google/osv-scalibr/plugin"
"github.com/google/osv-scalibr/purl"
"github.com/google/osv-scalibr/stats"

"golang.org/x/exp/maps"
)

type npmLockDependency struct {
// For an aliased package, Version is like "npm:[name]@[version]"
Version string `json:"version"`
Dependencies map[string]npmLockDependency `json:"dependencies,omitempty"`

Dev bool `json:"dev,omitempty"`
Optional bool `json:"optional,omitempty"`
}

type npmLockPackage struct {
// For an aliased package, Name is the real package name
Name string `json:"name"`
Version string `json:"version"`
Resolved string `json:"resolved"`

Dev bool `json:"dev,omitempty"`
DevOptional bool `json:"devOptional,omitempty"`
Optional bool `json:"optional,omitempty"`

Link bool `json:"link,omitempty"`
}

type npmLockfile struct {
Version int `json:"lockfileVersion"`
// npm v1- lockfiles use "dependencies"
Dependencies map[string]npmLockDependency `json:"dependencies,omitempty"`
// npm v2+ lockfiles use "packages"
Packages map[string]npmLockPackage `json:"packages,omitempty"`
}

type packageDetails struct {
Name string
Version string
Expand Down Expand Up @@ -105,21 +76,7 @@ func (pdm npmPackageDetailsMap) add(key string, details packageDetails) {
pdm[key] = details
}

func (dep npmLockDependency) depGroups() []string {
if dep.Dev && dep.Optional {
return []string{"dev", "optional"}
}
if dep.Dev {
return []string{"dev"}
}
if dep.Optional {
return []string{"optional"}
}

return nil
}

func parseNpmLockDependencies(dependencies map[string]npmLockDependency) map[string]packageDetails {
func parseNpmLockDependencies(dependencies map[string]packagelockjson.Dependency) map[string]packageDetails {
details := npmPackageDetailsMap{}

for name, detail := range dependencies {
Expand Down Expand Up @@ -162,7 +119,7 @@ func parseNpmLockDependencies(dependencies map[string]npmLockDependency) map[str
Name: name,
Version: finalVersion,
Commit: commit,
DepGroups: detail.depGroups(),
DepGroups: detail.DepGroups(),
})
}

Expand All @@ -180,21 +137,7 @@ func extractNpmPackageName(name string) string {
return pkgName
}

func (pkg npmLockPackage) depGroups() []string {
if pkg.Dev {
return []string{"dev"}
}
if pkg.Optional {
return []string{"optional"}
}
if pkg.DevOptional {
return []string{"dev", "optional"}
}

return nil
}

func parseNpmLockPackages(packages map[string]npmLockPackage) map[string]packageDetails {
func parseNpmLockPackages(packages map[string]packagelockjson.Package) map[string]packageDetails {
details := npmPackageDetailsMap{}

for namePath, detail := range packages {
Expand All @@ -221,14 +164,14 @@ func parseNpmLockPackages(packages map[string]npmLockPackage) map[string]package
Name: finalName,
Version: detail.Version,
Commit: commit,
DepGroups: detail.depGroups(),
DepGroups: detail.DepGroups(),
})
}

return details
}

func parseNpmLock(lockfile npmLockfile) map[string]packageDetails {
func parseNpmLock(lockfile packagelockjson.LockFile) map[string]packageDetails {
if lockfile.Packages != nil {
return parseNpmLockPackages(lockfile.Packages)
}
Expand Down Expand Up @@ -341,7 +284,7 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]
}

func (e Extractor) extractPkgLock(_ context.Context, input *filesystem.ScanInput) ([]*extractor.Inventory, error) {
var parsedLockfile *npmLockfile
var parsedLockfile *packagelockjson.LockFile

err := json.NewDecoder(input.Reader).Decode(&parsedLockfile)

Expand Down
90 changes: 90 additions & 0 deletions internal/dependencyfile/packagelockjson/packagelockjson.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// Copyright 2025 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package packagelockjson provides the structures for npm's package-lock.json lockfile format.
package packagelockjson

// Lockfile is the npm package-lock.json lockfile.
type LockFile struct {
Version int `json:"lockfileVersion"`
// npm v1- lockfiles use "dependencies"
Dependencies map[string]Dependency `json:"dependencies,omitempty"`
// npm v2+ lockfiles use "packages"
Packages map[string]Package `json:"packages,omitempty"`
}

// Dependency is the representation of an installed dependency in lockfileVersion 1
type Dependency struct {
// For an aliased package, Version is like "npm:[name]@[version]"
Version string `json:"version"`

Dev bool `json:"dev,omitempty"`
Optional bool `json:"optional,omitempty"`

Requires map[string]string `json:"requires,omitempty"`
Dependencies map[string]Dependency `json:"dependencies,omitempty"`
}

// DepGroups returns the list of groups this dependency belongs to.
// May be empty, or one or both of "dev", "optional".
func (dep Dependency) DepGroups() []string {
if dep.Dev && dep.Optional {
return []string{"dev", "optional"}
}
if dep.Dev {
return []string{"dev"}
}
if dep.Optional {
return []string{"optional"}
}

return nil
}

// Package is the representation of an installed dependency in lockfileVersion 2+
type Package struct {
// For an aliased package, Name is the real package name
Name string `json:"name,omitempty"`
Version string `json:"version"`
Resolved string `json:"resolved"`
Link bool `json:"link,omitempty"`

Dev bool `json:"dev,omitempty"`
DevOptional bool `json:"devOptional,omitempty"`
Optional bool `json:"optional,omitempty"`

Dependencies map[string]string `json:"dependencies,omitempty"`
DevDependencies map[string]string `json:"devDependencies,omitempty"`
OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"`
PeerDependencies map[string]string `json:"peerDependencies,omitempty"`
PeerDependenciesMeta map[string]struct {
Optional bool `json:"optional,omitempty"`
} `json:"peerDependenciesMeta,omitempty"`
}

// DepGroups returns the list of groups this package belongs to.
// May be empty, or one or both of "dev", "optional".
func (pkg Package) DepGroups() []string {
if pkg.Dev {
return []string{"dev"}
}
if pkg.Optional {
return []string{"optional"}
}
if pkg.DevOptional {
return []string{"dev", "optional"}
}

return nil
}
43 changes: 2 additions & 41 deletions internal/guidedremediation/lockfile/npm/packagelockjson.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"deps.dev/util/resolve"
"deps.dev/util/resolve/dep"
scalibrfs "github.com/google/osv-scalibr/fs"
"github.com/google/osv-scalibr/internal/dependencyfile/packagelockjson"
"github.com/google/osv-scalibr/internal/guidedremediation/lockfile"
"github.com/google/osv-scalibr/internal/guidedremediation/manifest/npm"
"github.com/google/osv-scalibr/log"
Expand All @@ -41,46 +42,6 @@ func (r readWriter) System() resolve.System {
return resolve.NPM
}

type lockDependency struct {
// For an aliased package, Version is like "npm:[name]@[version]"
Version string `json:"version"`
Dependencies map[string]lockDependency `json:"dependencies,omitempty"`

Dev bool `json:"dev,omitempty"`
Optional bool `json:"optional,omitempty"`

Requires map[string]string `json:"requires,omitempty"`
}

type lockPackage struct {
// For an aliased package, Name is the real package name
Name string `json:"name"`
Version string `json:"version"`
Resolved string `json:"resolved"`

Dependencies map[string]string `json:"dependencies,omitempty"`
DevDependencies map[string]string `json:"devDependencies,omitempty"`
OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"`
PeerDependencies map[string]string `json:"peerDependencies,omitempty"`

PeerDependenciesMeta map[string]struct {
Optional bool `json:"optional,omitempty"`
} `json:"peerDependenciesMeta,omitempty"`

Dev bool `json:"dev,omitempty"`
DevOptional bool `json:"devOptional,omitempty"`
Optional bool `json:"optional,omitempty"`

Link bool `json:"link,omitempty"`
}

type packageLockJSON struct {
// npm v1- lockfiles use "dependencies"
Dependencies map[string]lockDependency `json:"dependencies,omitempty"`
// npm v2+ lockfiles use "packages"
Packages map[string]lockPackage `json:"packages,omitempty"`
}

type dependencyVersionSpec struct {
Version string
DepType dep.Type
Expand Down Expand Up @@ -108,7 +69,7 @@ func (r readWriter) Read(path string, fsys scalibrfs.FS) (*resolve.Graph, error)
defer f.Close()

dec := json.NewDecoder(f)
var lockJSON packageLockJSON
var lockJSON packagelockjson.LockFile
if err := dec.Decode(&lockJSON); err != nil {
return nil, err
}
Expand Down
5 changes: 3 additions & 2 deletions internal/guidedremediation/lockfile/npm/packagelockjsonv1.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (

"deps.dev/util/resolve"
"deps.dev/util/resolve/dep"
"github.com/google/osv-scalibr/internal/dependencyfile/packagelockjson"
"github.com/google/osv-scalibr/internal/guidedremediation/manifest/npm"
)

Expand All @@ -28,7 +29,7 @@ import (
// Installed packages stored in recursive "dependencies" object
// with "requires" field listing direct dependencies, and each possibly having their own "dependencies"
// No dependency information package-lock.json for the root node, so we must also have the package.json
func nodesFromDependencies(lockJSON packageLockJSON, packageJSON io.Reader) (*resolve.Graph, *nodeModule, error) {
func nodesFromDependencies(lockJSON packagelockjson.LockFile, packageJSON io.Reader) (*resolve.Graph, *nodeModule, error) {
// Need to grab the root requirements from the package.json, since it's not in the lockfile
var manifestJSON npm.PackageJSON
if err := json.NewDecoder(packageJSON).Decode(&manifestJSON); err != nil {
Expand Down Expand Up @@ -75,7 +76,7 @@ func nodesFromDependencies(lockJSON packageLockJSON, packageJSON io.Reader) (*re
return g, nodeModuleTree, err
}

func computeDependenciesRecursive(g *resolve.Graph, parent *nodeModule, deps map[string]lockDependency) error {
func computeDependenciesRecursive(g *resolve.Graph, parent *nodeModule, deps map[string]packagelockjson.Dependency) error {
for name, d := range deps {
actualName, version := npm.SplitNPMAlias(d.Version)
nID := g.AddNode(resolve.VersionKey{
Expand Down
7 changes: 4 additions & 3 deletions internal/guidedremediation/lockfile/npm/packagelockjsonv2.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,15 @@ import (

"deps.dev/util/resolve"
"deps.dev/util/resolve/dep"
"github.com/google/osv-scalibr/internal/dependencyfile/packagelockjson"
)

// nodesFromPackages extracts graph from new-style (npm >= 7 / lockfileVersion 2+) structure
// https://docs.npmjs.com/cli/v9/configuring-npm/package-lock-json
// Installed packages are in the flat "packages" object, keyed by the install path
// e.g. "node_modules/foo/node_modules/bar"
// packages contain most information from their own manifests.
func nodesFromPackages(lockJSON packageLockJSON) (*resolve.Graph, *nodeModule, error) {
func nodesFromPackages(lockJSON packagelockjson.LockFile) (*resolve.Graph, *nodeModule, error) {
g := &resolve.Graph{}
// Create graph nodes and reconstruct the node_modules folder structure in memory
root, ok := lockJSON.Packages[""]
Expand Down Expand Up @@ -155,7 +156,7 @@ func nodesFromPackages(lockJSON packageLockJSON) (*resolve.Graph, *nodeModule, e
return g, nodeModuleTree, nil
}

func makeNodeModuleDeps(pkg lockPackage, includeDev bool) *nodeModule {
func makeNodeModuleDeps(pkg packagelockjson.Package, includeDev bool) *nodeModule {
nm := nodeModule{
Children: make(map[string]*nodeModule),
Deps: make(map[string]dependencyVersionSpec),
Expand Down Expand Up @@ -186,7 +187,7 @@ func makeNodeModuleDeps(pkg lockPackage, includeDev bool) *nodeModule {
return &nm
}

func packageNamesByNodeModuleDepth(packages map[string]lockPackage) []string {
func packageNamesByNodeModuleDepth(packages map[string]packagelockjson.Package) []string {
keys := slices.Collect(maps.Keys(packages))
slices.SortFunc(keys, func(a, b string) int {
aSplit := strings.Split(a, "node_modules/")
Expand Down

0 comments on commit fab3a03

Please sign in to comment.