fix(python): add poetry v2 support (aquasecurity#8323)

Co-authored-by: Nikita Pivkin <[email protected]>
DmitriyLewen · Feb 3, 2025 · 10cd98c · 10cd98c
1 parent 9b74384
commit 10cd98c
Show file tree

Hide file tree

Showing 13 changed files with 1,065 additions and 2,199 deletions.
diff --git a/pkg/dependency/parser/python/poetry/parse.go b/pkg/dependency/parser/python/poetry/parse.go
@@ -1,6 +1,7 @@
 package poetry
 
 import (
+	"slices"
 	"sort"
 
 	"github.com/BurntSushi/toml"
@@ -17,6 +18,7 @@ import (
 type Lockfile struct {
 	Packages []struct {
 		Category       string         `toml:"category"`
+		Groups         []string       `toml:"groups"`
 		Description    string         `toml:"description"`
 		Marker         string         `toml:"marker,omitempty"`
 		Name           string         `toml:"name"`
@@ -50,15 +52,16 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc
 	var pkgs []ftypes.Package
 	var deps []ftypes.Dependency
 	for _, pkg := range lockfile.Packages {
-		if pkg.Category == "dev" {
-			continue
-		}
-
 		pkgID := packageID(pkg.Name, pkg.Version)
 		pkgs = append(pkgs, ftypes.Package{
 			ID:      pkgID,
 			Name:    pkg.Name,
 			Version: pkg.Version,
+			// TODO upgrade logic for working with groups
+			// Mark only:
+			// - `category = "dev"`
+			// - groups without `main`. e.g. `groups = ["dev"]`
+			Dev: pkg.Category == "dev" || (len(pkg.Groups) > 0 && !slices.Contains(pkg.Groups, "main")),
 		})
 
 		dependsOn := p.parseDependencies(pkg.Dependencies, pkgVersions)

diff --git a/pkg/dependency/parser/python/poetry/parse_test.go b/pkg/dependency/parser/python/poetry/parse_test.go
@@ -25,20 +25,20 @@ func TestParser_Parse(t *testing.T) {
 			wantPkgs: poetryNormal,
 			wantErr:  assert.NoError,
 		},
-		{
-			name:     "many",
-			file:     "testdata/poetry_many.lock",
-			wantPkgs: poetryMany,
-			wantDeps: poetryManyDeps,
-			wantErr:  assert.NoError,
-		},
 		{
 			name:     "flask",
 			file:     "testdata/poetry_flask.lock",
 			wantPkgs: poetryFlask,
 			wantDeps: poetryFlaskDeps,
 			wantErr:  assert.NoError,
 		},
+		{
+			name:     "flask + poetry v2",
+			file:     "testdata/poetry_v2_flask.lock",
+			wantPkgs: poetryFlask,
+			wantDeps: poetryV2FlaskDeps,
+			wantErr:  assert.NoError,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

diff --git a/pkg/dependency/parser/python/poetry/parse_testcase.go b/pkg/dependency/parser/python/poetry/parse_testcase.go
@@ -14,122 +14,55 @@ var (
 		{ID: "[email protected]", Name: "pypi", Version: "2.1"},
 	}
 
-	// docker run --name poetry --rm -it python@sha256:e1141f10176d74d1a0e87a7c0a0a5a98dd98ec5ac12ce867768f40c6feae2fd9 sh
-	// apk add curl
-	// curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.1.7 python3 -
-	// export PATH=/root/.local/bin:$PATH
-	// poetry new many && cd many
-	// curl -o poetry.lock https://raw.githubusercontent.com/python-poetry/poetry/c8945eb110aeda611cc6721565d7ad0c657d453a/poetry.lock
-	// curl -o pyproject.toml https://raw.githubusercontent.com/python-poetry/poetry/c8945eb110aeda611cc6721565d7ad0c657d453a/pyproject.toml
-	// poetry show -a | awk '{gsub(/\(!\)/, ""); printf("{ID: \""$1"@"$2"\", Name: \""$1"\", Version: \""$2"\"},\n") }'
-	// `--no-dev` flag uncorrected returns deps. Then need to remove `dev` deps manually
-	// list of dev deps - cat poetry.lock | grep 'category = "dev"' -B 3
-	poetryMany = []ftypes.Package{
-		{ID: "[email protected]", Name: "attrs", Version: "22.2.0"},
-		{ID: "[email protected]", Name: "backports-cached-property", Version: "1.0.2"},
-		{ID: "[email protected]", Name: "build", Version: "0.10.0"},
-		{ID: "[email protected]", Name: "cachecontrol", Version: "0.12.11"},
-		{ID: "[email protected]", Name: "certifi", Version: "2022.12.7"},
-		{ID: "[email protected]", Name: "cffi", Version: "1.15.1"},
-		{ID: "[email protected]", Name: "charset-normalizer", Version: "3.0.1"},
-		{ID: "[email protected]", Name: "cleo", Version: "2.0.1"},
-		{ID: "[email protected]", Name: "colorama", Version: "0.4.6"},
-		{ID: "[email protected]", Name: "crashtest", Version: "0.4.1"},
-		{ID: "[email protected]", Name: "cryptography", Version: "39.0.0"},
-		{ID: "[email protected]", Name: "distlib", Version: "0.3.6"},
-		{ID: "[email protected]", Name: "dulwich", Version: "0.21.2"},
-		{ID: "[email protected]", Name: "filelock", Version: "3.9.0"},
-		{ID: "[email protected]", Name: "html5lib", Version: "1.1"},
-		{ID: "[email protected]", Name: "idna", Version: "3.4"},
-		{ID: "[email protected]", Name: "importlib-metadata", Version: "6.0.0"},
-		{ID: "[email protected]", Name: "importlib-resources", Version: "5.10.2"},
-		{ID: "[email protected]", Name: "installer", Version: "0.6.0"},
-		{ID: "[email protected]", Name: "jaraco-classes", Version: "3.2.3"},
-		{ID: "[email protected]", Name: "jeepney", Version: "0.8.0"},
-		{ID: "[email protected]", Name: "jsonschema", Version: "4.17.3"},
-		{ID: "[email protected]", Name: "keyring", Version: "23.13.1"},
-		{ID: "[email protected]", Name: "lockfile", Version: "0.12.2"},
-		{ID: "[email protected]", Name: "more-itertools", Version: "9.0.0"},
-		{ID: "[email protected]", Name: "msgpack", Version: "1.0.4"},
-		{ID: "[email protected]", Name: "packaging", Version: "23.0"},
-		{ID: "[email protected]", Name: "pexpect", Version: "4.8.0"},
-		{ID: "[email protected]", Name: "pkginfo", Version: "1.9.6"},
-		{ID: "[email protected]", Name: "pkgutil-resolve-name", Version: "1.3.10"},
-		{ID: "[email protected]", Name: "platformdirs", Version: "2.6.2"},
-		{ID: "[email protected]", Name: "poetry-core", Version: "1.5.0"},
-		{ID: "[email protected]", Name: "poetry-plugin-export", Version: "1.3.0"},
-		{ID: "[email protected]", Name: "ptyprocess", Version: "0.7.0"},
-		{ID: "[email protected]", Name: "pycparser", Version: "2.21"},
-		{ID: "[email protected]", Name: "pyproject-hooks", Version: "1.0.0"},
-		{ID: "[email protected]", Name: "pyrsistent", Version: "0.19.3"},
-		{ID: "[email protected]", Name: "pywin32-ctypes", Version: "0.2.0"},
-		{ID: "[email protected]", Name: "rapidfuzz", Version: "2.13.7"},
-		{ID: "[email protected]", Name: "requests", Version: "2.28.2"},
-		{ID: "[email protected]", Name: "requests-toolbelt", Version: "0.10.1"},
-		{ID: "[email protected]", Name: "secretstorage", Version: "3.3.3"},
-		{ID: "[email protected]", Name: "shellingham", Version: "1.5.0.post1"},
-		{ID: "[email protected]", Name: "six", Version: "1.16.0"},
-		{ID: "[email protected]", Name: "tomli", Version: "2.0.1"},
-		{ID: "[email protected]", Name: "tomlkit", Version: "0.11.6"},
-		{ID: "[email protected]", Name: "trove-classifiers", Version: "2023.1.20"},
-		{ID: "[email protected]", Name: "typing-extensions", Version: "4.4.0"},
-		{ID: "[email protected]", Name: "urllib3", Version: "1.26.14"},
-		{ID: "[email protected]", Name: "virtualenv", Version: "20.16.5"},
-		{ID: "[email protected]", Name: "virtualenv", Version: "20.17.1"},
-		{ID: "[email protected]", Name: "webencodings", Version: "0.5.1"},
-		{ID: "[email protected]", Name: "xattr", Version: "0.10.1"},
-		{ID: "[email protected]", Name: "zipp", Version: "3.12.0"},
-	}
-
-	// cat poetry.lock | grep "\[package.dependencies\]" -B 3 -A 8 - it might help to complete this slice
-	poetryManyDeps = []ftypes.Dependency{
-		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]", "[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]", "[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
-	}
-
 	// docker run --name poetry --rm -it python@sha256:e1141f10176d74d1a0e87a7c0a0a5a98dd98ec5ac12ce867768f40c6feae2fd9 sh
 	// apk add curl
 	// curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.1.7 python3 -
 	// export PATH=/root/.local/bin:$PATH
 	// poetry new web && cd web
+	// poetry add [email protected] -E pre-commit,tox
 	// poetry add [email protected]
+	// poetry add [email protected] --dev
 	// poetry show -a | awk '{gsub(/\(!\)/, ""); printf("{ID: \""$1"@"$2"\", Name: \""$1"\", Version: \""$2"\"},\n") }'
+	// mark dev deps
 	poetryFlask = []ftypes.Package{
-		{ID: "[email protected]", Name: "click", Version: "8.1.3"},
+		{ID: "[email protected]", Name: "atomicwrites", Version: "1.4.1", Dev: true},
+		{ID: "[email protected]", Name: "attrs", Version: "25.1.0", Dev: true},
+		{ID: "[email protected]", Name: "click", Version: "8.1.8"},
 		{ID: "[email protected]", Name: "colorama", Version: "0.4.6"},
 		{ID: "[email protected]", Name: "flask", Version: "1.0.3"},
-		{ID: "[email protected]", Name: "itsdangerous", Version: "2.1.2"},
-		{ID: "[email protected]", Name: "jinja2", Version: "3.1.2"},
-		{ID: "[email protected]", Name: "markupsafe", Version: "2.1.2"},
-		{ID: "[email protected]", Name: "werkzeug", Version: "2.2.3"},
+		{ID: "[email protected]", Name: "itsdangerous", Version: "2.2.0"},
+		{ID: "[email protected]", Name: "jinja2", Version: "3.1.5"},
+		{ID: "[email protected]", Name: "markupsafe", Version: "3.0.2"},
+		{ID: "[email protected]", Name: "more-itertools", Version: "10.6.0", Dev: true},
+		{ID: "[email protected]", Name: "packaging", Version: "24.2", Dev: true},
+		{ID: "[email protected]", Name: "pluggy", Version: "0.13.1", Dev: false},
+		{ID: "[email protected]", Name: "py", Version: "1.11.0", Dev: true},
+		{ID: "[email protected]", Name: "pytest", Version: "5.4.3", Dev: true},
+		{ID: "[email protected]", Name: "wcwidth", Version: "0.2.13", Dev: true},
+		{ID: "[email protected]", Name: "werkzeug", Version: "3.1.3"},
 	}
 
 	// cat poetry.lock | grep "\[package.dependencies\]" -B 3 -A 8 - it might help to complete this slice
 	poetryFlaskDeps = []ftypes.Dependency{
-		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]", "[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
-		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
+		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
+		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]", "[email protected]"}},
+		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
+		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]"}},
+		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
+	}
+
+	// use instruction above with `POETRY_VERSION=2.0.1`
+
+	poetryV2FlaskDeps = []ftypes.Dependency{
+		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
+		{ID: "[email protected]", DependsOn: []string{"[email protected]", "[email protected]", "[email protected]", "[email protected]"}},
+		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
+		{
+			ID: "[email protected]", DependsOn: []string{
+				"[email protected]", "[email protected]", "[email protected]", "[email protected]", "[email protected]",
+				"[email protected]", "[email protected]", "[email protected]",
+			},
+		},
+		{ID: "[email protected]", DependsOn: []string{"[email protected]"}},
 	}
 )