Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(poetrylock): support extracting groups from v2+ poetry.lock files #376

Merged
merged 7 commits into from
Jan 24, 2025
+315 −9
Merged

feat(poetrylock): support extracting groups from v2+ poetry.lock files #376

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3c73ce3
test: add fixture
G-Rath Jan 7, 2025
d1a8e86
feat: support reading package groups for Poetry v2 lockfiles
G-Rath Jan 7, 2025
7d89768
fix: omit the `main` group
G-Rath Jan 7, 2025
764cfc9
test: improve fixture a little
G-Rath Jan 7, 2025
bdbc629
fix: remove all other groups to indicate production dependencies
G-Rath Jan 7, 2025
340f76b
refactor: optimize grouping logic
G-Rath Jan 7, 2025
f91ddcd
test: update case to include a python version constraint
G-Rath Jan 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 26 additions & 9 deletions extractor/filesystem/language/python/poetrylock/poetrylock.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type poetryLockPackage struct {
Name string `toml:"name"`
Version string `toml:"version"`
Optional bool `toml:"optional"`
Groups []string `toml:"groups"`
Source poetryLockPackageSource `toml:"source"`
}

Expand Down Expand Up @@ -65,6 +66,28 @@ func (e Extractor) FileRequired(api filesystem.FileAPI) bool {
return filepath.Base(api.Path()) == "poetry.lock"
}

func resolveGroups(pkg poetryLockPackage) []string {
// by definition an optional package cannot be in any other group,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not clear on how optional works here, does this refer to this? https://python-poetry.org/docs/managing-dependencies/#optional-groups or are optional groups and optional packages different.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

umm ok I didn't know about that - I believe it was referring to https://python-poetry.org/docs/pyproject/#optional-dependencies but I'll check what happens when using these optional groups.

fwiw right now "optional" logically is focused on "is the optional property set to true" - so I know that optional-dependencies is one way of having that property come out as true, and now the question is if "optional groups" result in that too or something different

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it looks like "optional groups" just manage what gets installed by default when you run poetry install, so they don't have any bearing on the lockfile itself, which makes sense

// otherwise that would make it a required package
if pkg.Optional {
return []string{"optional"}
}

if pkg.Groups == nil {
return []string{}
}

for _, group := range pkg.Groups {
// the "main" group is the default group used for "production" dependencies,
// which we represent by an empty slice aka no groups
if group == "main" {
return []string{}
}
}

return pkg.Groups
}

// Extract extracts packages from poetry.lock files passed through the scan input.
func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]*extractor.Inventory, error) {
var parsedLockfile *poetryLockFile
Expand All @@ -82,21 +105,15 @@ func (e Extractor) Extract(ctx context.Context, input *filesystem.ScanInput) ([]
Name: lockPackage.Name,
Version: lockPackage.Version,
Locations: []string{input.Path},
Metadata: osv.DepGroupMetadata{
DepGroupVals: resolveGroups(lockPackage),
},
}
if lockPackage.Source.Commit != "" {
pkgDetails.SourceCode = &extractor.SourceCodeIdentifier{
Commit: lockPackage.Source.Commit,
}
}
if lockPackage.Optional {
pkgDetails.Metadata = osv.DepGroupMetadata{
DepGroupVals: []string{"optional"},
}
} else {
pkgDetails.Metadata = osv.DepGroupMetadata{
DepGroupVals: []string{},
}
}
packages = append(packages, pkgDetails)
}

Expand Down
96 changes: 96 additions & 0 deletions extractor/filesystem/language/python/poetrylock/poetrylock_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,102 @@ func TestExtractor_Extract(t *testing.T) {
},
},
},
{
Name: "multiple packages with a v2 lockfile",
InputConfig: extracttest.ScanInputMockConfig{
Path: "testdata/multiple-packages.v2.lock",
},
WantInventory: []*extractor.Inventory{
{
Name: "async-timeout",
Version: "5.0.1",
Locations: []string{"testdata/multiple-packages.v2.lock"},
Metadata: osv.DepGroupMetadata{
DepGroupVals: []string{"optional"},
},
},
{
Name: "factory-boy",
Version: "3.3.1",
Locations: []string{"testdata/multiple-packages.v2.lock"},
Metadata: osv.DepGroupMetadata{
DepGroupVals: []string{"dev"},
},
},
{
Name: "faker",
Version: "33.3.0",
Locations: []string{"testdata/multiple-packages.v2.lock"},
Metadata: osv.DepGroupMetadata{
DepGroupVals: []string{"dev", "test"},
},
},
{
Name: "proto-plus",
Version: "1.22.0",
Locations: []string{"testdata/multiple-packages.v2.lock"},
Metadata: osv.DepGroupMetadata{
DepGroupVals: []string{},
},
},
{
Name: "proto-plus",
Version: "1.23.0",
Locations: []string{"testdata/multiple-packages.v2.lock"},
Metadata: osv.DepGroupMetadata{
DepGroupVals: []string{},
},
},
{
Name: "protobuf",
Version: "4.25.5",
Locations: []string{"testdata/multiple-packages.v2.lock"},
Metadata: osv.DepGroupMetadata{
DepGroupVals: []string{},
},
},
{
Name: "python-dateutil",
Version: "2.9.0.post0",
Locations: []string{"testdata/multiple-packages.v2.lock"},
Metadata: osv.DepGroupMetadata{
DepGroupVals: []string{"dev", "test"},
},
},
{
Name: "six",
Version: "1.17.0",
Locations: []string{"testdata/multiple-packages.v2.lock"},
Metadata: osv.DepGroupMetadata{
DepGroupVals: []string{},
},
},
{
Name: "typing-extensions",
Version: "4.12.2",
Locations: []string{"testdata/multiple-packages.v2.lock"},
Metadata: osv.DepGroupMetadata{
DepGroupVals: []string{"dev", "test"},
},
},
{
Name: "urllib3",
Version: "2.3.0",
Locations: []string{"testdata/multiple-packages.v2.lock"},
Metadata: osv.DepGroupMetadata{
DepGroupVals: []string{"dev"},
},
},
{
Name: "redis",
Version: "5.2.1",
Locations: []string{"testdata/multiple-packages.v2.lock"},
Metadata: osv.DepGroupMetadata{
DepGroupVals: []string{"optional"},
},
},
},
},
}

for _, tt := range tests {
Expand Down
193 changes: 193 additions & 0 deletions extractor/filesystem/language/python/poetrylock/testdata/multiple-packages.v2.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
# This file is automatically @generated by Poetry 2.0.0 and should not be changed by hand.

[[package]]
name = "async-timeout"
version = "5.0.1"
description = "Timeout context manager for asyncio programs"
optional = true
python-versions = ">=3.8"
groups = ["main"]
markers = "extra == \"caching\" and python_full_version < \"3.11.3\""
files = [
{file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"},
{file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"},
]

[[package]]
name = "factory-boy"
version = "3.3.1"
description = "A versatile test fixtures replacement based on thoughtbot's factory_bot for Ruby."
optional = false
python-versions = ">=3.8"
groups = ["dev"]
files = [
{file = "factory_boy-3.3.1-py2.py3-none-any.whl", hash = "sha256:7b1113c49736e1e9995bc2a18f4dbf2c52cf0f841103517010b1d825712ce3ca"},
{file = "factory_boy-3.3.1.tar.gz", hash = "sha256:8317aa5289cdfc45f9cae570feb07a6177316c82e34d14df3c2e1f22f26abef0"},
]

[package.dependencies]
Faker = ">=0.7.0"

[package.extras]
dev = ["Django", "Pillow", "SQLAlchemy", "coverage", "flake8", "isort", "mongoengine", "mongomock", "mypy", "tox", "wheel (>=0.32.0)", "zest.releaser[recommended]"]
doc = ["Sphinx", "sphinx-rtd-theme", "sphinxcontrib-spelling"]

[[package]]
name = "faker"
version = "33.3.0"
description = "Faker is a Python package that generates fake data for you."
optional = false
python-versions = ">=3.8"
groups = ["dev", "test"]
files = [
{file = "Faker-33.3.0-py3-none-any.whl", hash = "sha256:ae074d9c7ef65817a93b448141a5531a16b2ea2e563dc5774578197c7c84060c"},
{file = "faker-33.3.0.tar.gz", hash = "sha256:2abb551a05b75d268780b6095100a48afc43c53e97422002efbfc1272ebf5f26"},
]

[package.dependencies]
python-dateutil = ">=2.4"
typing-extensions = "*"

[[package]]
name = "proto-plus"
version = "1.22.0"
description = "Beautiful, Pythonic protocol buffers."
optional = false
python-versions = ">=3.6"
groups = ["main"]
markers = "python_version < \"3.10\""
files = [
{file = "proto-plus-1.22.0.tar.gz", hash = "sha256:c2e6693fdf68c405a6428226915a8625d21d0513793598ae3287a1210478d8ec"},
{file = "proto_plus-1.22.0-py3-none-any.whl", hash = "sha256:a27192d8cdc54e044f137b4c9053c9108cf5c065b46d067f1bcd389a911faf5b"},
]

[package.dependencies]
protobuf = ">=3.19.0,<5.0.0dev"

[package.extras]
testing = ["google-api-core[grpc] (>=1.31.5)"]

[[package]]
name = "proto-plus"
version = "1.23.0"
description = "Beautiful, Pythonic protocol buffers."
optional = false
python-versions = ">=3.6"
groups = ["main"]
markers = "python_version >= \"3.10\""
files = [
{file = "proto-plus-1.23.0.tar.gz", hash = "sha256:89075171ef11988b3fa157f5dbd8b9cf09d65fffee97e29ce403cd8defba19d2"},
{file = "proto_plus-1.23.0-py3-none-any.whl", hash = "sha256:a829c79e619e1cf632de091013a4173deed13a55f326ef84f05af6f50ff4c82c"},
]

[package.dependencies]
protobuf = ">=3.19.0,<5.0.0dev"

[package.extras]
testing = ["google-api-core[grpc] (>=1.31.5)"]

[[package]]
name = "protobuf"
version = "4.25.5"
description = ""
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "protobuf-4.25.5-cp310-abi3-win32.whl", hash = "sha256:5e61fd921603f58d2f5acb2806a929b4675f8874ff5f330b7d6f7e2e784bbcd8"},
{file = "protobuf-4.25.5-cp310-abi3-win_amd64.whl", hash = "sha256:4be0571adcbe712b282a330c6e89eae24281344429ae95c6d85e79e84780f5ea"},
{file = "protobuf-4.25.5-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:b2fde3d805354df675ea4c7c6338c1aecd254dfc9925e88c6d31a2bcb97eb173"},
{file = "protobuf-4.25.5-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:919ad92d9b0310070f8356c24b855c98df2b8bd207ebc1c0c6fcc9ab1e007f3d"},
{file = "protobuf-4.25.5-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fe14e16c22be926d3abfcb500e60cab068baf10b542b8c858fa27e098123e331"},
{file = "protobuf-4.25.5-cp38-cp38-win32.whl", hash = "sha256:98d8d8aa50de6a2747efd9cceba361c9034050ecce3e09136f90de37ddba66e1"},
{file = "protobuf-4.25.5-cp38-cp38-win_amd64.whl", hash = "sha256:b0234dd5a03049e4ddd94b93400b67803c823cfc405689688f59b34e0742381a"},
{file = "protobuf-4.25.5-cp39-cp39-win32.whl", hash = "sha256:abe32aad8561aa7cc94fc7ba4fdef646e576983edb94a73381b03c53728a626f"},
{file = "protobuf-4.25.5-cp39-cp39-win_amd64.whl", hash = "sha256:7a183f592dc80aa7c8da7ad9e55091c4ffc9497b3054452d629bb85fa27c2a45"},
{file = "protobuf-4.25.5-py3-none-any.whl", hash = "sha256:0aebecb809cae990f8129ada5ca273d9d670b76d9bfc9b1809f0a9c02b7dbf41"},
{file = "protobuf-4.25.5.tar.gz", hash = "sha256:7f8249476b4a9473645db7f8ab42b02fe1488cbe5fb72fddd445e0665afd8584"},
]

[[package]]
name = "python-dateutil"
version = "2.9.0.post0"
description = "Extensions to the standard Python datetime module"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
groups = ["dev", "test"]
files = [
{file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"},
{file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"},
]

[package.dependencies]
six = ">=1.5"

[[package]]
name = "redis"
version = "5.2.1"
description = "Python client for Redis database and key-value store"
optional = true
python-versions = ">=3.8"
groups = ["main"]
markers = "extra == \"caching\""
files = [
{file = "redis-5.2.1-py3-none-any.whl", hash = "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4"},
{file = "redis-5.2.1.tar.gz", hash = "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f"},
]

[package.dependencies]
async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""}

[package.extras]
hiredis = ["hiredis (>=3.0.0)"]
ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==23.2.1)", "requests (>=2.31.0)"]

[[package]]
name = "six"
version = "1.17.0"
description = "Python 2 and 3 compatibility utilities"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
groups = ["main", "dev", "test"]
files = [
{file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"},
{file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"},
]

[[package]]
name = "typing-extensions"
version = "4.12.2"
description = "Backported and Experimental Type Hints for Python 3.8+"
optional = false
python-versions = ">=3.8"
groups = ["dev", "test"]
files = [
{file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
{file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
]

[[package]]
name = "urllib3"
version = "2.3.0"
description = "HTTP library with thread-safe connection pooling, file post, and more."
optional = false
python-versions = ">=3.9"
groups = ["dev"]
files = [
{file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"},
{file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"},
]

[package.extras]
brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"]
h2 = ["h2 (>=4,<5)"]
socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"]
zstd = ["zstandard (>=0.18.0)"]

[extras]
caching = ["redis"]

[metadata]
lock-version = "2.1"
python-versions = ">=3.9"
content-hash = "ab8a643d59b9404536d1a035b0f7324ca1e2b906b5a69cf6ae8321a65efc3870"
Loading