Merge pull request #712 from romain-intel/feat/improve-poetry-support

Add/improve support for Path dependencies and GIT dependencies

conda_lock/interfaces/vendored_poetry.py

@@ -5,6 +5,12 @@
 from conda_lock._vendor.poetry.core.packages.dependency import (
     Dependency as PoetryDependency,
 )
+from conda_lock._vendor.poetry.core.packages.directory_dependency import (
+    DirectoryDependency as PoetryDirectoryDependency,
+)
+from conda_lock._vendor.poetry.core.packages.file_dependency import (
+    FileDependency as PoetryFileDependency,
+)
 from conda_lock._vendor.poetry.core.packages.package import Package as PoetryPackage
 from conda_lock._vendor.poetry.core.packages.project_package import (
     ProjectPackage as PoetryProjectPackage,
@@ -37,6 +43,8 @@
     "Operation",
     "PoetryDependency",
     "PoetryPackage",
+    "PoetryDirectoryDependency",
+    "PoetryFileDependency",
     "PoetryProjectPackage",
     "PoetrySolver",
     "PoetryURLDependency",

conda_lock/models/lock_spec.py

@@ -43,9 +43,16 @@ class VCSDependency(_BaseDependency):
     source: str
     vcs: str
     rev: Optional[str] = None
+    subdirectory: Optional[str] = None
 
 
-Dependency = Union[VersionedDependency, URLDependency, VCSDependency]
+class PathDependency(_BaseDependency):
+    path: str
+    is_directory: bool
+    subdirectory: Optional[str] = None
+
+
+Dependency = Union[VersionedDependency, URLDependency, VCSDependency, PathDependency]
 
 
 class Package(StrictModel):

conda_lock/pypi_solver.py

@@ -33,6 +33,8 @@
     Link,
     Operation,
     PoetryDependency,
+    PoetryDirectoryDependency,
+    PoetryFileDependency,
     PoetryPackage,
     PoetryProjectPackage,
     PoetrySolver,
@@ -322,6 +324,15 @@ def get_dependency(dep: lock_spec.Dependency) -> PoetryDependency:
             source=dep.source,
             rev=dep.rev,
         )
+    elif isinstance(dep, lock_spec.PathDependency):
+        if dep.is_directory:
+            return PoetryDirectoryDependency(
+                name=dep.name, path=Path(dep.path), extras=dep.extras
+            )
+        else:
+            return PoetryFileDependency(
+                name=dep.name, path=Path(dep.path), extras=dep.extras
+            )
     else:
         raise ValueError(f"Unknown requirement {dep}")
 
@@ -387,6 +398,10 @@ def get_requirements(
                 # TODO: FIXME git ls-remote
                 hash = HashModel(**{"sha256": op.package.source_resolved_reference})
                 source = DependencySource(type="url", url=url)
+            elif op.package.source_type in ("directory", "file"):
+                url = f"file://{op.package.source_url}"
+                hash = HashModel()
+                source = DependencySource(type="url", url=url)
             # Choose the most specific distribution for the target
             # TODO: need to handle  git here
             # https://github.com/conda/conda-lock/blob/ac31f5ddf2951ed4819295238ccf062fb2beb33c/conda_lock/_vendor/poetry/installation/executor.py#L557

conda_lock/src_parser/pyproject_toml.py

@@ -34,6 +34,7 @@
 from conda_lock.models.lock_spec import (
     Dependency,
     LockSpecification,
+    PathDependency,
     PoetryMappedDependencySpec,
     URLDependency,
     VCSDependency,
@@ -128,6 +129,17 @@ def handle_mapping(
     if "git" in depattrs:
         url: Optional[str] = depattrs.get("git", None)
         manager = "pip"
+        # Order is the same as the one used by poetry
+        branch_ident = depattrs.get(
+            "branch", depattrs.get("tag", depattrs.get("rev", None))
+        )
+        if branch_ident is not None:
+            url += "@" + branch_ident
+        if "subdirectory" in depattrs:
+            url += "#subdirectory=" + depattrs["subdirectory"]
+    elif "path" in depattrs:
+        url = depattrs.get("path", None)
+        manager = "pip"
     else:
         poetry_version_spec = depattrs.get("version", None)
         url = depattrs.get("url", None)
@@ -284,7 +296,7 @@ def parse_poetry_pyproject_toml(
             version = poetry_version_to_conda_version(poetry_version_spec)
 
             if "git" in depattrs and url is not None:
-                url, rev = unpack_git_url(url)
+                url, rev, subdir = unpack_git_url(url)
                 dependencies.append(
                     VCSDependency(
                         name=name,
@@ -293,6 +305,20 @@ def parse_poetry_pyproject_toml(
                         manager=manager,
                         vcs="git",
                         rev=rev,
+                        subdirectory=subdir,
+                    )
+                )
+            elif "path" in depattrs and url is not None:
+                path = pathlib.Path(url)
+                path.resolve()
+                is_dir = path.is_dir()
+                dependencies.append(
+                    PathDependency(
+                        name=name,
+                        markers=markers,
+                        path=path.as_posix(),
+                        is_directory=is_dir,
+                        manager=manager,
                     )
                 )
             elif version is None:
@@ -423,20 +449,23 @@ def parse_requirement_specifier(
         return RequirementWithHash(requirement)
 
 
-def unpack_git_url(url: str) -> Tuple[str, Optional[str]]:
+def unpack_git_url(url: str) -> Tuple[str, Optional[str], Optional[str]]:
     if url.endswith(".git"):
         url = url[:-4]
     if url.startswith("git+"):
         url = url[4:]
     rev = None
+    subdir = None
     if "@" in url:
         try:
             url, rev = url.split("@")
         except ValueError:
             # SSH URLs can have multiple @s
             url1, url2, rev = url.split("@")
             url = f"{url1}@{url2}"
-    return url, rev
+    if rev and "#subdirectory=" in rev:
+        rev, subdir = rev.split("#subdirectory=")
+    return url, rev, subdir
 
 
 def parse_python_requirement(
@@ -487,7 +516,15 @@ def parse_python_requirement(
     ... )  # doctest: +NORMALIZE_WHITESPACE
     VCSDependency(name='conda-lock', manager='conda', category='main', extras=[],
         markers=None, source='https://github.com/conda/conda-lock.git', vcs='git',
-        rev='v2.4.1')
+        rev='v2.4.1', subdirectory=None)
+
+    >>> parse_python_requirement(
+    ...     "conda-lock @ git+https://github.com/conda/[email protected]#subdirectory=src",
+    ...     mapping_url=DEFAULT_MAPPING_URL,
+    ... )  # doctest: +NORMALIZE_WHITESPACE
+    VCSDependency(name='conda-lock', manager='conda', category='main', extras=[],
+        markers=None, source='https://github.com/conda/conda-lock.git', vcs='git',
+        rev='v2.4.1', subdirectory='src')
 
     >>> parse_python_requirement(
     ...     "some-package @ https://some-repository.org/some-package-1.2.3.tar.gz",
@@ -504,6 +541,24 @@ def parse_python_requirement(
     VersionedDependency(name='some-package', manager='conda', category='main',
         extras=[], markers="sys_platform == 'darwin'", version='*', build=None,
         conda_channel=None, hash=None)
+
+    >>> parse_python_requirement(
+    ...     "mypkg @ /path/to/some-package",
+    ...     manager="pip",
+    ...     mapping_url=DEFAULT_MAPPING_URL,
+    ... )  # doctest: +NORMALIZE_WHITESPACE
+    PathDependency(name='mypkg', manager='pip', category='main',
+        extras=[], markers=None, path='/path/to/some-package', is_directory=False,
+        subdirectory=None)
+
+    >>> parse_python_requirement(
+    ...     "mypkg @ file:///path/to/some-package",
+    ...     manager="pip",
+    ...     mapping_url=DEFAULT_MAPPING_URL,
+    ... )  # doctest: +NORMALIZE_WHITESPACE
+    PathDependency(name='mypkg', manager='pip', category='main',
+        extras=[], markers=None, path='/path/to/some-package', is_directory=False,
+        subdirectory=None)
     """
     if ";" in requirement:
         requirement, markers = (s.strip() for s in requirement.rsplit(";", 1))
@@ -523,7 +578,7 @@ def parse_python_requirement(
     extras = list(parsed_req.extras)
 
     if parsed_req.url and parsed_req.url.startswith("git+"):
-        url, rev = unpack_git_url(parsed_req.url)
+        url, rev, subdir = unpack_git_url(parsed_req.url)
         return VCSDependency(
             name=conda_dep_name,
             source=url,
@@ -532,17 +587,39 @@ def parse_python_requirement(
             vcs="git",
             rev=rev,
             markers=markers,
+            subdirectory=subdir,
         )
     elif parsed_req.url:
         assert conda_version in {"", "*", None}
-        url, frag = urldefrag(parsed_req.url)
-        return URLDependency(
+        if (
+            parsed_req.url.startswith("git+")
+            or parsed_req.url.startswith("https://")
+            or parsed_req.url.startswith("ssh://")
+        ):
+            url, frag = urldefrag(parsed_req.url)
+            return URLDependency(
+                name=conda_dep_name,
+                manager=manager,
+                category=category,
+                extras=extras,
+                url=url,
+                hashes=[frag.replace("=", ":")],
+                markers=markers,
+            )
+        # Local file/directory URL
+        url = parsed_req.url
+        if url.startswith("file://"):
+            url = url[7:]
+        path = pathlib.Path(url)
+        path.resolve()
+        is_dir = path.is_dir()
+        return PathDependency(
             name=conda_dep_name,
             manager=manager,
             category=category,
             extras=extras,
-            url=url,
-            hashes=[frag.replace("=", ":")],
+            path=path.as_posix(),
+            is_directory=is_dir,
             markers=markers,
         )
     else:

tests/test-poetry-git-subdir/pyproject.toml

@@ -0,0 +1,19 @@
+[tool.poetry]
+name = "conda-lock-test-poetry"
+version = "0.0.1"
+description = ""
+authors = ["conda-lock"]
+
+[tool.poetry.dependencies]
+python = "^3.7"
+requests = "^2.13.0"
+toml = ">=0.10"
+tomlkit = { version = ">=0.7.0,<1.0.0", optional = true }
+
+[tool.poetry.dev-dependencies]
+pytest = "~5.1.0"
+tensorflow = { git = "git+https://github.com/tensorflow/[email protected]", subdirectory = "tensorflow/tools/pip_package" }
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"

tests/test-poetry-path/.gitignore

@@ -0,0 +1 @@
+*.tar.gz

tests/test-poetry-path/fake-private-package-1.0.0/PKG-INFO

@@ -0,0 +1,3 @@
+Metadata-Version: 2.1
+Name: fake-private-package
+Version: 1.0.0

tests/test-poetry-path/fake-private-package-1.0.0/setup.cfg

@@ -0,0 +1,3 @@
+[egg_info]
+tag_build =
+tag_date = 0

tests/test-poetry-path/fake-private-package-1.0.0/setup.py

@@ -0,0 +1,7 @@
+from setuptools import setup
+
+
+setup(
+    name="fake-private-package",
+    version="1.0.0",
+)

tests/test-poetry-path/pyproject.toml

@@ -0,0 +1,14 @@
+[tool.poetry]
+name = "conda-lock-test-poetry"
+version = "0.0.1"
+description = ""
+authors = ["conda-lock"]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+pytest = "~5.1.0"
+fake-private-package = { path = "./fake-private-package-1.0.0" }
+
+[build-system]
+requires = ["poetry>=0.12"]
+build-backend = "poetry.masonry.api"

tests/test_conda_lock.py

@@ -63,7 +63,12 @@
 )
 from conda_lock.lookup import DEFAULT_MAPPING_URL, conda_name_to_pypi_name
 from conda_lock.models.channel import Channel
-from conda_lock.models.lock_spec import Dependency, VCSDependency, VersionedDependency
+from conda_lock.models.lock_spec import (
+    Dependency,
+    PathDependency,
+    VCSDependency,
+    VersionedDependency,
+)
 from conda_lock.models.pip_repository import PipRepository
 from conda_lock.pypi_solver import (
     MANYLINUX_TAGS,
@@ -224,6 +229,16 @@ def poetry_pyproject_toml_git(tmp_path: Path):
     return clone_test_dir("test-poetry-git", tmp_path).joinpath("pyproject.toml")
 
 
+@pytest.fixture
+def poetry_pyproject_toml_git_subdir(tmp_path: Path):
+    return clone_test_dir("test-poetry-git-subdir", tmp_path).joinpath("pyproject.toml")
+
+
+@pytest.fixture
+def poetry_pyproject_toml_path(tmp_path: Path):
+    return clone_test_dir("test-poetry-path", tmp_path).joinpath("pyproject.toml")
+
+
 @pytest.fixture
 def poetry_pyproject_toml_no_pypi(tmp_path: Path):
     return clone_test_dir("test-poetry-no-pypi", tmp_path).joinpath("pyproject.toml")
@@ -803,6 +818,35 @@ def test_parse_poetry_git(poetry_pyproject_toml_git: Path):
     assert isinstance(specs["pydantic"], VCSDependency)
     assert specs["pydantic"].vcs == "git"
     assert specs["pydantic"].rev == "v2.0b2"
+    assert specs["pydantic"].subdirectory is None
+
+
+def test_parse_poetry_git_subdir(poetry_pyproject_toml_git_subdir: Path):
+    res = parse_pyproject_toml(
+        poetry_pyproject_toml_git_subdir,
+        platforms=["linux-64"],
+        mapping_url=DEFAULT_MAPPING_URL,
+    )
+
+    specs = {dep.name: dep for dep in res.dependencies["linux-64"]}
+
+    assert isinstance(specs["tensorflow"], VCSDependency)
+    assert specs["tensorflow"].vcs == "git"
+    assert specs["tensorflow"].subdirectory == "tensorflow/tools/pip_package"
+    assert specs["tensorflow"].rev == "v2.17.0"
+
+
+def test_parse_poetry_path(poetry_pyproject_toml_path: Path):
+    res = parse_pyproject_toml(
+        poetry_pyproject_toml_path,
+        platforms=["linux-64"],
+        mapping_url=DEFAULT_MAPPING_URL,
+    )
+
+    specs = {dep.name: dep for dep in res.dependencies["linux-64"]}
+
+    assert isinstance(specs["fake-private-package"], PathDependency)
+    assert specs["fake-private-package"].path == "fake-private-package-1.0.0"
 
 
 def test_parse_poetry_no_pypi(poetry_pyproject_toml_no_pypi: Path):