From e6988eecd5d92aa8093f712db38f1c03f52230de Mon Sep 17 00:00:00 2001 From: Francisco Ramirez de Anton Date: Tue, 12 Mar 2024 22:27:29 +0100 Subject: [PATCH 1/8] Added fetch_libraries function --- conan/tools/google/bazeldeps.py | 83 +------- conans/client/tools/files.py | 87 +++++++++ .../tools/files/test_collect_libs2.py | 184 ++++++++++++++++++ .../test/unittests/tools/google/test_bazel.py | 73 +------ 4 files changed, 274 insertions(+), 153 deletions(-) create mode 100644 conans/test/unittests/tools/files/test_collect_libs2.py diff --git a/conan/tools/google/bazeldeps.py b/conan/tools/google/bazeldeps.py index 7b3438e776c..7200f517500 100644 --- a/conan/tools/google/bazeldeps.py +++ b/conan/tools/google/bazeldeps.py @@ -7,6 +7,7 @@ from conan.errors import ConanException from conan.tools._check_build_profile import check_using_build_profile +from conans.client.tools import fetch_libraries from conans.util.files import save _BazelTargetInfo = namedtuple("DepInfo", ['repository_name', 'name', 'requires', 'cpp_info']) @@ -71,86 +72,6 @@ def _get_requirements(conanfile, build_context_activated): yield require, dep -def _get_libs(dep, cpp_info=None) -> list: - """ - Get the static/shared library paths - - :param dep: normally a - :param cpp_info: of the component. - :return: list of tuples per static/shared library -> - [(lib_name, is_shared, library_path, interface_library_path)] - Note: ``library_path`` could be both static and shared ones in case of UNIX systems. - Windows would have: - * shared: library_path as DLL, and interface_library_path as LIB - * static: library_path as LIB, and interface_library_path as None - """ - def _is_shared(): - """ - Checking traits and shared option - """ - default_value = dep.options.get_safe("shared") if dep.options else False - # Conan 2.x - # return {"shared-library": True, - # "static-library": False}.get(str(dep.package_type), default_value) - return default_value - - def _save_lib_path(lib_, lib_path_): - """Add each lib with its full library path""" - formatted_path = lib_path_.replace("\\", "/") - _, ext_ = os.path.splitext(formatted_path) - if is_shared and ext_ == ".lib": # Windows interface library - interface_lib_paths[lib_] = formatted_path - else: - lib_paths[lib_] = formatted_path - - cpp_info = cpp_info or dep.cpp_info - is_shared = _is_shared() - libdirs = cpp_info.libdirs - bindirs = cpp_info.bindirs if is_shared else [] # just want to get shared libraries - libs = cpp_info.libs[:] # copying the values - lib_paths = {} - interface_lib_paths = {} - for libdir in set(libdirs + bindirs): - if not os.path.exists(libdir): - continue - files = os.listdir(libdir) - for f in files: - full_path = os.path.join(libdir, f) - if not os.path.isfile(full_path): # Make sure that directories are excluded - continue - name, ext = os.path.splitext(f) - # Users may not name their libraries in a conventional way. For example, directly - # use the basename of the lib file as lib name, e.g., cpp_info.libs = ["liblib1.a"] - # Issue related: https://github.com/conan-io/conan/issues/11331 - if ext and f in libs: # let's ensure that it has any extension - _save_lib_path(f, full_path) - continue - if name not in libs and name.startswith("lib"): - name = name[3:] # libpkg -> pkg - # FIXME: Should it read a conf variable to know unexpected extensions? - if (is_shared and ext in (".so", ".dylib", ".lib", ".dll")) or \ - (not is_shared and ext in (".a", ".lib")): - if name in libs: - _save_lib_path(name, full_path) - continue - else: # last chance: some cases the name could be pkg.if instead of pkg - name = name.split(".", maxsplit=1)[0] - if name in libs: - _save_lib_path(name, full_path) - - libraries = [] - for lib, lib_path in lib_paths.items(): - interface_lib_path = None - if lib_path.endswith(".dll"): - if lib not in interface_lib_paths: - raise ConanException(f"Windows needs a .lib for link-time and .dll for runtime." - f" Only found {lib_path}") - interface_lib_path = interface_lib_paths.pop(lib) - libraries.append((lib, is_shared, lib_path, interface_lib_path)) - # TODO: Would we want to manage the cases where DLLs are provided by the system? - return libraries - - def _get_headers(cpp_info, package_folder_path): return ['"{}/**"'.format(_relativize_path(path, package_folder_path)) for path in cpp_info.includedirs] @@ -417,7 +338,7 @@ def fill_info(info): # os_build = self._dep.settings_build.get_safe("os") os_build = self._dep.settings.get_safe("os") linkopts = _get_linkopts(cpp_info, os_build) - libs = _get_libs(self._dep, cpp_info) + libs = fetch_libraries(self._dep, cpp_info) libs_info = [] bindirs = [_relativize_path(bindir, package_folder_path) for bindir in cpp_info.bindirs] diff --git a/conans/client/tools/files.py b/conans/client/tools/files.py index a8444b23c07..cf837f64a0d 100644 --- a/conans/client/tools/files.py +++ b/conans/client/tools/files.py @@ -1,6 +1,7 @@ import gzip import logging import os +import pathlib import platform import stat import subprocess @@ -19,6 +20,8 @@ UNIT_SIZE = 1000.0 # Library extensions supported by collect_libs VALID_LIB_EXTENSIONS = (".so", ".lib", ".a", ".dylib", ".bc") +# Library extensions supported by fetch_libraries +EXTENDED_VALID_LIB_EXTENSIONS = (".so", ".lib", ".a", ".dylib", ".bc", ".dll") @contextmanager @@ -378,6 +381,90 @@ def collect_libs(conanfile, folder=None): return result +def fetch_libraries(conanfile, cpp_info=None, extra_folders=None, raise_error=True) -> list[tuple[str, str, str]]: + """ + Get the static/shared library paths. Analyze if DLLs are present and raise an exception if + + + :param conanfile: normally a + :param cpp_info: of the component. + :param extra_folders: list of relative (to `package_folder`) folders to search more libraries. + :param raise_error: if raise_error == True, it'll raise an exception if DLL lib does not have + an associated *.lib interface library. + :return: list of tuples per static/shared library -> + [(lib_name, is_shared, library_path, interface_library_path)] + Note: ``library_path`` could be both static and shared ones in case of UNIX systems. + Windows would have: + * shared: library_path as DLL, and interface_library_path as LIB + * static: library_path as LIB, and interface_library_path as None + """ + def _save_lib_path(lib_, lib_path_): + """Add each lib with its full library path""" + formatted_path = lib_path_.replace("\\", "/") + # In case of symlinks, only keep the shortest file name in the same "group" + if os.path.islink(lib_path_): + # If libmylib.dylib -> lib/libmylib.1.dylib, this gets libmylib.1.dylib + real_lib = os.path.basename(os.path.realpath(lib_path_)) + if real_lib not in symlink_paths or len(lib_) < len(symlink_paths[lib_]): + symlink_paths[lib_] = real_lib + + _, ext_ = os.path.splitext(formatted_path) + if ext_ == ".lib": # Possibly Windows interface library, putting it apart from the rest + interface_lib_paths[lib_] = formatted_path + else: + lib_paths[lib_] = formatted_path + + cpp_info = cpp_info or conanfile.cpp_info + libdirs = cpp_info.libdirs + bindirs = cpp_info.bindirs # just want to get shared libraries (likely DLLs) + libs = cpp_info.libs[:] # copying the values if they are already declared + lib_paths = {} + interface_lib_paths = {} + symlink_paths = {} + # Composing absolute folders if extra folders were passed + extra_paths = [os.path.join(conanfile.package_folder, path) for path in extra_folders or []] + for libdir in set(libdirs + bindirs + (extra_paths or [])): + if not os.path.exists(libdir): + continue + files = os.listdir(libdir) + for f in files: + full_path = os.path.join(libdir, f) + if not os.path.isfile(full_path): # Make sure that directories are excluded + continue + + name, ext = os.path.splitext(f) + # Users may not name their libraries in a conventional way. For example, directly + # use the basename of the lib file as lib name, e.g., cpp_info.libs = ["liblib1.a"] + # Issue related: https://github.com/conan-io/conan/issues/11331 + if ext and f in libs: # let's ensure that it has any extension + _save_lib_path(f, full_path) + continue + if name not in libs and name.startswith("lib"): + name = name[3:] # libpkg -> pkg + if ext in EXTENDED_VALID_LIB_EXTENSIONS: + if not libs: # we want to save all the libs + _save_lib_path(name, full_path) + else: + # Alternative name: in some cases the name could be pkg.if instead of pkg + alternative_name = name.split(".", maxsplit=1)[0] + if name in libs or alternative_name in libs: + _save_lib_path(name, full_path) + + libraries = [] + for lib, lib_path in lib_paths.items(): + interface_lib_path = "" + if lib_path.endswith(".dll"): + if lib not in interface_lib_paths and raise_error: + raise ConanException(f"Windows needs a .lib for link-time and .dll for runtime." + f" Only found {lib_path}") + interface_lib_path = interface_lib_paths.pop(lib) + libraries.append((lib, lib_path, interface_lib_path)) + if interface_lib_paths: # Rest of static .lib Windows libraries + libraries.extend([(lib_name, lib_path, "") for lib_name, lib_path in interface_lib_paths.items()]) + libraries.sort() + return libraries + + def which(filename): """ same affect as posix which command or shutil.which from python3 """ # FIXME: Replace with shutil.which in Conan 2.0 diff --git a/conans/test/unittests/tools/files/test_collect_libs2.py b/conans/test/unittests/tools/files/test_collect_libs2.py new file mode 100644 index 00000000000..ba0a5836a3b --- /dev/null +++ b/conans/test/unittests/tools/files/test_collect_libs2.py @@ -0,0 +1,184 @@ +import os +import platform +from unittest.mock import MagicMock + +import pytest + +from conan.tools.files import save +from conans.client.tools import fetch_libraries +from conans.model.build_info import CppInfo +from conans.test.utils.mocks import ConanFileMock +from conans.test.utils.test_files import temp_folder + + +@pytest.fixture(scope="module") +def cpp_info(): + folder = temp_folder(path_with_spaces=False) + bindirs = os.path.join(folder, "bin") + libdirs = os.path.join(folder, "lib") + # Shared Windows with interface ending with .if.lib + save(ConanFileMock(), os.path.join(bindirs, "mylibwin2.dll"), "") + save(ConanFileMock(), os.path.join(libdirs, "mylibwin2.if.lib"), "") + # Shared Windows + save(ConanFileMock(), os.path.join(bindirs, "mylibwinsh.dll"), "") + save(ConanFileMock(), os.path.join(libdirs, "mylibwinsh.lib"), "") + # Static Windows + save(ConanFileMock(), os.path.join(libdirs, "mylibwinst.lib"), "") + # Shared macOS + save(ConanFileMock(), os.path.join(bindirs, "mylibmacsh.dylib"), "") + # Binary + save(ConanFileMock(), os.path.join(bindirs, "protoc"), "") + # Static Linux and macOS + save(ConanFileMock(), os.path.join(libdirs, "myliblin.a"), "") + save(ConanFileMock(), os.path.join(libdirs, "mylibmacst.a"), "") + # Shared Linux and shared name starting with "lib" + save(ConanFileMock(), os.path.join(bindirs, "myliblinsh.so"), "") + save(ConanFileMock(), os.path.join(libdirs, "libmylibsh.so"), "") + # Recursive folder + save(ConanFileMock(), os.path.join(libdirs, "subfolder", "libmylib.a"), "") + cpp_info_mock = MagicMock(_base_folder=None, libdirs=None, bindirs=None, libs=None) + cpp_info_mock._base_folder = folder.replace("\\", "/") + cpp_info_mock.libdirs = [libdirs] + cpp_info_mock.bindirs = [bindirs] + return cpp_info_mock + + +@pytest.mark.parametrize("libs, expected", [ + # expected == (lib_name, is_shared, library_path, interface_library_path) + (["mylibwinst"], [('mylibwinst', '{base_folder}/lib/mylibwinst.lib', "")]), + # Win + shared + (["mylibwinsh"], [('mylibwinsh', '{base_folder}/bin/mylibwinsh.dll', '{base_folder}/lib/mylibwinsh.lib')]), + # Win + shared (interface with another ext) + (["mylibwin2"], + [('mylibwin2', '{base_folder}/bin/mylibwin2.dll', '{base_folder}/lib/mylibwin2.if.lib')]), + # Win + Mac + shared + (["mylibwinsh", "mylibmacsh"], [('mylibmacsh', '{base_folder}/bin/mylibmacsh.dylib', ""), + ('mylibwinsh', '{base_folder}/bin/mylibwinsh.dll', '{base_folder}/lib/mylibwinsh.lib')]), + # Linux + Mac + static + (["myliblin", "mylibmacst"], [('mylibmacst', '{base_folder}/lib/mylibmacst.a', ""), + ('myliblin', '{base_folder}/lib/myliblin.a', "")]), + # mylib + shared (saved as libmylib.so) -> removing the leading "lib" if it matches + (["mylibsh"], [('mylibsh', '{base_folder}/lib/libmylibsh.so', "")]), + # mylib + static (saved in a subfolder subfolder/libmylib.a) -> non-recursive at this moment + (["mylib"], []), + # no lib matching + (["noexist"], []), + # no lib matching + Win + static + (["noexist", "mylibwinst"], [('mylibwinst', '{base_folder}/lib/mylibwinst.lib', "")]), + # protobuf (Issue related https://github.com/conan-io/conan/issues/15390) + (["protoc"], []), + # non-conventional library name (Issue related https://github.com/conan-io/conan/pull/11343) + (["libmylibsh.so"], [('libmylibsh.so', '{base_folder}/lib/libmylibsh.so', "")]), +]) +def test_fetch_libraries(cpp_info, libs, expected): + cpp_info.libs = libs + ret = [] + for (lib, lib_path, interface_lib_path) in expected: + if lib_path: + lib_path = lib_path.format(base_folder=cpp_info._base_folder) + if interface_lib_path: + interface_lib_path = interface_lib_path.format(base_folder=cpp_info._base_folder) + ret.append((lib, lib_path, interface_lib_path)) + found_libs = fetch_libraries(ConanFileMock(), cpp_info) + ret.sort() + assert found_libs == ret + + +@pytest.mark.skipif(platform.system() == "Windows", reason="Needs symlinks support") +def test_fetch_libraries_symlinks(): + """ + Tests how fetch_libraries function saves the shortest path (symlink) defined + + Folder tree structure: + + . + ├── custom_folder + │ └── libmylib.3.dylib + └── lib + ├── libmylib.1.0.0.dylib + ├── libmylib.1.dylib -> lib/libmylib.1.0.0.dylib + ├── libmylib.2.dylib + └── libmylib.dylib -> lib/libmylib.1.dylib + """ + # Keep only the shortest lib name per group of symlinks + conanfile = ConanFileMock(options_values={"shared": True}) + conanfile.folders.set_base_package(temp_folder()) + conanfile.cpp_info = CppInfo(conanfile.name, "") + # Lib dirs and libraries + lib_folder = os.path.join(conanfile.package_folder, "lib") + custom_folder = os.path.join(conanfile.package_folder, "custom_folder") + version_mylib_path = os.path.join(lib_folder, "libmylib.1.0.0.dylib") + soversion_mylib_path = os.path.join(lib_folder, "libmylib.1.dylib") + lib_mylib_path = os.path.join(lib_folder, "libmylib.dylib") + lib_mylib2_path = os.path.join(lib_folder, "libmylib.2.dylib") + lib_mylib3_path = os.path.join(custom_folder, "libmylib.3.dylib") + save(conanfile, version_mylib_path, "") + os.symlink(version_mylib_path, soversion_mylib_path) # libmylib.1.dylib + os.symlink(soversion_mylib_path, lib_mylib_path) # libmylib.dylib + save(conanfile, lib_mylib2_path, "") + save(conanfile, lib_mylib3_path, "") + conanfile.cpp_info.libdirs = [lib_folder, custom_folder] + result = fetch_libraries(conanfile) + assert ["mylib", "mylib.2", "mylib.3"] == result + + +def test_basic_fetch_libraries(): + conanfile = ConanFileMock() + conanfile.cpp_info = CppInfo(conanfile.name, "") + # Without package_folder + result = fetch_libraries(conanfile) + assert [] == result + + # Default behavior + conanfile.folders.set_base_package(temp_folder()) + mylib_path = os.path.join(conanfile.package_folder, "lib", "mylib.lib") + save(conanfile, mylib_path, "") + + result = fetch_libraries(conanfile) + assert ["mylib"] == result + + # Custom folder + customlib_path = os.path.join(conanfile.package_folder, "custom_folder", "customlib.lib") + save(conanfile, customlib_path, "") + result = fetch_libraries(conanfile, extra_folders=["custom_folder"]) + assert ["customlib"] == result + + # Custom folder doesn't exist + result = fetch_libraries(conanfile, extra_folders=["fake_folder"]) + assert [] == result + assert "Lib folder doesn't exist, can't collect libraries:" in conanfile.output + + # Use cpp_info.libdirs + conanfile.cpp_info.libdirs = ["lib", "custom_folder"] + result = fetch_libraries(conanfile) + assert ["customlib", "mylib"] == result + + # Custom folder with multiple libdirs should only collect from custom folder + assert ["lib", "custom_folder"] == conanfile.cpp_info.libdirs + result = fetch_libraries(conanfile, extra_folders=["custom_folder"]) + assert ["customlib"] == result + + # Unicity of lib names + conanfile = ConanFileMock() + conanfile.folders.set_base_package(temp_folder()) + conanfile.cpp_info = CppInfo(conanfile.name, "") + custom_mylib_path = os.path.join(conanfile.package_folder, "custom_folder", "mylib.lib") + lib_mylib_path = os.path.join(conanfile.package_folder, "lib", "mylib.lib") + save(conanfile, custom_mylib_path, "") + save(conanfile, lib_mylib_path, "") + conanfile.cpp_info.libdirs = ["lib", "custom_folder"] + result = fetch_libraries(conanfile) + assert ["mylib"] == result + + # Warn lib folder does not exist with correct result + conanfile = ConanFileMock() + conanfile.folders.set_base_package(temp_folder()) + conanfile.cpp_info = CppInfo(conanfile.name, "") + lib_mylib_path = os.path.join(conanfile.package_folder, "lib", "mylib.lib") + save(conanfile, lib_mylib_path, "") + no_folder_path = os.path.join(conanfile.package_folder, "no_folder") + conanfile.cpp_info.libdirs = ["no_folder", "lib"] # 'no_folder' does NOT exist + result = fetch_libraries(conanfile) + assert ["mylib"] == result + assert "WARN: Lib folder doesn't exist, can't collect libraries: %s" % no_folder_path \ + in conanfile.output diff --git a/conans/test/unittests/tools/google/test_bazel.py b/conans/test/unittests/tools/google/test_bazel.py index 1dc015cf6a6..df8ce1dbcaa 100644 --- a/conans/test/unittests/tools/google/test_bazel.py +++ b/conans/test/unittests/tools/google/test_bazel.py @@ -1,37 +1,10 @@ -import os import platform -from unittest.mock import MagicMock import pytest -from conan.tools.files import save from conan.tools.google import Bazel -from conan.tools.google.bazeldeps import _relativize_path, _get_libs +from conan.tools.google.bazeldeps import _relativize_path from conans.test.utils.mocks import ConanFileMock -from conans.test.utils.test_files import temp_folder - - -@pytest.fixture(scope="module") -def cpp_info(): - folder = temp_folder(path_with_spaces=False) - bindirs = os.path.join(folder, "bin") - libdirs = os.path.join(folder, "lib") - save(ConanFileMock(), os.path.join(bindirs, "mylibwin.dll"), "") - save(ConanFileMock(), os.path.join(bindirs, "mylibwin2.dll"), "") - save(ConanFileMock(), os.path.join(bindirs, "myliblin.so"), "") - save(ConanFileMock(), os.path.join(bindirs, "mylibmac.dylib"), "") - save(ConanFileMock(), os.path.join(bindirs, "protoc"), "") # binary - save(ConanFileMock(), os.path.join(libdirs, "myliblin.a"), "") - save(ConanFileMock(), os.path.join(libdirs, "mylibmac.a"), "") - save(ConanFileMock(), os.path.join(libdirs, "mylibwin.lib"), "") - save(ConanFileMock(), os.path.join(libdirs, "mylibwin2.if.lib"), "") - save(ConanFileMock(), os.path.join(libdirs, "libmylib.so"), "") - save(ConanFileMock(), os.path.join(libdirs, "subfolder", "libmylib.a"), "") # recursive - cpp_info_mock = MagicMock(_base_folder=None, libdirs=None, bindirs=None, libs=None) - cpp_info_mock._base_folder = folder.replace("\\", "/") - cpp_info_mock.libdirs = [libdirs] - cpp_info_mock.bindirs = [bindirs] - return cpp_info_mock @pytest.mark.skipif(platform.system() == "Windows", reason="Remove this skip for Conan 2.x" @@ -77,47 +50,3 @@ def test_bazel_command_with_config_values(): ]) def test_bazeldeps_relativize_path(path, pattern, expected): assert _relativize_path(path, pattern) == expected - - -@pytest.mark.parametrize("libs, is_shared, expected", [ - # expected == (lib_name, is_shared, library_path, interface_library_path) - (["mylibwin"], False, [('mylibwin', False, '{base_folder}/lib/mylibwin.lib', None)]), - # Win + shared - (["mylibwin"], True, [('mylibwin', True, '{base_folder}/bin/mylibwin.dll', '{base_folder}/lib/mylibwin.lib')]), - # Win + shared (interface with another ext) - (["mylibwin2"], True, - [('mylibwin2', True, '{base_folder}/bin/mylibwin2.dll', '{base_folder}/lib/mylibwin2.if.lib')]), - # Win + Mac + shared - (["mylibwin", "mylibmac"], True, [('mylibmac', True, '{base_folder}/bin/mylibmac.dylib', None), - ('mylibwin', True, '{base_folder}/bin/mylibwin.dll', - '{base_folder}/lib/mylibwin.lib')]), - # Linux + Mac + static - (["myliblin", "mylibmac"], False, [('mylibmac', False, '{base_folder}/lib/mylibmac.a', None), - ('myliblin', False, '{base_folder}/lib/myliblin.a', None)]), - # mylib + shared (saved as libmylib.so) -> removing the leading "lib" if it matches - (["mylib"], True, [('mylib', True, '{base_folder}/lib/libmylib.so', None)]), - # mylib + static (saved in a subfolder subfolder/libmylib.a) -> non-recursive at this moment - (["mylib"], False, []), - # no lib matching - (["noexist"], False, []), - # no lib matching + Win + static - (["noexist", "mylibwin"], False, [('mylibwin', False, '{base_folder}/lib/mylibwin.lib', None)]), - # protobuf (Issue related https://github.com/conan-io/conan/issues/15390) - (["protoc"], True, []), - # non-conventional library name (Issue related https://github.com/conan-io/conan/pull/11343) - (["libmylib.so"], True, [('libmylib.so', True, '{base_folder}/lib/libmylib.so', None)]), -]) -def test_bazeldeps_get_libs(cpp_info, libs, is_shared, expected): - cpp_info.libs = libs - ret = [] - for (lib, is_shared, lib_path, interface_lib_path) in expected: - if lib_path: - lib_path = lib_path.format(base_folder=cpp_info._base_folder) - if interface_lib_path: - interface_lib_path = interface_lib_path.format(base_folder=cpp_info._base_folder) - ret.append((lib, is_shared, lib_path, interface_lib_path)) - found_libs = _get_libs(ConanFileMock(options_values={"shared": is_shared}), - cpp_info) - found_libs.sort() - ret.sort() - assert found_libs == ret From 60e2f48bfdc347e80c401f4c34535d7bf655b36d Mon Sep 17 00:00:00 2001 From: Francisco Ramirez de Anton Date: Tue, 12 Mar 2024 22:28:27 +0100 Subject: [PATCH 2/8] renamed --- .../files/{test_collect_libs2.py => test_fetch_libraries.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename conans/test/unittests/tools/files/{test_collect_libs2.py => test_fetch_libraries.py} (100%) diff --git a/conans/test/unittests/tools/files/test_collect_libs2.py b/conans/test/unittests/tools/files/test_fetch_libraries.py similarity index 100% rename from conans/test/unittests/tools/files/test_collect_libs2.py rename to conans/test/unittests/tools/files/test_fetch_libraries.py From 9745601b108a1b02244e76d3e6221e9b97224836 Mon Sep 17 00:00:00 2001 From: Francisco Ramirez de Anton Date: Wed, 13 Mar 2024 10:39:05 +0100 Subject: [PATCH 3/8] Typo --- conans/client/tools/files.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conans/client/tools/files.py b/conans/client/tools/files.py index cf837f64a0d..27aca8f570a 100644 --- a/conans/client/tools/files.py +++ b/conans/client/tools/files.py @@ -385,14 +385,13 @@ def fetch_libraries(conanfile, cpp_info=None, extra_folders=None, raise_error=Tr """ Get the static/shared library paths. Analyze if DLLs are present and raise an exception if - :param conanfile: normally a :param cpp_info: of the component. :param extra_folders: list of relative (to `package_folder`) folders to search more libraries. :param raise_error: if raise_error == True, it'll raise an exception if DLL lib does not have an associated *.lib interface library. :return: list of tuples per static/shared library -> - [(lib_name, is_shared, library_path, interface_library_path)] + [(lib_name, library_path, interface_library_path)] Note: ``library_path`` could be both static and shared ones in case of UNIX systems. Windows would have: * shared: library_path as DLL, and interface_library_path as LIB From dda6b6691dc483fe8f270711520599f8118b6aee Mon Sep 17 00:00:00 2001 From: Francisco Ramirez de Anton Date: Wed, 13 Mar 2024 13:17:42 +0100 Subject: [PATCH 4/8] wip --- conan/tools/files/__init__.py | 2 +- conan/tools/files/files.py | 110 +++++++++++++++++- conan/tools/google/bazeldeps.py | 2 +- conans/client/tools/files.py | 86 -------------- .../tools/files/test_fetch_libraries.py | 45 ++++--- 5 files changed, 136 insertions(+), 109 deletions(-) diff --git a/conan/tools/files/__init__.py b/conan/tools/files/__init__.py index a697136753c..8a073d578dc 100644 --- a/conan/tools/files/__init__.py +++ b/conan/tools/files/__init__.py @@ -1,6 +1,6 @@ from conan.tools.files.files import load, save, mkdir, rmdir, rm, ftp_download, download, get, \ rename, chdir, unzip, replace_in_file, collect_libs, check_md5, check_sha1, check_sha256, \ - move_folder_contents + move_folder_contents, fetch_libraries from conan.tools.files.patches import patch, apply_conandata_patches, export_conandata_patches from conan.tools.files.cpp_package import CppPackage diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index 3a1d9cacd08..a8b0558e0f0 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -9,16 +9,16 @@ import sys from contextlib import contextmanager from fnmatch import fnmatch - -import six +from types import NoneType from urllib.parse import urlparse from urllib.request import url2pathname +import six + from conan.tools import CONAN_TOOLCHAIN_ARGS_FILE, CONAN_TOOLCHAIN_ARGS_SECTION from conans.client.downloaders.download import run_downloader from conans.errors import ConanException from conans.util.files import rmdir as _internal_rmdir -from conans.util.runners import check_output_runner if six.PY3: # Remove this IF in develop2 from shutil import which @@ -543,6 +543,110 @@ def collect_libs(conanfile, folder=None): return result +def fetch_libraries(conanfile, cpp_info=None, cpp_info_libs=None, + extra_folders=None, win_interface_error=False) -> list[tuple[str, str, str, str]]: + """ + Get all the static/shared library paths, or those associated with the ``cpp_info.libs`` or the given + by ``cpp_info_libs`` parameter (useful to search libraries associated to a library name/es). + If Windows, it analyzes if the DLLs are present and raises an exception if the interface library + is not present (if ``win_interface_error == True``). + + :param conanfile: normally a ````. + :param cpp_info: Likely ```` of the component. + :param cpp_info_libs: list of associated libraries to search. If an empty list is passed, + the function will look for all the libraries in the folders. + Otherwise, if ``None``, it'll use the ``cpp_info.libs``. + :param extra_folders: list of relative folders (relative to `package_folder`) to search more libraries. + :param win_interface_error: if ``raise_error == True``, it'll raise an exception if DLL lib does not have + an associated *.lib interface library. + :return: list of tuples per static/shared library -> + [(lib_name, library_path, interface_library_path, symlink_library_path)] + Note: ``library_path`` could be both static and shared ones in case of UNIX systems. + Windows would have: + * shared: library_path as DLL, and interface_library_path as LIB + * static: library_path as LIB, and interface_library_path as "" + """ + def _save_lib_path(lib_, lib_path_): + """Add each lib with its full library path""" + if lib_ in lib_paths: + conanfile.output.warn(f"{lib_} name is duplicated, but it could have a different path: " + f"{lib_path_} or even be a symlink. If any doubt, call the " + f"fetch_libraries function with cpp_info_libs=[] to gather all the" + f" existing ones and compare both results.") + formatted_path = lib_path_.replace("\\", "/") + # In case of symlinks, let's save where they point to (all of them) + if os.path.islink(formatted_path): + # If libmylib.dylib -> /path/to/lib/libmylib.1.dylib + symlink_paths[lib_] = os.path.realpath(formatted_path) + lib_paths[lib_] = formatted_path + return + _, ext_ = os.path.splitext(formatted_path) + # Likely Windows interface library (or static one), putting it apart from the rest + if ext_ == ".lib": + interface_lib_paths[lib_] = formatted_path + else: + lib_paths[lib_] = formatted_path + + cpp_info = cpp_info or conanfile.cpp_info + libdirs = cpp_info.libdirs + # Just want to get shared libraries (likely DLLs) + bindirs = cpp_info.bindirs + # Composing absolute folders if extra folders were passed + extra_paths = [os.path.join(conanfile.package_folder, path) for path in extra_folders or []] + folders = set(libdirs + bindirs + (extra_paths or [])) + # Gather all the libraries based on the cpp_info.libs if cpp_info_libs arg is None + # Pass libs=[] if you want to collect all the libraries regardless the matches with the name + associated_libs = cpp_info.libs[:] if cpp_info_libs is None else cpp_info_libs + lib_paths = {} + interface_lib_paths = {} + symlink_paths = {} + for libdir in folders: + if not os.path.exists(libdir): + continue + files = os.listdir(libdir) + for f in files: + full_path = os.path.join(libdir, f) + if not os.path.isfile(full_path): # Make sure that directories are excluded + continue + + name, ext = os.path.splitext(f) + # Users may not name their libraries in a conventional way. For example, directly + # use the basename of the lib file as lib name, e.g., cpp_info.libs = ["liblib1.a"] + # Issue related: https://github.com/conan-io/conan/issues/11331 + if ext and f in associated_libs: # let's ensure that it has any extension + _save_lib_path(f, full_path) + continue + if name not in associated_libs and name.startswith("lib"): + name = name[3:] # libpkg -> pkg + if ext in (".so", ".lib", ".a", ".dylib", ".bc", ".dll"): + if associated_libs: + # Alternative name: in some cases the name could be pkg.if instead of pkg + base_name = name.split(".", maxsplit=1)[0] + if base_name in associated_libs: + _save_lib_path(base_name, full_path) # passing pkg instead of pkg.if + else: # we want to save all the libs + _save_lib_path(name, full_path) + + libraries = [] + for lib, lib_path in lib_paths.items(): + interface_lib_path = "" + symlink_path = symlink_paths.get(lib, "") + if lib_path.endswith(".dll"): + if lib not in interface_lib_paths: + msg = f"Windows needs a .lib for link-time and .dll for runtime. Only found {lib_path}" + if win_interface_error: + raise ConanException(msg) + else: + conanfile.output.warn(msg) + interface_lib_path = interface_lib_paths.pop(lib) + libraries.append((lib, lib_path, interface_lib_path, symlink_path)) + if interface_lib_paths: # Rest of static .lib Windows libraries + libraries.extend([(lib_name, lib_path, "", "") + for lib_name, lib_path in interface_lib_paths.items()]) + libraries.sort() + return libraries + + def move_folder_contents(conanfile, src_folder, dst_folder): """ replaces the dst_folder contents with the contents of the src_folder, which can be a child folder of dst_folder. This is used in the SCM monorepo flow, when it is necessary diff --git a/conan/tools/google/bazeldeps.py b/conan/tools/google/bazeldeps.py index 7200f517500..85bb231cc03 100644 --- a/conan/tools/google/bazeldeps.py +++ b/conan/tools/google/bazeldeps.py @@ -7,7 +7,7 @@ from conan.errors import ConanException from conan.tools._check_build_profile import check_using_build_profile -from conans.client.tools import fetch_libraries +from conan.tools.files import fetch_libraries from conans.util.files import save _BazelTargetInfo = namedtuple("DepInfo", ['repository_name', 'name', 'requires', 'cpp_info']) diff --git a/conans/client/tools/files.py b/conans/client/tools/files.py index 27aca8f570a..a8444b23c07 100644 --- a/conans/client/tools/files.py +++ b/conans/client/tools/files.py @@ -1,7 +1,6 @@ import gzip import logging import os -import pathlib import platform import stat import subprocess @@ -20,8 +19,6 @@ UNIT_SIZE = 1000.0 # Library extensions supported by collect_libs VALID_LIB_EXTENSIONS = (".so", ".lib", ".a", ".dylib", ".bc") -# Library extensions supported by fetch_libraries -EXTENDED_VALID_LIB_EXTENSIONS = (".so", ".lib", ".a", ".dylib", ".bc", ".dll") @contextmanager @@ -381,89 +378,6 @@ def collect_libs(conanfile, folder=None): return result -def fetch_libraries(conanfile, cpp_info=None, extra_folders=None, raise_error=True) -> list[tuple[str, str, str]]: - """ - Get the static/shared library paths. Analyze if DLLs are present and raise an exception if - - :param conanfile: normally a - :param cpp_info: of the component. - :param extra_folders: list of relative (to `package_folder`) folders to search more libraries. - :param raise_error: if raise_error == True, it'll raise an exception if DLL lib does not have - an associated *.lib interface library. - :return: list of tuples per static/shared library -> - [(lib_name, library_path, interface_library_path)] - Note: ``library_path`` could be both static and shared ones in case of UNIX systems. - Windows would have: - * shared: library_path as DLL, and interface_library_path as LIB - * static: library_path as LIB, and interface_library_path as None - """ - def _save_lib_path(lib_, lib_path_): - """Add each lib with its full library path""" - formatted_path = lib_path_.replace("\\", "/") - # In case of symlinks, only keep the shortest file name in the same "group" - if os.path.islink(lib_path_): - # If libmylib.dylib -> lib/libmylib.1.dylib, this gets libmylib.1.dylib - real_lib = os.path.basename(os.path.realpath(lib_path_)) - if real_lib not in symlink_paths or len(lib_) < len(symlink_paths[lib_]): - symlink_paths[lib_] = real_lib - - _, ext_ = os.path.splitext(formatted_path) - if ext_ == ".lib": # Possibly Windows interface library, putting it apart from the rest - interface_lib_paths[lib_] = formatted_path - else: - lib_paths[lib_] = formatted_path - - cpp_info = cpp_info or conanfile.cpp_info - libdirs = cpp_info.libdirs - bindirs = cpp_info.bindirs # just want to get shared libraries (likely DLLs) - libs = cpp_info.libs[:] # copying the values if they are already declared - lib_paths = {} - interface_lib_paths = {} - symlink_paths = {} - # Composing absolute folders if extra folders were passed - extra_paths = [os.path.join(conanfile.package_folder, path) for path in extra_folders or []] - for libdir in set(libdirs + bindirs + (extra_paths or [])): - if not os.path.exists(libdir): - continue - files = os.listdir(libdir) - for f in files: - full_path = os.path.join(libdir, f) - if not os.path.isfile(full_path): # Make sure that directories are excluded - continue - - name, ext = os.path.splitext(f) - # Users may not name their libraries in a conventional way. For example, directly - # use the basename of the lib file as lib name, e.g., cpp_info.libs = ["liblib1.a"] - # Issue related: https://github.com/conan-io/conan/issues/11331 - if ext and f in libs: # let's ensure that it has any extension - _save_lib_path(f, full_path) - continue - if name not in libs and name.startswith("lib"): - name = name[3:] # libpkg -> pkg - if ext in EXTENDED_VALID_LIB_EXTENSIONS: - if not libs: # we want to save all the libs - _save_lib_path(name, full_path) - else: - # Alternative name: in some cases the name could be pkg.if instead of pkg - alternative_name = name.split(".", maxsplit=1)[0] - if name in libs or alternative_name in libs: - _save_lib_path(name, full_path) - - libraries = [] - for lib, lib_path in lib_paths.items(): - interface_lib_path = "" - if lib_path.endswith(".dll"): - if lib not in interface_lib_paths and raise_error: - raise ConanException(f"Windows needs a .lib for link-time and .dll for runtime." - f" Only found {lib_path}") - interface_lib_path = interface_lib_paths.pop(lib) - libraries.append((lib, lib_path, interface_lib_path)) - if interface_lib_paths: # Rest of static .lib Windows libraries - libraries.extend([(lib_name, lib_path, "") for lib_name, lib_path in interface_lib_paths.items()]) - libraries.sort() - return libraries - - def which(filename): """ same affect as posix which command or shutil.which from python3 """ # FIXME: Replace with shutil.which in Conan 2.0 diff --git a/conans/test/unittests/tools/files/test_fetch_libraries.py b/conans/test/unittests/tools/files/test_fetch_libraries.py index ba0a5836a3b..1c29e81baa9 100644 --- a/conans/test/unittests/tools/files/test_fetch_libraries.py +++ b/conans/test/unittests/tools/files/test_fetch_libraries.py @@ -4,8 +4,7 @@ import pytest -from conan.tools.files import save -from conans.client.tools import fetch_libraries +from conan.tools.files import save, fetch_libraries from conans.model.build_info import CppInfo from conans.test.utils.mocks import ConanFileMock from conans.test.utils.test_files import temp_folder @@ -45,47 +44,57 @@ def cpp_info(): @pytest.mark.parametrize("libs, expected", [ # expected == (lib_name, is_shared, library_path, interface_library_path) - (["mylibwinst"], [('mylibwinst', '{base_folder}/lib/mylibwinst.lib', "")]), + (["mylibwinst"], [('mylibwinst', '{base_folder}/lib/mylibwinst.lib', "", "")]), # Win + shared - (["mylibwinsh"], [('mylibwinsh', '{base_folder}/bin/mylibwinsh.dll', '{base_folder}/lib/mylibwinsh.lib')]), + (["mylibwinsh"], [('mylibwinsh', '{base_folder}/bin/mylibwinsh.dll', '{base_folder}/lib/mylibwinsh.lib', "")]), # Win + shared (interface with another ext) (["mylibwin2"], - [('mylibwin2', '{base_folder}/bin/mylibwin2.dll', '{base_folder}/lib/mylibwin2.if.lib')]), + [('mylibwin2', '{base_folder}/bin/mylibwin2.dll', '{base_folder}/lib/mylibwin2.if.lib', "")]), # Win + Mac + shared - (["mylibwinsh", "mylibmacsh"], [('mylibmacsh', '{base_folder}/bin/mylibmacsh.dylib', ""), - ('mylibwinsh', '{base_folder}/bin/mylibwinsh.dll', '{base_folder}/lib/mylibwinsh.lib')]), + (["mylibwinsh", "mylibmacsh"], [('mylibmacsh', '{base_folder}/bin/mylibmacsh.dylib', "", ""), + ('mylibwinsh', '{base_folder}/bin/mylibwinsh.dll', '{base_folder}/lib/mylibwinsh.lib', "")]), # Linux + Mac + static - (["myliblin", "mylibmacst"], [('mylibmacst', '{base_folder}/lib/mylibmacst.a', ""), - ('myliblin', '{base_folder}/lib/myliblin.a', "")]), + (["myliblin", "mylibmacst"], [('mylibmacst', '{base_folder}/lib/mylibmacst.a', "", ""), + ('myliblin', '{base_folder}/lib/myliblin.a', "", "")]), # mylib + shared (saved as libmylib.so) -> removing the leading "lib" if it matches - (["mylibsh"], [('mylibsh', '{base_folder}/lib/libmylibsh.so', "")]), + (["mylibsh"], [('mylibsh', '{base_folder}/lib/libmylibsh.so', "", "")]), # mylib + static (saved in a subfolder subfolder/libmylib.a) -> non-recursive at this moment (["mylib"], []), # no lib matching (["noexist"], []), # no lib matching + Win + static - (["noexist", "mylibwinst"], [('mylibwinst', '{base_folder}/lib/mylibwinst.lib', "")]), + (["noexist", "mylibwinst"], [('mylibwinst', '{base_folder}/lib/mylibwinst.lib', "", "")]), # protobuf (Issue related https://github.com/conan-io/conan/issues/15390) (["protoc"], []), # non-conventional library name (Issue related https://github.com/conan-io/conan/pull/11343) - (["libmylibsh.so"], [('libmylibsh.so', '{base_folder}/lib/libmylibsh.so', "")]), + (["libmylibsh.so"], [('libmylibsh.so', '{base_folder}/lib/libmylibsh.so', "", "")]), ]) def test_fetch_libraries(cpp_info, libs, expected): cpp_info.libs = libs ret = [] - for (lib, lib_path, interface_lib_path) in expected: + for (lib, lib_path, interface_lib_path, symlink_path) in expected: if lib_path: lib_path = lib_path.format(base_folder=cpp_info._base_folder) if interface_lib_path: interface_lib_path = interface_lib_path.format(base_folder=cpp_info._base_folder) - ret.append((lib, lib_path, interface_lib_path)) + ret.append((lib, lib_path, interface_lib_path, symlink_path)) found_libs = fetch_libraries(ConanFileMock(), cpp_info) ret.sort() assert found_libs == ret - +@pytest.mark.parametrize("cpp_info_libs, expected", [ + # Only mylib associated + (["mylib"], [('mylib', '{base_folder}/lib/libmylib.2.dylib', '', '{base_folder}/lib/libmylib.1.0.0.dylib')]), + # All the existing ones + ([], [ + ('mylib', '{base_folder}/lib/libmylib.dylib', '', '{base_folder}/lib/libmylib.1.0.0.dylib'), + ('mylib.1', '{base_folder}/lib/libmylib.1.dylib', '', '{base_folder}/lib/libmylib.1.0.0.dylib'), + ('mylib.1.0.0', '{base_folder}/lib/libmylib.1.0.0.dylib', '', ''), + ('mylib.2', '{base_folder}/lib/libmylib.2.dylib', '', ''), + ('mylib.3', '{base_folder}/custom_folder/libmylib.3.dylib', '', '')]), +]) @pytest.mark.skipif(platform.system() == "Windows", reason="Needs symlinks support") -def test_fetch_libraries_symlinks(): +def test_fetch_libraries_symlinks(cpp_info_libs, expected): """ Tests how fetch_libraries function saves the shortest path (symlink) defined @@ -118,8 +127,8 @@ def test_fetch_libraries_symlinks(): save(conanfile, lib_mylib2_path, "") save(conanfile, lib_mylib3_path, "") conanfile.cpp_info.libdirs = [lib_folder, custom_folder] - result = fetch_libraries(conanfile) - assert ["mylib", "mylib.2", "mylib.3"] == result + result = fetch_libraries(conanfile, cpp_info_libs=cpp_info_libs) + assert expected == result def test_basic_fetch_libraries(): From 52f59368310cfd888dc0ce422a2ff0dbf1f4a465 Mon Sep 17 00:00:00 2001 From: Francisco Ramirez de Anton Date: Wed, 13 Mar 2024 16:11:13 +0100 Subject: [PATCH 5/8] Wip --- conan/tools/files/files.py | 28 +++++++++---------- .../tools/files/test_fetch_libraries.py | 25 +++++++++++------ 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index a8b0558e0f0..45833d65c79 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -566,26 +566,24 @@ def fetch_libraries(conanfile, cpp_info=None, cpp_info_libs=None, * shared: library_path as DLL, and interface_library_path as LIB * static: library_path as LIB, and interface_library_path as "" """ - def _save_lib_path(lib_, lib_path_): + def _save_lib_path(lib_name, lib_path_): """Add each lib with its full library path""" - if lib_ in lib_paths: - conanfile.output.warn(f"{lib_} name is duplicated, but it could have a different path: " - f"{lib_path_} or even be a symlink. If any doubt, call the " - f"fetch_libraries function with cpp_info_libs=[] to gather all the" - f" existing ones and compare both results.") formatted_path = lib_path_.replace("\\", "/") + _, ext_ = os.path.splitext(formatted_path) # In case of symlinks, let's save where they point to (all of them) if os.path.islink(formatted_path): - # If libmylib.dylib -> /path/to/lib/libmylib.1.dylib - symlink_paths[lib_] = os.path.realpath(formatted_path) - lib_paths[lib_] = formatted_path + # Important! os.path.realpath returns the final path of the symlink even if it points + # to another symlink, i.e., libmylib.dylib -> libmylib.1.dylib -> libmylib.1.0.0.dylib + # then os.path.realpath("libmylib.dylib") == "libmylib.1.0.0.dylib" + # Better to use os.readlink as it returns the path which the symbolic link points to. + symlink_paths[lib_name] = os.readlink(formatted_path) + lib_paths[lib_name] = formatted_path return - _, ext_ = os.path.splitext(formatted_path) # Likely Windows interface library (or static one), putting it apart from the rest if ext_ == ".lib": - interface_lib_paths[lib_] = formatted_path + interface_lib_paths[lib_name] = formatted_path else: - lib_paths[lib_] = formatted_path + lib_paths[lib_name] = formatted_path cpp_info = cpp_info or conanfile.cpp_info libdirs = cpp_info.libdirs @@ -600,7 +598,7 @@ def _save_lib_path(lib_, lib_path_): lib_paths = {} interface_lib_paths = {} symlink_paths = {} - for libdir in folders: + for libdir in sorted(folders): # deterministic if not os.path.exists(libdir): continue files = os.listdir(libdir) @@ -623,7 +621,7 @@ def _save_lib_path(lib_, lib_path_): # Alternative name: in some cases the name could be pkg.if instead of pkg base_name = name.split(".", maxsplit=1)[0] if base_name in associated_libs: - _save_lib_path(base_name, full_path) # passing pkg instead of pkg.if + _save_lib_path(name, full_path) # passing pkg instead of pkg.if else: # we want to save all the libs _save_lib_path(name, full_path) @@ -638,7 +636,7 @@ def _save_lib_path(lib_, lib_path_): raise ConanException(msg) else: conanfile.output.warn(msg) - interface_lib_path = interface_lib_paths.pop(lib) + interface_lib_path = interface_lib_paths.pop(lib, "") libraries.append((lib, lib_path, interface_lib_path, symlink_path)) if interface_lib_paths: # Rest of static .lib Windows libraries libraries.extend([(lib_name, lib_path, "", "") diff --git a/conans/test/unittests/tools/files/test_fetch_libraries.py b/conans/test/unittests/tools/files/test_fetch_libraries.py index 1c29e81baa9..7b3b8950be3 100644 --- a/conans/test/unittests/tools/files/test_fetch_libraries.py +++ b/conans/test/unittests/tools/files/test_fetch_libraries.py @@ -69,7 +69,7 @@ def cpp_info(): # non-conventional library name (Issue related https://github.com/conan-io/conan/pull/11343) (["libmylibsh.so"], [('libmylibsh.so', '{base_folder}/lib/libmylibsh.so', "", "")]), ]) -def test_fetch_libraries(cpp_info, libs, expected): +def test_fetch_libraries(libs, expected, cpp_info): cpp_info.libs = libs ret = [] for (lib, lib_path, interface_lib_path, symlink_path) in expected: @@ -78,16 +78,17 @@ def test_fetch_libraries(cpp_info, libs, expected): if interface_lib_path: interface_lib_path = interface_lib_path.format(base_folder=cpp_info._base_folder) ret.append((lib, lib_path, interface_lib_path, symlink_path)) + found_libs = fetch_libraries(ConanFileMock(), cpp_info) ret.sort() assert found_libs == ret @pytest.mark.parametrize("cpp_info_libs, expected", [ # Only mylib associated - (["mylib"], [('mylib', '{base_folder}/lib/libmylib.2.dylib', '', '{base_folder}/lib/libmylib.1.0.0.dylib')]), + (["mylib"], [('mylib', '{base_folder}/lib/libmylib.dylib', '', '{base_folder}/lib/libmylib.1.dylib')]), # All the existing ones ([], [ - ('mylib', '{base_folder}/lib/libmylib.dylib', '', '{base_folder}/lib/libmylib.1.0.0.dylib'), + ('mylib', '{base_folder}/lib/libmylib.dylib', '', '{base_folder}/lib/libmylib.1.dylib'), ('mylib.1', '{base_folder}/lib/libmylib.1.dylib', '', '{base_folder}/lib/libmylib.1.0.0.dylib'), ('mylib.1.0.0', '{base_folder}/lib/libmylib.1.0.0.dylib', '', ''), ('mylib.2', '{base_folder}/lib/libmylib.2.dylib', '', ''), @@ -110,8 +111,9 @@ def test_fetch_libraries_symlinks(cpp_info_libs, expected): └── libmylib.dylib -> lib/libmylib.1.dylib """ # Keep only the shortest lib name per group of symlinks + base_folder = temp_folder() conanfile = ConanFileMock(options_values={"shared": True}) - conanfile.folders.set_base_package(temp_folder()) + conanfile.folders.set_base_package(base_folder) conanfile.cpp_info = CppInfo(conanfile.name, "") # Lib dirs and libraries lib_folder = os.path.join(conanfile.package_folder, "lib") @@ -122,13 +124,20 @@ def test_fetch_libraries_symlinks(cpp_info_libs, expected): lib_mylib2_path = os.path.join(lib_folder, "libmylib.2.dylib") lib_mylib3_path = os.path.join(custom_folder, "libmylib.3.dylib") save(conanfile, version_mylib_path, "") - os.symlink(version_mylib_path, soversion_mylib_path) # libmylib.1.dylib - os.symlink(soversion_mylib_path, lib_mylib_path) # libmylib.dylib + os.symlink(version_mylib_path, soversion_mylib_path) # libmylib.1.dylib -> lib/libmylib.1.0.0.dylib + os.symlink(soversion_mylib_path, lib_mylib_path) # libmylib.dylib -> lib/libmylib.1.dylib save(conanfile, lib_mylib2_path, "") save(conanfile, lib_mylib3_path, "") conanfile.cpp_info.libdirs = [lib_folder, custom_folder] + ret = [] + for (lib, lib_path, _, symlink_path) in expected: + if lib_path: + lib_path = lib_path.format(base_folder=base_folder) + if symlink_path: + symlink_path = symlink_path.format(base_folder=base_folder) + ret.append((lib, lib_path, _, symlink_path)) result = fetch_libraries(conanfile, cpp_info_libs=cpp_info_libs) - assert expected == result + assert ret == result def test_basic_fetch_libraries(): @@ -143,7 +152,7 @@ def test_basic_fetch_libraries(): mylib_path = os.path.join(conanfile.package_folder, "lib", "mylib.lib") save(conanfile, mylib_path, "") - result = fetch_libraries(conanfile) + result = fetch_libraries(conanfile, cpp_info_libs=[]) assert ["mylib"] == result # Custom folder From 5c0603a38e99b584278d73546633113f713cf2e9 Mon Sep 17 00:00:00 2001 From: Francisco Ramirez de Anton Date: Wed, 13 Mar 2024 16:29:51 +0100 Subject: [PATCH 6/8] Wip --- conan/tools/files/files.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index 45833d65c79..a9935b83a86 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -566,8 +566,11 @@ def fetch_libraries(conanfile, cpp_info=None, cpp_info_libs=None, * shared: library_path as DLL, and interface_library_path as LIB * static: library_path as LIB, and interface_library_path as "" """ - def _save_lib_path(lib_name, lib_path_): + def _save_lib_path(lib_name, lib_path_, base_lib_name=""): """Add each lib with its full library path""" + if base_lib_name and base_lib_name != lib_name: + lib_name = base_lib_name + formatted_path = lib_path_.replace("\\", "/") _, ext_ = os.path.splitext(formatted_path) # In case of symlinks, let's save where they point to (all of them) @@ -577,13 +580,26 @@ def _save_lib_path(lib_name, lib_path_): # then os.path.realpath("libmylib.dylib") == "libmylib.1.0.0.dylib" # Better to use os.readlink as it returns the path which the symbolic link points to. symlink_paths[lib_name] = os.readlink(formatted_path) - lib_paths[lib_name] = formatted_path - return - # Likely Windows interface library (or static one), putting it apart from the rest - if ext_ == ".lib": + # Likely Windows interface library (or static one) + elif ext_ == ".lib": interface_lib_paths[lib_name] = formatted_path - else: - lib_paths[lib_name] = formatted_path + return # putting it apart from the rest + # Save the library + lib_paths[lib_name] = formatted_path + + def _filter_if_associated_libraries(libraries_): + if not associated_libs: + return libraries_ + ret = {} + for lib_ in associated_libs: + if lib_ in libraries_: # perfect match, it has prio + ret[lib_] = libraries_[lib_] + else: + for k, v in libraries_.items(): + if k.split(".", maxsplit=1)[0] == lib_: + ret[lib_] = v + break + return ret cpp_info = cpp_info or conanfile.cpp_info libdirs = cpp_info.libdirs @@ -621,7 +637,7 @@ def _save_lib_path(lib_name, lib_path_): # Alternative name: in some cases the name could be pkg.if instead of pkg base_name = name.split(".", maxsplit=1)[0] if base_name in associated_libs: - _save_lib_path(name, full_path) # passing pkg instead of pkg.if + _save_lib_path(name, full_path, base_lib_name=base_name) # passing pkg instead of pkg.if else: # we want to save all the libs _save_lib_path(name, full_path) From 53536e45cf6363dd1c33140bd79e18cf23692911 Mon Sep 17 00:00:00 2001 From: Francisco Ramirez de Anton Date: Wed, 13 Mar 2024 16:46:33 +0100 Subject: [PATCH 7/8] wip --- conan/tools/files/files.py | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index a9935b83a86..fbb6ea14780 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -566,40 +566,26 @@ def fetch_libraries(conanfile, cpp_info=None, cpp_info_libs=None, * shared: library_path as DLL, and interface_library_path as LIB * static: library_path as LIB, and interface_library_path as "" """ - def _save_lib_path(lib_name, lib_path_, base_lib_name=""): + def _save_lib_path(lib_name, lib_path_): """Add each lib with its full library path""" - if base_lib_name and base_lib_name != lib_name: - lib_name = base_lib_name - formatted_path = lib_path_.replace("\\", "/") _, ext_ = os.path.splitext(formatted_path) # In case of symlinks, let's save where they point to (all of them) - if os.path.islink(formatted_path): + if os.path.islink(formatted_path) and lib_name not in symlink_paths: # Important! os.path.realpath returns the final path of the symlink even if it points # to another symlink, i.e., libmylib.dylib -> libmylib.1.dylib -> libmylib.1.0.0.dylib # then os.path.realpath("libmylib.dylib") == "libmylib.1.0.0.dylib" # Better to use os.readlink as it returns the path which the symbolic link points to. symlink_paths[lib_name] = os.readlink(formatted_path) + lib_paths[lib_name] = formatted_path + return # Likely Windows interface library (or static one) - elif ext_ == ".lib": + elif ext_ == ".lib" and lib_name not in interface_lib_paths: interface_lib_paths[lib_name] = formatted_path return # putting it apart from the rest - # Save the library - lib_paths[lib_name] = formatted_path - - def _filter_if_associated_libraries(libraries_): - if not associated_libs: - return libraries_ - ret = {} - for lib_ in associated_libs: - if lib_ in libraries_: # perfect match, it has prio - ret[lib_] = libraries_[lib_] - else: - for k, v in libraries_.items(): - if k.split(".", maxsplit=1)[0] == lib_: - ret[lib_] = v - break - return ret + if lib_name not in lib_paths: + # Save the library + lib_paths[lib_name] = formatted_path cpp_info = cpp_info or conanfile.cpp_info libdirs = cpp_info.libdirs @@ -637,7 +623,7 @@ def _filter_if_associated_libraries(libraries_): # Alternative name: in some cases the name could be pkg.if instead of pkg base_name = name.split(".", maxsplit=1)[0] if base_name in associated_libs: - _save_lib_path(name, full_path, base_lib_name=base_name) # passing pkg instead of pkg.if + _save_lib_path(base_name, full_path) # passing pkg instead of pkg.if else: # we want to save all the libs _save_lib_path(name, full_path) From 78c52d1116d7bdbc16fce13b44dc85990315e786 Mon Sep 17 00:00:00 2001 From: Francisco Ramirez de Anton Date: Wed, 13 Mar 2024 16:47:32 +0100 Subject: [PATCH 8/8] imports --- conan/tools/files/files.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conan/tools/files/files.py b/conan/tools/files/files.py index fbb6ea14780..6b23de3f90d 100644 --- a/conan/tools/files/files.py +++ b/conan/tools/files/files.py @@ -9,7 +9,6 @@ import sys from contextlib import contextmanager from fnmatch import fnmatch -from types import NoneType from urllib.parse import urlparse from urllib.request import url2pathname