Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

xz_utils: conan v2 support #13038

Merged
merged 11 commits into from
Sep 28, 2022
251 changes: 149 additions & 102 deletions recipes/xz_utils/all/conanfile.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
from conans import ConanFile, tools, AutoToolsBuildEnvironment, MSBuild
from conans.tools import Version
from conan import ConanFile
from conan.errors import ConanException
from conan.tools.apple import fix_apple_shared_install_name
from conan.tools.env import VirtualBuildEnv
from conan.tools.files import collect_libs, copy, get, rename, replace_in_file, rm, rmdir, save
from conan.tools.gnu import Autotools, AutotoolsToolchain
from conan.tools.layout import basic_layout
from conan.tools.microsoft import is_msvc, is_msvc_static_runtime, MSBuild, MSBuildToolchain, unix_path
from conan.tools.scm import Version
import os
import textwrap

required_conan_version = ">=1.43.0"
required_conan_version = ">=1.52.0"


class XZUtils(ConanFile):
Expand All @@ -28,24 +35,18 @@ class XZUtils(ConanFile):
"fPIC": True,
}

_autotools = None

@property
def _source_subfolder(self):
return "source_subfolder"

@property
def _is_msvc(self):
return str(self.settings.compiler) in ["Visual Studio", "msvc"]

@property
def _settings_build(self):
return getattr(self, "settings_build", self.settings)

@property
def _effective_msbuild_type(self):
# treat "RelWithDebInfo" and "MinSizeRel" as "Release"
return "Debug" if self.settings.build_type == "Debug" else "Release"
# there is no DebugMT configuration in upstream vcxproj, we patch Debug configuration afterwards
return "{}{}".format(
"Debug" if self.settings.build_type == "Debug" else "Release",
"MT" if is_msvc_static_runtime(self) and self.settings.build_type != "Debug" else "",
)

def config_options(self):
if self.settings.os == "Windows":
Expand All @@ -54,133 +55,179 @@ def config_options(self):
def configure(self):
if self.options.shared:
del self.options.fPIC
del self.settings.compiler.cppstd
del self.settings.compiler.libcxx
try:
del self.settings.compiler.libcxx
except Exception:
pass
try:
del self.settings.compiler.cppstd
except Exception:
pass

def layout(self):
basic_layout(self, src_folder="src")

def build_requirements(self):
if self._settings_build.os == "Windows" and not self._is_msvc and \
not tools.get_env("CONAN_BASH_PATH"):
self.build_requires("msys2/cci.latest")
if self._settings_build.os == "Windows" and not is_msvc(self) and \
not self.conf.get("tools.microsoft.bash:path", default=False, check_type=bool):
self.tool_requires("msys2/cci.latest")

def source(self):
tools.get(**self.conan_data["sources"][self.version],
destination=self._source_subfolder, strip_root=True)
get(self, **self.conan_data["sources"][self.version],
destination=self.source_folder, strip_root=True)

def generate(self):
if is_msvc(self):
tc = MSBuildToolchain(self)
tc.generate()
else:
tc = AutotoolsToolchain(self)
tc.configure_args.append("--disable-doc")
if self.settings.build_type == "Debug":
tc.configure_args.append("--enable-debug")
tc.generate()
env = VirtualBuildEnv(self)
env.generate()

def _fix_msvc_platform_toolset(self, vcxproj_file, old_toolset):
platform_toolset = {
"Visual Studio": {
"8": "v80",
"9": "v90",
"10": "v100",
"11": "v110",
"12": "v120",
"14": "v140",
"15": "v141",
"16": "v142",
"17": "v143",
},
"msvc": {
"170": "v110",
"180": "v120",
"190": "v140",
"191": "v141",
"192": "v142",
"193": "v143",
}
}.get(str(self.settings.compiler), {}).get(str(self.settings.compiler.version))
if not platform_toolset:
raise ConanException(
f"Unkown platform toolset for {self.settings.compiler} {self.settings.compiler.version}",
)
replace_in_file(
self,
vcxproj_file,
f"<PlatformToolset>{old_toolset}</PlatformToolset>",
f"<PlatformToolset>{platform_toolset}</PlatformToolset>",
)

def _apply_patches(self):
if tools.Version(self.version) == "5.2.4" and self._is_msvc:
def _build_msvc(self):
if Version(self.version) == "5.2.4":
# Relax Windows SDK restriction
# Workaround is required only for 5.2.4 because since 5.2.5 WindowsTargetPlatformVersion is dropped from vcproj file
#
# emulate VS2019+ meaning of WindowsTargetPlatformVersion == "10.0"
# undocumented method, but officially recommended workaround by microsoft at at
# https://developercommunity.visualstudio.com/content/problem/140294/windowstargetplatformversion-makes-it-impossible-t.html
windows_target_platform_version_old = "<WindowsTargetPlatformVersion>10.0.15063.0</WindowsTargetPlatformVersion>"
if self.settings.compiler.version == 15:
windows_target_platform_version_new = "<WindowsTargetPlatformVersion>$([Microsoft.Build.Utilities.ToolLocationHelper]::GetLatestSDKTargetPlatformVersion('Windows', '10.0'))</WindowsTargetPlatformVersion>"
else:
windows_target_platform_version_new = "<WindowsTargetPlatformVersion>10.0</WindowsTargetPlatformVersion>"
tools.replace_in_file(os.path.join(self._source_subfolder, "windows", "vs2017", "liblzma.vcxproj"),
windows_target_platform_version_old,
windows_target_platform_version_new)
tools.replace_in_file(os.path.join(self._source_subfolder, "windows", "vs2017", "liblzma_dll.vcxproj"),
windows_target_platform_version_old,
windows_target_platform_version_new)

# Allow to install relocatable shared lib on macOS
if tools.is_apple_os(self.settings.os):
tools.replace_in_file(
os.path.join(self._source_subfolder, "configure"),
"-install_name \\$rpath/",
"-install_name @rpath/",
)
replace_in_file(self, os.path.join(self.source_folder, "windows", "vs2017", "liblzma.vcxproj"),
windows_target_platform_version_old, "")
replace_in_file(self, os.path.join(self.source_folder, "windows", "vs2017", "liblzma_dll.vcxproj"),
windows_target_platform_version_old, "")

# TODO: Find a way to inject conantoolchain.props content from MSBuildToolchain
# For the moment all the logic below is a big trick & doesn't honor custom cflags, cxxflags & ldflags from profile
# and arch different than x86 & x86_64

def _build_msvc(self):
# windows\INSTALL-MSVC.txt
msvc_version = "vs2017" if Version(self.settings.compiler.version) >= "15" else "vs2013"
with tools.chdir(os.path.join(self._source_subfolder, "windows", msvc_version)):
target = "liblzma_dll" if self.options.shared else "liblzma"
msbuild = MSBuild(self)
msbuild.build(
"xz_win.sln",
targets=[target],
build_type=self._effective_msbuild_type,
platforms={"x86": "Win32", "x86_64": "x64"},
upgrade_project=Version(self.settings.compiler.version) >= "17")
Copy link
Contributor Author

@SpaceIm SpaceIm Sep 20, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

upgrade_project logic is lost in this conan v2 migration, I don't know what is the equivalent in the new MSBuild build helper.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

and it's required, it fails on Windows with Visual Studio 2022

Copy link
Contributor Author

@SpaceIm SpaceIm Sep 20, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.


def _configure_autotools(self):
if self._autotools:
return self._autotools
self._autotools = AutoToolsBuildEnvironment(self, win_bash=tools.os_info.is_windows)
args = ["--disable-doc"]
if self.settings.os != "Windows" and self.options.get_safe("fPIC", True):
args.append("--with-pic")
if self.options.shared:
args.extend(["--disable-static", "--enable-shared"])
if (self.settings.compiler == "Visual Studio" and Version(self.settings.compiler) >= "15") or \
(self.settings.compiler == "msvc" and Version(self.settings.compiler) >= "191"):
msvc_version = "vs2017"
old_toolset = "v141"
else:
args.extend(["--enable-static", "--disable-shared"])
if self.settings.build_type == "Debug":
args.append("--enable-debug")
self._autotools.configure(configure_dir=self._source_subfolder, args=args, build=False)
return self._autotools
msvc_version = "vs2013"
old_toolset = "v120"
build_script_folder = os.path.join(self.source_folder, "windows", msvc_version)

# TODO: replace by some conan helper function (https://github.com/conan-io/conan/issues/12155)?
liblzma_vcxproj = os.path.join(build_script_folder, "liblzma.vcxproj")
liblzma_dll_vcxproj = os.path.join(build_script_folder, "liblzma_dll.vcxproj")
self._fix_msvc_platform_toolset(liblzma_vcxproj, old_toolset)
self._fix_msvc_platform_toolset(liblzma_dll_vcxproj, old_toolset)

# Patch Debug configuration if runtime is MT since there is no DebugMT configuration in upstream vcxproj
if self.settings.build_type == "Debug" and is_msvc_static_runtime(self):
replace_in_file(self, liblzma_vcxproj, "MultiThreadedDebugDLL", "MultiThreadedDebug")
replace_in_file(self, liblzma_dll_vcxproj, "MultiThreadedDebugDLL", "MultiThreadedDebug")

target = "liblzma_dll" if self.options.shared else "liblzma"
msbuild = MSBuild(self)
msbuild.build_type = self._effective_msbuild_type
msbuild.platform = "Win32" if self.settings.arch == "x86" else msbuild.platform
msbuild.build(os.path.join(build_script_folder, "xz_win.sln"), targets=[target])

def build(self):
self._apply_patches()
if self._is_msvc:
if is_msvc(self):
self._build_msvc()
else:
autotools = self._configure_autotools()
autotools = Autotools(self)
self.win_bash = True
autotools.configure()
autotools.make()
self.win_bash = None
Copy link
Contributor Author

@SpaceIm SpaceIm Sep 20, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@memsharded could you confirm this is the canonical way to conditionally enable win_bash in conan v2 methods (for this case it's for MinGW, and we ensure that a bash is available on windows build machine by adding msys2 recipe to tool_requires if user did not define tools.microsoft.bash:path in its conan configuration)?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've tested locally on windows with MinGW. It works... almost:

It fails during installation because install path is not converted to unix path:

 /usr/bin/install -c -m 644  /c/users/spaceim/.conan/data/xz_utils/5.2.5/_/_/build/3cbc13750c61092b2c900ee6caef68cf1cd320b4/src/src/liblzma/api/lzma.h 'C:Usersspaceim.conandataxz_utils5.2.5__package3cbc13750c61092b2c900ee6caef68cf1cd320b4/include/.'
make[4]: Leaving directory '/c/Users/spaceim/.conan/data/xz_utils/5.2.5/_/_/build/3cbc13750c61092b2c900ee6caef68cf1cd320b4/build-release/src/liblzma/api'
make[3]: Leaving directory '/c/Users/spaceim/.conan/data/xz_utils/5.2.5/_/_/build/3cbc13750c61092b2c900ee6caef68cf1cd320b4/build-release/src/liblzma/api'
make[3]: Entering directory '/c/Users/spaceim/.conan/data/xz_utils/5.2.5/_/_/build/3cbc13750c61092b2c900ee6caef68cf1cd320b4/build-release/src/liblzma'
make[4]: Entering directory '/c/Users/spaceim/.conan/data/xz_utils/5.2.5/_/_/build/3cbc13750c61092b2c900ee6caef68cf1cd320b4/build-release/src/liblzma'
 /usr/bin/mkdir -p 'C:Usersspaceim.conandataxz_utils5.2.5__package3cbc13750c61092b2c900ee6caef68cf1cd320b4/lib'
 /usr/bin/mkdir -p 'C:Usersspaceim.conandataxz_utils5.2.5__package3cbc13750c61092b2c900ee6caef68cf1cd320b4/lib/pkgconfig'
 /bin/sh ../../libtool   --mode=install /usr/bin/install -c   liblzma.la 'C:Usersspaceim.conandataxz_utils5.2.5__package3cbc13750c61092b2c900ee6caef68cf1cd320b4/lib'
 /usr/bin/install -c -m 644 liblzma.pc 'C:Usersspaceim.conandataxz_utils5.2.5__package3cbc13750c61092b2c900ee6caef68cf1cd320b4/lib/pkgconfig'
Usage: /c/Users/spaceim/.conan/data/xz_utils/5.2.5/_/_/build/3cbc13750c61092b2c900ee6caef68cf1cd320b4/build-release/libtool [OPTION]... [MODE-ARG]...
Try 'libtool --help' for more information.
libtool:   error: 'C:Usersspaceim.conandataxz_utils5.2.5__package3cbc13750c61092b2c900ee6caef68cf1cd320b4/lib' must be an absolute directory name
make[4]: *** [Makefile:876: install-libLTLIBRARIES] Error 1
make[4]: Leaving directory '/c/Users/spaceim/.conan/data/xz_utils/5.2.5/_/_/build/3cbc13750c61092b2c900ee6caef68cf1cd320b4/build-release/src/liblzma'
make[3]: *** [Makefile:1802: install-am] Error 2
make[3]: Leaving directory '/c/Users/spaceim/.conan/data/xz_utils/5.2.5/_/_/build/3cbc13750c61092b2c900ee6caef68cf1cd320b4/build-release/src/liblzma'
make[2]: *** [Makefile:1638: install-recursive] Error 1
make[2]: Leaving directory '/c/Users/spaceim/.conan/data/xz_utils/5.2.5/_/_/build/3cbc13750c61092b2c900ee6caef68cf1cd320b4/build-release/src/liblzma'
make[1]: *** [Makefile:422: install-recursive] Error 1
make[1]: Leaving directory '/c/Users/spaceim/.conan/data/xz_utils/5.2.5/_/_/build/3cbc13750c61092b2c900ee6caef68cf1cd320b4/build-release/src'
make: *** [Makefile:616: install-recursive] Error 1

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's it. If I replace autotools.install() by autotools.install(args=[f"DESTDIR={unix_path(self, self.package_folder)}"]), it works (note: if I call conan create -kb it fails with ConanException: The config 'tools.microsoft.bash:subsystem' is needed to run commands in a Windows subsystem, funny since this config is injected by msys2 recipe)

Copy link
Contributor Author

@SpaceIm SpaceIm Sep 20, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@SpaceIm Thank you very much for checking it. The new MSBuild is still limited in terms of features, compared to the legacy.


def package(self):
self.copy(pattern="COPYING", dst="licenses", src=self._source_subfolder)
if self._is_msvc:
inc_dir = os.path.join(self._source_subfolder, "src", "liblzma", "api")
self.copy(pattern="*.h", dst="include", src=inc_dir, keep_path=True)
copy(self, "COPYING", src=self.source_folder, dst=os.path.join(self.package_folder, "licenses"))
if is_msvc(self):
inc_dir = os.path.join(self.source_folder, "src", "liblzma", "api")
copy(self, "*.h", src=inc_dir, dst=os.path.join(self.package_folder, "include"), keep_path=True)
arch = {"x86": "Win32", "x86_64": "x64"}.get(str(self.settings.arch))
target = "liblzma_dll" if self.options.shared else "liblzma"
msvc_version = "vs2017" if Version(self.settings.compiler.version) >= "15" else "vs2013"
bin_dir = os.path.join(self._source_subfolder, "windows", msvc_version,
if (self.settings.compiler == "Visual Studio" and Version(self.settings.compiler) >= "15") or \
(self.settings.compiler == "msvc" and Version(self.settings.compiler) >= "191"):
msvc_version = "vs2017"
else:
msvc_version = "vs2013"
bin_dir = os.path.join(self.source_folder, "windows", msvc_version,
self._effective_msbuild_type, arch, target)
self.copy(pattern="*.lib", dst="lib", src=bin_dir, keep_path=False)
copy(self, "*.lib", src=bin_dir, dst=os.path.join(self.package_folder, "lib"), keep_path=False)
if self.options.shared:
self.copy(pattern="*.dll", dst="bin", src=bin_dir, keep_path=False)
tools.rename(os.path.join(self.package_folder, "lib", "liblzma.lib"),
copy(self, "*.dll", src=bin_dir, dst=os.path.join(self.package_folder, "bin"), keep_path=False)
rename(self, os.path.join(self.package_folder, "lib", "liblzma.lib"),
os.path.join(self.package_folder, "lib", "lzma.lib"))
else:
autotools = self._configure_autotools()
autotools.install()
tools.rmdir(os.path.join(self.package_folder, "lib", "pkgconfig"))
tools.rmdir(os.path.join(self.package_folder, "share"))
tools.remove_files_by_mask(os.path.join(self.package_folder, "lib"), "*.la")
autotools = Autotools(self)
self.win_bash = True
# TODO: replace by autotools.install() once https://github.com/conan-io/conan/issues/12153 fixed
autotools.install(args=[f"DESTDIR={unix_path(self, self.package_folder)}"])
self.win_bash = None
rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig"))
rmdir(self, os.path.join(self.package_folder, "share"))
rm(self, "*.la", os.path.join(self.package_folder, "lib"))
fix_apple_shared_install_name(self)

self._create_cmake_module_variables(
os.path.join(self.package_folder, self._module_file_rel_path),
tools.Version(self.version)
)

@staticmethod
def _create_cmake_module_variables(module_file, version):
def _create_cmake_module_variables(self, module_file):
# TODO: also add LIBLZMA_HAS_AUTO_DECODER, LIBLZMA_HAS_EASY_ENCODER & LIBLZMA_HAS_LZMA_PRESET
content = textwrap.dedent("""\
if(DEFINED LibLZMA_FOUND)
set(LIBLZMA_FOUND ${{LibLZMA_FOUND}})
endif()
content = textwrap.dedent(f"""\
set(LIBLZMA_FOUND TRUE)
if(DEFINED LibLZMA_INCLUDE_DIRS)
set(LIBLZMA_INCLUDE_DIRS ${{LibLZMA_INCLUDE_DIRS}})
endif()
if(DEFINED LibLZMA_LIBRARIES)
set(LIBLZMA_LIBRARIES ${{LibLZMA_LIBRARIES}})
endif()
set(LIBLZMA_VERSION_MAJOR {major})
set(LIBLZMA_VERSION_MINOR {minor})
set(LIBLZMA_VERSION_PATCH {patch})
set(LIBLZMA_VERSION_STRING "{major}.{minor}.{patch}")
""".format(major=version.major, minor=version.minor, patch=version.patch))
tools.save(module_file, content)
set(LIBLZMA_VERSION_MAJOR {Version(self.version).major})
set(LIBLZMA_VERSION_MINOR {Version(self.version).minor})
set(LIBLZMA_VERSION_PATCH {Version(self.version).patch})
set(LIBLZMA_VERSION_STRING "{self.version}")
""")
save(self, module_file, content)

@property
def _module_file_rel_path(self):
return os.path.join("lib", "cmake", "conan-official-{}-variables.cmake".format(self.name))
return os.path.join("lib", "cmake", f"conan-official-{self.name}-variables.cmake")

def package_info(self):
self.cpp_info.set_property("cmake_find_mode", "both")
Expand All @@ -192,7 +239,7 @@ def package_info(self):
self.cpp_info.defines.append("LZMA_API_STATIC")
if self.settings.os in ["Linux", "FreeBSD"]:
self.cpp_info.system_libs.append("pthread")
self.cpp_info.libs = tools.collect_libs(self)
self.cpp_info.libs = collect_libs(self)

# TODO: to remove in conan v2 once cmake_find_package* & pkg_config generators removed
self.cpp_info.names["cmake_find_package"] = "LibLZMA"
Expand Down
26 changes: 21 additions & 5 deletions recipes/xz_utils/all/test_package/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,10 +1,26 @@
cmake_minimum_required(VERSION 3.1)
project(test_package C)

include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake)
conan_basic_setup(TARGETS)
project(test_package LANGUAGES C)

find_package(LibLZMA REQUIRED)

add_executable(${PROJECT_NAME} test_package.c)
target_link_libraries(${PROJECT_NAME} LibLZMA::LibLZMA)
target_link_libraries(${PROJECT_NAME} PRIVATE LibLZMA::LibLZMA)

# Test whether variables from https://cmake.org/cmake/help/latest/module/FindLibLZMA.html
# are properly defined in conan generators
set(_custom_vars
LIBLZMA_FOUND
LIBLZMA_INCLUDE_DIRS
LIBLZMA_LIBRARIES
LIBLZMA_VERSION_MAJOR
LIBLZMA_VERSION_MINOR
LIBLZMA_VERSION_PATCH
LIBLZMA_VERSION_STRING
)
foreach(_custom_var ${_custom_vars})
if(DEFINED _custom_var)
message(STATUS "${_custom_var}: ${${_custom_var}}")
else()
message(FATAL_ERROR "${_custom_var} not defined")
endif()
endforeach()
28 changes: 14 additions & 14 deletions recipes/xz_utils/all/test_package/conanfile.py
Original file line number Diff line number Diff line change
@@ -1,26 +1,26 @@
from conans import ConanFile, CMake, tools
from conan import ConanFile
from conan.tools.build import can_run
from conan.tools.cmake import CMake, cmake_layout
import os


class TestPackageConan(ConanFile):
settings = "os", "compiler", "build_type", "arch"
generators = "cmake", "cmake_find_package"
settings = "os", "arch", "compiler", "build_type"
generators = "CMakeToolchain", "CMakeDeps", "VirtualRunEnv"
test_type = "explicit"

def build_requirements(self):
if self.settings.os == "Macos" and self.settings.arch == "armv8":
# Workaround for CMake bug with error message:
# Attempting to use @rpath without CMAKE_SHARED_LIBRARY_RUNTIME_C_FLAG being
# set. This could be because you are using a Mac OS X version less than 10.5
# or because CMake's platform configuration is corrupt.
# FIXME: Remove once CMake on macOS/M1 CI runners is upgraded.
self.build_requires("cmake/3.22.0")
def layout(self):
cmake_layout(self)

def requirements(self):
self.requires(self.tested_reference_str)

def build(self):
cmake = CMake(self)
cmake.configure()
cmake.build()

def test(self):
if not tools.cross_building(self):
bin_path = os.path.join("bin", "test_package")
self.run(bin_path, run_environment=True)
if can_run(self):
bin_path = os.path.join(self.cpp.build.bindirs[0], "test_package")
self.run(bin_path, env="conanrun")
Loading