From df0c0f6df3c9cce0db4817c4363d003d2bb3f055 Mon Sep 17 00:00:00 2001 From: Russell Martin Date: Thu, 13 Jul 2023 20:04:15 -0400 Subject: [PATCH] Improve support for arbitrary machine architectures - Prevent running on 32bit Windows or 32bit Python on 64bit Windows - Support building an AppImage for i386; verify LinuxDeploy for all Commands - Support building Linux System packages on i386 - Support installing JDK for armv7/8 - Let users know when Android SDK must be manually installed - Derive the Linux system target distro architecture from the build environment - Download a 32bit JDK and 32bit standalone Python when host Python is 32bit - Ensure `makepkg` creates a `.pkg.tar.zst` distributable --- src/briefcase/commands/create.py | 2 + src/briefcase/exceptions.py | 20 ++- src/briefcase/integrations/android_sdk.py | 39 ++++-- src/briefcase/integrations/base.py | 2 + src/briefcase/integrations/java.py | 54 +++++--- src/briefcase/integrations/linuxdeploy.py | 20 ++- src/briefcase/platforms/linux/__init__.py | 9 +- src/briefcase/platforms/linux/appimage.py | 13 +- src/briefcase/platforms/linux/system.py | 131 ++++++++++++-------- src/briefcase/platforms/windows/__init__.py | 24 +++- 10 files changed, 219 insertions(+), 95 deletions(-) diff --git a/src/briefcase/commands/create.py b/src/briefcase/commands/create.py index d04e5c660..3a165c7ca 100644 --- a/src/briefcase/commands/create.py +++ b/src/briefcase/commands/create.py @@ -344,6 +344,7 @@ def _download_support_package(self, app: AppConfig): python_version_tag=self.python_version_tag, platform=self.platform, host_arch=self.tools.host_arch, + is_32bit=self.tools.is_32bit_python, ) support_package_url = self.support_package_url(support_revision) @@ -384,6 +385,7 @@ def _download_support_package(self, app: AppConfig): python_version_tag=self.python_version_tag, platform=self.platform, host_arch=self.tools.host_arch, + is_32bit=self.tools.is_32bit_python, ) from e def _write_requirements_file( diff --git a/src/briefcase/exceptions.py b/src/briefcase/exceptions.py index 005f78096..14fcd4034 100644 --- a/src/briefcase/exceptions.py +++ b/src/briefcase/exceptions.py @@ -103,6 +103,18 @@ def __init__(self, tool): super().__init__(msg=f"Unable to locate {tool!r}. Has it been installed?") +class IncompatibleToolError(BriefcaseCommandError): + def __init__(self, tool: str, env_var: str): + self.tool = tool + super().__init__( + msg=f"""\ +Briefcase cannot install {tool} on this machine. + +Install {tool} manually and specify the installation directory in the {env_var} environment variable. + """ + ) + + class NonManagedToolError(BriefcaseCommandError): def __init__(self, tool): self.tool = tool @@ -153,16 +165,16 @@ def __init__(self, app_bundle_path): class MissingSupportPackage(BriefcaseCommandError): - def __init__(self, python_version_tag, platform, host_arch): + def __init__(self, python_version_tag, platform, host_arch, is_32bit): self.python_version_tag = python_version_tag - self.platform = platform + self.platform = f"{'32 bit ' if is_32bit else ''}{platform}" self.host_arch = host_arch super().__init__( f"""\ Unable to download {self.platform} support package for Python {self.python_version_tag} on {self.host_arch}. -This is likely because either Python {self.python_version_tag} and/or {self.host_arch} -is not yet supported on {self.platform}. You will need to: +This is likely because either Python {self.python_version_tag} and/or {self.host_arch} is not yet +supported on {self.platform}. You will need to: * Use an older version of Python; or * Compile your own custom support package. """ diff --git a/src/briefcase/integrations/android_sdk.py b/src/briefcase/integrations/android_sdk.py index 2147ab07a..643625fc8 100644 --- a/src/briefcase/integrations/android_sdk.py +++ b/src/briefcase/integrations/android_sdk.py @@ -15,6 +15,7 @@ from briefcase.console import InputDisabled, select_option from briefcase.exceptions import ( BriefcaseCommandError, + IncompatibleToolError, InvalidDeviceError, MissingToolError, ) @@ -57,15 +58,35 @@ def __init__(self, tools: ToolCache, root_path: Path): @property def cmdline_tools_url(self) -> str: - """The Android SDK Command-Line Tools URL appropriate to the current operating - system.""" - platform_name = self.tools.host_os.lower() - if self.tools.host_os.lower() == "darwin": - platform_name = "mac" - elif self.tools.host_os.lower() == "windows": # pragma: no branch - platform_name = "win" - - return f"https://dl.google.com/android/repository/commandlinetools-{platform_name}-{self.cmdline_tools_version}_latest.zip" # noqa: E501 + """The Android SDK Command-Line Tools URL appropriate for the current machine. + + The SDK largely only supports typical development environments; if a machine is + using an unsupported architecture, `sdkmanager` will error while installing the + emulator as a dependency of the build-tools. However, for some of the platforms + that are unsupported by sdkmanager, users can set up their own SDK install. + """ + try: + platform_name = { + "Darwin": { + "arm64": "mac", + "x86_64": "mac", + }, + "Linux": { + "x86_64": "linux", + }, + "Windows": { + "AMD64": "win", + }, + }[self.tools.host_os][self.tools.host_arch] + except KeyError as e: + raise IncompatibleToolError( + tool=self.full_name, env_var="ANDROID_HOME" + ) from e + + return ( + f"https://dl.google.com/android/repository/commandlinetools-" + f"{platform_name}-{self.cmdline_tools_version}_latest.zip" + ) @property def cmdline_tools_path(self) -> Path: diff --git a/src/briefcase/integrations/base.py b/src/briefcase/integrations/base.py index e054c02a7..472c43749 100644 --- a/src/briefcase/integrations/base.py +++ b/src/briefcase/integrations/base.py @@ -195,6 +195,8 @@ def __init__( self.host_arch = self.platform.machine() self.host_os = self.platform.system() + # Python is 32bit if its pointers can only address with 32 bits or fewer + self.is_32bit_python = self.sys.maxsize <= 2**32 self.app_tools: DefaultDict[AppConfig, ToolCache] = defaultdict( lambda: ToolCache( diff --git a/src/briefcase/integrations/java.py b/src/briefcase/integrations/java.py index 69f5bf8bf..bbd9d0213 100644 --- a/src/briefcase/integrations/java.py +++ b/src/briefcase/integrations/java.py @@ -5,7 +5,11 @@ import subprocess from pathlib import Path -from briefcase.exceptions import BriefcaseCommandError, MissingToolError +from briefcase.exceptions import ( + BriefcaseCommandError, + IncompatibleToolError, + MissingToolError, +) from briefcase.integrations.base import ManagedTool, ToolCache @@ -26,28 +30,40 @@ def __init__(self, tools: ToolCache, java_home: Path): @property def OpenJDK_download_url(self): - arch = { - "x86_64": "x64", # Linux\macOS x86-64 - "aarch64": "aarch64", # Linux arm64 - "armv6l": "arm", # Linux arm - "arm64": "aarch64", # macOS arm64 - "AMD64": "x64", # Windows x86-64 - }.get(self.tools.host_arch) - - platform = { - "Darwin": "mac", - "Windows": "windows", - "Linux": "linux", - }.get(self.tools.host_os) - - extension = { - "Windows": "zip", - }.get(self.tools.host_os, "tar.gz") + """The OpenJDK download URL appropriate for the current machine.""" + system_arch = self.tools.host_arch + # use a 32bit JDK if using 32bit Python on 64bit hardware + if self.tools.is_32bit_python and self.tools.host_arch == "aarch64": + system_arch = "armv7l" + + try: + jdk_download_arch = { + "armv7l": "arm", # Linux armv7 + "armv8l": "arm", # Linux armv8 + "aarch64": "aarch64", # Linux arm64 + "arm64": "aarch64", # macOS arm64 + "x86_64": "x64", # Linux/macOS x86-64 + "AMD64": "x64", # Windows x86-64 + }[system_arch] + + platform = { + "Darwin": "mac", + "Windows": "windows", + "Linux": "linux", + }[self.tools.host_os] + + extension = { + "Darwin": "tar.gz", + "Linux": "tar.gz", + "Windows": "zip", + }[self.tools.host_os] + except KeyError as e: + raise IncompatibleToolError(tool=self.full_name, env_var="JAVA_HOME") from e return ( f"https://github.com/adoptium/temurin{self.JDK_MAJOR_VER}-binaries/" f"releases/download/jdk-{self.JDK_RELEASE}+{self.JDK_BUILD}/" - f"OpenJDK{self.JDK_MAJOR_VER}U-jdk_{arch}_{platform}_hotspot_" + f"OpenJDK{self.JDK_MAJOR_VER}U-jdk_{jdk_download_arch}_{platform}_hotspot_" f"{self.JDK_RELEASE}_{self.JDK_BUILD}.{extension}" ) diff --git a/src/briefcase/integrations/linuxdeploy.py b/src/briefcase/integrations/linuxdeploy.py index 9d5d9f5f5..833e236f6 100644 --- a/src/briefcase/integrations/linuxdeploy.py +++ b/src/briefcase/integrations/linuxdeploy.py @@ -11,6 +11,7 @@ BriefcaseCommandError, CorruptToolError, MissingToolError, + UnsupportedHostError, ) from briefcase.integrations.base import ManagedTool, Tool, ToolCache @@ -48,6 +49,21 @@ def download_url(self) -> str: def file_path(self) -> Path: """The folder on the local filesystem that contains the file_name.""" + @classmethod + def arch(cls, host_os: str, host_arch: str) -> str: + # always use the x86-64 arch on macOS since Docker + # containers are always run in an x86-64 VM + arch = host_arch if host_os != "Darwin" else "x86_64" + try: + return { + "x86_64": "x86_64", + "i686": "i386", + }[arch] + except KeyError as e: + raise UnsupportedHostError( + f"Linux AppImages cannot be built on {host_arch}." + ) from e + def exists(self) -> bool: return (self.file_path / self.file_name).is_file() @@ -208,7 +224,7 @@ class LinuxDeployQtPlugin(LinuxDeployPluginBase, ManagedTool): @property def file_name(self) -> str: - return f"linuxdeploy-plugin-qt-{self.tools.host_arch}.AppImage" + return f"linuxdeploy-plugin-qt-{self.arch(self.tools.host_os, self.tools.host_arch)}.AppImage" @property def download_url(self) -> str: @@ -319,7 +335,7 @@ def file_path(self) -> Path: @property def file_name(self) -> str: - return f"linuxdeploy-{self.tools.host_arch}.AppImage" + return f"linuxdeploy-{self.arch(self.tools.host_os, self.tools.host_arch)}.AppImage" @property def download_url(self) -> str: diff --git a/src/briefcase/platforms/linux/__init__.py b/src/briefcase/platforms/linux/__init__.py index 1a2efadd8..12a1338a4 100644 --- a/src/briefcase/platforms/linux/__init__.py +++ b/src/briefcase/platforms/linux/__init__.py @@ -66,10 +66,17 @@ def support_package_url(self, support_revision): System packages don't use a support package; this is defined by the template, so this method won't be invoked """ + python_download_arch = self.tools.host_arch + # use a 32bit Python if using 32bit Python on 64bit hardware + if self.tools.is_32bit_python and self.tools.host_arch == "aarch64": + python_download_arch = "armv7" + elif self.tools.is_32bit_python and self.tools.host_arch == "x86_64": + python_download_arch = "i686" + version, datestamp = support_revision.split("+") return ( "https://github.com/indygreg/python-build-standalone/releases/download/" - f"{datestamp}/cpython-{support_revision}-{self.tools.host_arch}-unknown-linux-gnu-install_only.tar.gz" + f"{datestamp}/cpython-{support_revision}-{python_download_arch}-unknown-linux-gnu-install_only.tar.gz" ) def vendor_details(self, freedesktop_info): diff --git a/src/briefcase/platforms/linux/appimage.py b/src/briefcase/platforms/linux/appimage.py index 5af145b2e..3f9a4f590 100644 --- a/src/briefcase/platforms/linux/appimage.py +++ b/src/briefcase/platforms/linux/appimage.py @@ -44,7 +44,8 @@ def project_path(self, app): def binary_name(self, app): safe_name = app.formal_name.replace(" ", "_") - return f"{safe_name}-{app.version}-{self.tools.host_arch}.AppImage" + arch = LinuxDeploy.arch(self.tools.host_os, self.tools.host_arch) + return f"{safe_name}-{app.version}-{arch}.AppImage" def binary_path(self, app): return self.bundle_path(app) / self.binary_name(app) @@ -52,6 +53,11 @@ def binary_path(self, app): def distribution_path(self, app): return self.dist_path / self.binary_name(app) + def verify_tools(self): + """Verify the AppImage LinuxDeploy tool and its plugins exist.""" + super().verify_tools() + LinuxDeploy.verify(tools=self.tools) + def add_options(self, parser): super().add_options(parser) parser.add_argument( @@ -213,11 +219,6 @@ class LinuxAppImageOpenCommand(LinuxAppImageMostlyPassiveMixin, DockerOpenComman class LinuxAppImageBuildCommand(LinuxAppImageMixin, BuildCommand): description = "Build a Linux AppImage." - def verify_tools(self): - """Verify the AppImage linuxdeploy tool and plugins exist.""" - super().verify_tools() - LinuxDeploy.verify(tools=self.tools) - def build_app(self, app: AppConfig, **kwargs): # pragma: no-cover-if-is-windows """Build an application. diff --git a/src/briefcase/platforms/linux/system.py b/src/briefcase/platforms/linux/system.py index 2a7270844..0ac87a9cf 100644 --- a/src/briefcase/platforms/linux/system.py +++ b/src/briefcase/platforms/linux/system.py @@ -54,15 +54,6 @@ def parse_options(self, extra): return options - @property - def linux_arch(self): - # Linux uses different architecture identifiers for some platforms - return { - "x86_64": "amd64", - "aarch64": "arm64", - "armv6l": "armhf", - }.get(self.tools.host_arch, self.tools.host_arch) - def build_path(self, app): # Override the default build path to use the vendor name, # rather than "linux" @@ -85,31 +76,6 @@ def rpm_tag(self, app): else: return f"el{app.target_codename}" - def distribution_filename(self, app): - if app.packaging_format == "deb": - return ( - f"{app.app_name}_{app.version}-{getattr(app, 'revision', 1)}" - f"~{app.target_vendor}-{app.target_codename}_{self.linux_arch}.deb" - ) - elif app.packaging_format == "rpm": - return ( - f"{app.app_name}-{app.version}-{getattr(app, 'revision', 1)}" - f".{self.rpm_tag(app)}.{self.tools.host_arch}.rpm" - ) - elif app.packaging_format == "pkg": - return ( - f"{app.app_name}-{app.version}-{getattr(app, 'revision', 1)}" - f"-{self.tools.host_arch}.pkg.tar.zst" - ) - else: - raise BriefcaseCommandError( - "Briefcase doesn't currently know how to build system packages in " - f"{app.packaging_format.upper()} format." - ) - - def distribution_path(self, app): - return self.dist_path / self.distribution_filename(app) - def target_glibc_version(self, app): target_glibc = self.tools.os.confstr("CS_GNU_LIBC_VERSION").split()[1] return target_glibc @@ -235,7 +201,7 @@ def use_docker(self): # what "use docker" means in terms of target_image. return bool(self.target_image) - def app_python_version_tag(self, app): + def app_python_version_tag(self, app: AppConfig): if self.use_docker: # If we're running in Docker, we can't know the Python3 version # before rolling out the template; so we fall back to "3". Later, @@ -247,6 +213,68 @@ def app_python_version_tag(self, app): python_version_tag = super().app_python_version_tag(app) return python_version_tag + def deb_abi(self, app: AppConfig) -> str: + """The default ABI for dpkg for the target environment.""" + try: + return self._deb_abi + except AttributeError: + self._deb_abi = ( + self.tools[app] + .app_context.check_output(["dpkg", "--print-architecture"]) + .strip() + ) + return self._deb_abi + + def rpm_abi(self, app: AppConfig) -> str: + """The default ABI for rpm for the target environment.""" + try: + return self._rpm_abi + except AttributeError: + self._rpm_abi = ( + self.tools[app] + .app_context.check_output(["rpm", "--eval", "%_target_cpu"]) + .strip() + ) + return self._rpm_abi + + def pkg_abi(self, app: AppConfig) -> str: + """The default ABI for pacman for the target environment.""" + try: + return self._pkg_abi + except AttributeError: + self._pkg_abi = ( + self.tools[app] + .app_context.check_output(["pacman-conf", "Architecture"]) + .split("\n")[0] + .strip() + ) + return self._pkg_abi + + def distribution_filename(self, app: AppConfig) -> str: + if app.packaging_format == "deb": + return ( + f"{app.app_name}_{app.version}-{getattr(app, 'revision', 1)}" + f"~{app.target_vendor}-{app.target_codename}_{self.deb_abi(app)}.deb" + ) + elif app.packaging_format == "rpm": + return ( + f"{app.app_name}-{app.version}-{getattr(app, 'revision', 1)}" + f".{self.rpm_tag(app)}.{self.rpm_abi(app)}.rpm" + ) + elif app.packaging_format == "pkg": + return ( + f"{app.app_name}-{app.version}-{getattr(app, 'revision', 1)}" + f"-{self.pkg_abi(app)}.pkg.tar.zst" + ) + else: + raise BriefcaseCommandError( + "Briefcase doesn't currently know how to build system packages in " + f"{app.packaging_format.upper()} format." + ) + + def distribution_path(self, app: AppConfig): + return self.dist_path / self.distribution_filename(app) + def target_glibc_version(self, app): """Determine the glibc version. @@ -289,7 +317,7 @@ def target_glibc_version(self, app): return target_glibc - def platform_freedesktop_info(self, app): + def platform_freedesktop_info(self, app: AppConfig): if self.use_docker: # Preserve the target image on the command line as the app's target app.target_image = self.target_image @@ -308,7 +336,7 @@ def platform_freedesktop_info(self, app): return freedesktop_info - def docker_image_tag(self, app): + def docker_image_tag(self, app: AppConfig): """The Docker image tag for an app.""" return f"briefcase/{app.bundle_identifier.lower()}:{app.target_vendor}-{app.target_codename}" @@ -339,7 +367,7 @@ def clone_options(self, command): super().clone_options(command) self.target_image = command.target_image - def verify_python(self, app): + def verify_python(self, app: AppConfig): """Verify that the version of Python being used to build the app in Docker is compatible with the version being used to run Briefcase. @@ -487,7 +515,7 @@ def verify_system_packages(self, app: AppConfig): ) return - # Run a check for each packages listed in the app's system_requires, + # Run a check for each package listed in the app's system_requires, # plus the baseline system packages that are required. missing = [] for package in base_system_packages + getattr(app, "system_requires", []): @@ -880,15 +908,15 @@ def _package_deb(self, app: AppConfig, **kwargs): f.write( "\n".join( [ - f"Package: { app.app_name }", - f"Version: { app.version }", - f"Architecture: { self.linux_arch }", - f"Maintainer: { app.author } <{ app.author_email }>", - f"Homepage: { app.url }", - f"Description: { app.description }", - f" { debian_multiline_description(app.long_description) }", - f"Depends: { system_runtime_requires }", - f"Section: { getattr(app, 'system_section', 'utils') }", + f"Package: {app.app_name}", + f"Version: {app.version}", + f"Architecture: {self.deb_abi(app)}", + f"Maintainer: {app.author } <{app.author_email}>", + f"Homepage: {app.url}", + f"Description: {app.description}", + f" {debian_multiline_description(app.long_description)}", + f"Depends: {system_runtime_requires}", + f"Section: {getattr(app, 'system_section', 'utils')}", "Priority: optional\n", ] ) @@ -994,7 +1022,7 @@ def _package_rpm(self, app: AppConfig, **kwargs): # pragma: no-cover-if-is-wind ] + [ "", - f"ExclusiveArch: {self.tools.host_arch}", + f"ExclusiveArch: {self.rpm_abi(app)}", "", "%description", app.long_description, @@ -1072,7 +1100,7 @@ def _package_rpm(self, app: AppConfig, **kwargs): # pragma: no-cover-if-is-wind self.tools.shutil.move( rpmbuild_path / "RPMS" - / self.tools.host_arch + / self.rpm_abi(app) / self.distribution_filename(app), self.distribution_path(app), ) @@ -1140,7 +1168,7 @@ def _package_pkg(self, app: AppConfig, **kwargs): # pragma: no-cover-if-is-wind f"pkgver={app.version}", f"pkgrel={getattr(app, 'revision', 1)}", f'pkgdesc="{app.description}"', - f"arch=('{self.tools.host_arch}')", + f"arch=('{self.pkg_abi(app)}')", f'url="{app.url}"', f"license=('{app.license}')", f"depends=({system_runtime_requires})", @@ -1162,6 +1190,7 @@ def _package_pkg(self, app: AppConfig, **kwargs): # pragma: no-cover-if-is-wind [ "makepkg", ], + env={"PKGEXT": ".pkg.tar.zst"}, check=True, cwd=pkgbuild_path, ) diff --git a/src/briefcase/platforms/windows/__init__.py b/src/briefcase/platforms/windows/__init__.py index fbd2c18f0..c205efd5e 100644 --- a/src/briefcase/platforms/windows/__init__.py +++ b/src/briefcase/platforms/windows/__init__.py @@ -8,7 +8,7 @@ from briefcase.commands import CreateCommand, PackageCommand, RunCommand from briefcase.config import AppConfig, parsed_version -from briefcase.exceptions import BriefcaseCommandError +from briefcase.exceptions import BriefcaseCommandError, UnsupportedHostError from briefcase.integrations.windows_sdk import WindowsSDK from briefcase.integrations.wix import WiX @@ -27,6 +27,23 @@ def distribution_path(self, app): suffix = "zip" if app.packaging_format == "zip" else "msi" return self.dist_path / f"{app.formal_name}-{app.version}.{suffix}" + def verify_host(self): + super().verify_host() + # the stub app only supports x86-64 right now + if self.tools.host_arch != "AMD64": + raise UnsupportedHostError( + f"Windows applications cannot be built on an {self.tools.host_arch} machine." + ) + # 64bit Python is required to ensure 64bit wheels are installed/created for the app + if self.tools.is_32bit_python: + raise UnsupportedHostError( + """\ +Windows applications cannot be built using a 32bit version of Python. + +Install a 64bit version of Python and run Briefcase again. +""" + ) + class WindowsCreateCommand(CreateCommand): def support_package_filename(self, support_revision): @@ -34,8 +51,9 @@ def support_package_filename(self, support_revision): def support_package_url(self, support_revision): return ( - f"https://www.python.org/ftp/python/{self.python_version_tag}.{support_revision}/" - + self.support_package_filename(support_revision) + f"https://www.python.org/ftp/python/" + f"{self.python_version_tag}.{support_revision}/" + f"{self.support_package_filename(support_revision)}" ) def output_format_template_context(self, app: AppConfig):