Skip to content

Commit

Permalink
Support PyPi suffix and local suffix at the same time
Browse files Browse the repository at this point in the history
  • Loading branch information
perry2of5 committed Nov 15, 2024
1 parent a89d952 commit 478f3e2
Show file tree
Hide file tree
Showing 3 changed files with 69 additions and 31 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@
run_command,
)
from airflow_breeze.utils.shared_options import get_dry_run, get_verbose
from airflow_breeze.utils.version_utils import get_latest_airflow_version, get_latest_helm_chart_version
from airflow_breeze.utils.version_utils import get_latest_airflow_version, get_latest_helm_chart_version, create_package_version, is_local_package_version
from airflow_breeze.utils.versions import is_pre_release
from airflow_breeze.utils.virtualenv_utils import create_pip_command, create_venv

Expand Down Expand Up @@ -868,9 +868,10 @@ def basic_provider_checks(provider_package_id: str) -> dict[str, Any]:
"--version-suffix-for-local",
default=None,
show_default=False,
help="Version suffix for local builds. It must start with a plus sign ('+') and contain only ascii "
"letters, numbers, and periods. The first character after the plus must be an ascii letter or number "
"and the last character must be an ascii letter or number.",
help="Version suffix for local builds. Do not provide the leading plus sign ('+'). The suffix must "
"contain only ascii letters, numbers, and periods. The first character must be an ascii letter or number "
"and the last character must be an ascii letter or number. Note: the local suffix will be appended after "
"the PyPi suffix if both are provided.",
)
@click.option(
"--skip-deleting-generated-files",
Expand Down Expand Up @@ -935,7 +936,8 @@ def prepare_provider_packages(
include_removed=include_removed_providers,
include_not_ready=include_not_ready_providers,
)
if not skip_tag_check and not version_suffix_for_local:
package_version = create_package_version(version_suffix_for_pypi, version_suffix_for_local)
if not skip_tag_check and not is_local_package_version(package_version):
run_command(["git", "remote", "rm", "apache-https-for-providers"], check=False, stderr=DEVNULL)
make_sure_remote_apache_exists_and_fetch(github_repository=github_repository)
success_packages = []
Expand All @@ -948,7 +950,6 @@ def prepare_provider_packages(
shutil.rmtree(DIST_DIR, ignore_errors=True)
DIST_DIR.mkdir(parents=True, exist_ok=True)
for provider_id in packages_list:
package_version = version_suffix_for_pypi if version_suffix_for_pypi else version_suffix_for_local
try:
basic_provider_checks(provider_id)
if not skip_tag_check:
Expand Down
34 changes: 9 additions & 25 deletions dev/breeze/src/airflow_breeze/utils/packages.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@
from airflow_breeze.utils.run_utils import run_command
from airflow_breeze.utils.versions import get_version_tag, strip_leading_zeros_from_version

from airflow_breeze.utils.version_utils import remove_local_version_suffix

MIN_AIRFLOW_VERSION = "2.8.0"
HTTPS_REMOTE = "apache-https-for-providers"

Expand Down Expand Up @@ -420,11 +422,11 @@ def get_dist_package_name_prefix(provider_id: str) -> str:


def apply_version_suffix(install_clause: str, version_suffix: str) -> str:
if (
install_clause.startswith("apache-airflow")
# Need to resolve a version suffix based on PyPi versions, but can ignore local version suffix.
pypi_version_suffix = remove_local_version_suffix(version_suffix)
if (pypi_version_suffix
and install_clause.startswith("apache-airflow")
and ">=" in install_clause
and version_suffix
and not is_local_version_suffix(version_suffix)
):
# Applies version suffix to the apache-airflow and provider package dependencies to make
# sure that pre-release versions have correct limits - this address the issue with how
Expand All @@ -444,31 +446,13 @@ def apply_version_suffix(install_clause: str, version_suffix: str) -> str:

base_version = Version(version).base_version
# always use `pre-release`+ `0` as the version suffix
version_suffix = version_suffix.rstrip("0123456789") + "0"
pypi_version_suffix = pypi_version_suffix.rstrip("0123456789") + "0"

target_version = Version(str(base_version) + "." + version_suffix)
target_version = Version(str(base_version) + "." + pypi_version_suffix)
return prefix + ">=" + str(target_version)
return install_clause


def is_local_version_suffix(version_suffix: str) -> bool:
"""
Check if the given version suffix is a local version suffix. A local version suffix must start with a
plus sign ('+') and be followed only by ascii letters, digits, and periods. They must start and end
with an ascii letter or digit.
Args:
version_suffix (str): The version suffix to check.
Returns:
bool: True if the version suffix starts with '+', False otherwise. Please note this does not
guarantee that the version suffix is a valid local version suffix.
"""
if version_suffix:
return version_suffix.startswith("+")
return False


def get_install_requirements(provider_id: str, version_suffix: str) -> str:
"""
Returns install requirements for the package.
Expand Down Expand Up @@ -628,7 +612,7 @@ def format_version_suffix(version_suffix: str) -> str:
"""
if version_suffix:
if is_local_version_suffix(version_suffix):
if '.' == version_suffix[0] or '+' == version_suffix[0]:
return version_suffix
else:
return f".{version_suffix}"
Expand Down
53 changes: 53 additions & 0 deletions dev/breeze/src/airflow_breeze/utils/version_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,56 @@ def get_latest_airflow_version():
response.raise_for_status()
latest_released_version = response.json()["info"]["version"]
return latest_released_version

def create_package_version(version_suffix_for_pypi: str, version_suffix_for_local: str) -> str:
"""
Creates a package version by combining the version suffix for PyPI and the version suffix for local. If
either one is an empty string, it is ignored. If the local suffix does not have a leading plus sign,
the leading plus sign will be added.
Args:
version_suffix_for_pypi (str): The version suffix for PyPI.
version_suffix_for_local (str): The version suffix for local.
Returns:
str: The combined package version.
"""
# if there is no local version suffix, return the PyPi version suffix
if not version_suffix_for_local:
return version_suffix_for_pypi

# ensure the local version suffix starts with a plus sign
if version_suffix_for_local[0] != "+":
version_suffix_for_local = '+' + version_suffix_for_local

# if there is a PyPi version suffix, return the combined version. Otherwise just return the local version.
if version_suffix_for_pypi:
return version_suffix_for_pypi + version_suffix_for_local
else:
return version_suffix_for_local


def remove_local_version_suffix(version_suffix: str) -> str:
if '+' in version_suffix:
return version_suffix.split('+')[0]
else:
return version_suffix


def is_local_package_version(version_suffix: str) -> bool:
"""
Check if the given version suffix is a local version suffix. A local version suffix will contain a
plus sign ('+'). This function does not guarantee that the version suffix is a valid local version suffix.
Args:
version_suffix (str): The version suffix to check.
Returns:
bool: True if the version suffix contais a '+', False otherwise. Please note this does not
guarantee that the version suffix is a valid local version suffix.
"""
if version_suffix and ('+' in version_suffix):
return True
else:
return False

0 comments on commit 478f3e2

Please sign in to comment.