diff --git a/features/steps/ubuntu_advantage_tools.py b/features/steps/ubuntu_advantage_tools.py index 70ad25f9dc..448e7a2fa7 100644 --- a/features/steps/ubuntu_advantage_tools.py +++ b/features/steps/ubuntu_advantage_tools.py @@ -7,6 +7,7 @@ from features.steps.packages import when_i_apt_install from features.steps.shell import when_i_run_command, when_i_run_shell_command from features.util import ( + ALL_BINARY_PACKAGE_NAMES, SUT, InstallationSource, build_debs, @@ -29,38 +30,39 @@ def when_i_install_uat(context, machine_name=SUT): context, "ubuntu-advantage-pro", machine_name=machine_name ) elif context.pro_config.install_from is InstallationSource.PREBUILT: - deb_paths = sorted( - get_debs_for_series(context.pro_config.debs_path, series) + debs = get_debs_for_series(context.pro_config.debs_path, series) + logging.info("using debs: {}".format(debs)) + to_install = [] + for deb_name, deb_path in debs.non_cloud_pro_image_debs(): + instance_tmp_path = "/tmp/behave_{}.deb".format(deb_name) + instance.push_file(deb_path, instance_tmp_path) + to_install.append(instance_tmp_path) + if is_pro: + for deb_name, deb_path in debs.cloud_pro_image_debs(): + instance_tmp_path = "/tmp/behave_{}.deb".format(deb_name) + instance.push_file(deb_path, instance_tmp_path) + to_install.append(instance_tmp_path) + when_i_apt_install( + context, " ".join(to_install), machine_name=machine_name ) - logging.info("using debs: {}".format(deb_paths)) - for deb_path in deb_paths: - if "advantage-pro" not in deb_path or is_pro: - instance.push_file(deb_path, "/tmp/behave_ua.deb") - when_i_apt_install( - context, "/tmp/behave_ua.deb", machine_name=machine_name - ) - instance.execute("sudo rm /tmp/behave_ua.deb") elif context.pro_config.install_from is InstallationSource.LOCAL: - ua_deb_path, pro_deb_path, l10n_deb_path = build_debs( + debs = build_debs( series, sbuild_output_to_terminal=context.pro_config.sbuild_output_to_terminal, # noqa: E501 ) - instance.push_file(ua_deb_path, "/tmp/behave_ua.deb") - when_i_apt_install( - context, "/tmp/behave_ua.deb", machine_name=machine_name - ) - instance.execute("sudo rm /tmp/behave_ua.deb") - instance.push_file(l10n_deb_path, "/tmp/behave_ua.deb") + to_install = [] + for deb_name, deb_path in debs.non_cloud_pro_image_debs(): + instance_tmp_path = "/tmp/behave_{}.deb".format(deb_name) + instance.push_file(deb_path, instance_tmp_path) + to_install.append(instance_tmp_path) + if is_pro: + for deb_name, deb_path in debs.cloud_pro_image_debs(): + instance_tmp_path = "/tmp/behave_{}.deb".format(deb_name) + instance.push_file(deb_path, instance_tmp_path) + to_install.append(instance_tmp_path) when_i_apt_install( - context, "/tmp/behave_ua.deb", machine_name=machine_name + context, " ".join(to_install), machine_name=machine_name ) - instance.execute("sudo rm /tmp/behave_ua.deb") - if is_pro: - instance.push_file(pro_deb_path, "/tmp/behave_ua.deb") - when_i_apt_install( - context, "/tmp/behave_ua.deb", machine_name=machine_name - ) - instance.execute("sudo rm /tmp/behave_ua.deb") elif context.pro_config.install_from is InstallationSource.DAILY: instance.execute("sudo add-apt-repository ppa:ua-client/daily") instance.execute("sudo apt update") @@ -110,23 +112,16 @@ def when_i_install_uat(context, machine_name=SUT): machine_name=machine_name, ) - context.text = "Package: ubuntu-advantage-tools\nPin: release a={series}-proposed\nPin-Priority: 1001\n".format( # noqa: E501 - series=series - ) - when_i_create_file_with_content( - context, - "/etc/apt/preferences.d/uatools-proposed", - machine_name=machine_name, - ) - - context.text = "Package: ubuntu-advantage-pro\nPin: release a={series}-proposed\nPin-Priority: 1001\n".format( # noqa: E501 - series=series - ) - when_i_create_file_with_content( - context, - "/etc/apt/preferences.d/uapro-proposed", - machine_name=machine_name, - ) + for package in ALL_BINARY_PACKAGE_NAMES: + context.text = "Package: {package}\nPin: release a={series}-proposed\nPin-Priority: 1001\n".format( # noqa: E501 + package=package, + series=series, + ) + when_i_create_file_with_content( + context, + "/etc/apt/preferences.d/{}-proposed".format(package), + machine_name=machine_name, + ) instance.execute("sudo apt update") when_i_apt_install( @@ -153,15 +148,15 @@ def when_i_install_uat(context, machine_name=SUT): @when("I have the `{series}` debs under test in `{dest}`") def when_i_have_the_debs_under_test(context, series, dest): if context.pro_config.install_from is InstallationSource.LOCAL: - deb_paths = build_debs( + debs = build_debs( series, sbuild_output_to_terminal=context.pro_config.sbuild_output_to_terminal, # noqa: E501 ) - for deb_path in deb_paths: - tools_or_pro = "tools" if "tools" in deb_path else "pro" - dest_path = "{}/ubuntu-advantage-{}.deb".format(dest, tools_or_pro) - context.machines[SUT].instance.push_file(deb_path, dest_path) + for deb_name, deb_path in debs.all_debs(): + context.machines[SUT].instance.push_file( + deb_path, "{}/{}.deb".format(dest, deb_name) + ) else: if context.pro_config.install_from is InstallationSource.PROPOSED: ppa_opts = "" @@ -193,16 +188,12 @@ def when_i_have_the_debs_under_test(context, series, dest): logging.info("Download command `{}`".format(download_cmd)) logging.info("stdout: {}".format(context.process.stdout)) logging.info("stderr: {}".format(context.process.stderr)) - when_i_run_shell_command( - context, - "cp ubuntu-advantage-tools*.deb ubuntu-advantage-tools.deb", - "with sudo", - ) - when_i_run_shell_command( - context, - "cp ubuntu-advantage-pro*.deb ubuntu-advantage-pro.deb", - "with sudo", - ) + for package in ALL_BINARY_PACKAGE_NAMES: + when_i_run_shell_command( + context, + "cp {package}*.deb {package}.deb".format(package=package), + "with sudo", + ) @when( @@ -250,9 +241,9 @@ def create_local_ppa(context, release): release, sbuild_output_to_terminal=context.pro_config.sbuild_output_to_terminal, ) - for deb in debs: - deb_destination = "/tmp/" + deb.split("/")[-1] - context.machines["ppa"].instance.push_file(deb, deb_destination) + for deb_name, deb_path in debs.all_debs(): + deb_destination = "/tmp/{}.deb".format(deb_name) + context.machines["ppa"].instance.push_file(deb_path, deb_destination) when_i_run_command( context, "aptly repo add repo-{} {}".format(release, deb_destination), @@ -271,19 +262,21 @@ def create_local_ppa(context, release): def when_i_install_pro(context, machine_name=SUT): if context.pro_config.install_from is InstallationSource.LOCAL: series = context.machines[machine_name].series - deb_paths = build_debs( + debs = build_debs( series, sbuild_output_to_terminal=context.pro_config.sbuild_output_to_terminal, # noqa: E501 ) - for deb_path in deb_paths: - if "advantage-pro" in deb_path: - context.machines[machine_name].instance.push_file( - deb_path, "/tmp/pro.deb" - ) - when_i_run_command( - context, "dpkg -i /tmp/pro.deb", "with sudo" - ) + to_install = [] + for deb_name, deb_path in debs.cloud_pro_image_debs(): + instance_tmp_path = "/tmp/behave_{}.deb".format(deb_name) + context.machines[machine_name].instance.push_file( + deb_path, instance_tmp_path + ) + to_install.append(instance_tmp_path) + when_i_apt_install( + context, " ".join(to_install), machine_name=machine_name + ) else: when_i_run_command( context, "apt-get install ubuntu-advantage-pro", "with sudo" diff --git a/features/util.py b/features/util.py index 69f799fb88..758a2d82b4 100644 --- a/features/util.py +++ b/features/util.py @@ -11,8 +11,9 @@ import tempfile import time from base64 import b64encode +from dataclasses import dataclass from enum import Enum -from typing import Callable, Iterable, List, Optional +from typing import Callable, Iterable, List, Optional, Tuple from urllib.parse import quote from urllib.request import Request, urlopen @@ -33,6 +34,43 @@ UA_DEB_BUILD_CACHE = os.path.join(UA_TMP_DIR, "deb-cache") +ALL_BINARY_PACKAGE_NAMES = [ + "ubuntu-pro-client", + "ubuntu-pro-client-l10n", + "ubuntu-pro-image-auto-attach", + "ubuntu-advantage-tools", + "ubuntu-advantage-pro", +] + + +@dataclass +class ProDebPaths: + ubuntu_pro_client: str + ubuntu_pro_image_auto_attach: str + ubuntu_pro_client_l10n: str + ubuntu_advantage_tools: str + ubuntu_advantage_pro: str + + def non_cloud_pro_image_debs(self) -> List[Tuple[str, str]]: + return [ + ("ubuntu-pro-client", self.ubuntu_pro_client), + ("ubuntu-advantage-tools", self.ubuntu_advantage_tools), + ("ubuntu-pro-client-l10n", self.ubuntu_pro_client_l10n), + ] + + def cloud_pro_image_debs(self) -> List[Tuple[str, str]]: + return [ + ( + "ubuntu-pro-image-auto-attach", + self.ubuntu_pro_image_auto_attach, + ), + ("ubuntu-advantage-pro", self.ubuntu_advantage_pro), + ] + + def all_debs(self) -> List[Tuple[str, str]]: + return self.non_cloud_pro_image_debs() + self.cloud_pro_image_debs() + + class InstallationSource(Enum): ARCHIVE = "archive" PREBUILT = "prebuilt" @@ -114,12 +152,36 @@ def repo_state_hash( return hashlib.md5(output_to_hash).hexdigest() -def get_debs_for_series(debs_path: str, series: str) -> List[str]: - return [ - os.path.join(debs_path, deb_file) - for deb_file in os.listdir(debs_path) - if series in deb_file - ] +def get_debs_for_series(debs_path: str, series: str) -> ProDebPaths: + ubuntu_pro_client = "" + ubuntu_pro_client_l10n = "" + ubuntu_pro_image_auto_attach = "" + ubuntu_advantage_tools = "" + ubuntu_advantage_pro = "" + for deb_file in os.listdir(debs_path): + if series in deb_file: + full_path = os.path.join(debs_path, deb_file) + if "ubuntu-pro-client-l10n" in deb_file: + ubuntu_pro_client_l10n = full_path + elif "ubuntu-pro-client" in deb_file: + ubuntu_pro_client = full_path + elif "ubuntu-pro-image-auto-attach" in deb_file: + ubuntu_pro_image_auto_attach = full_path + elif "ubuntu-advantage-tools" in deb_file: + ubuntu_advantage_tools = full_path + elif "ubuntu-advantage-pro" in deb_file: + ubuntu_advantage_pro = full_path + return ProDebPaths( + ubuntu_pro_client=ubuntu_pro_client, + ubuntu_pro_client_l10n=ubuntu_pro_client_l10n, + ubuntu_pro_image_auto_attach=ubuntu_pro_image_auto_attach, + ubuntu_advantage_tools=ubuntu_advantage_tools, + ubuntu_advantage_pro=ubuntu_advantage_pro, + ) + + +def _create_deb_path(prefix: str, name: str): + return os.path.join(UA_DEB_BUILD_CACHE, "{}{}.deb".format(prefix, name)) def build_debs( @@ -127,7 +189,7 @@ def build_debs( architecture: Optional[str] = None, chroot: Optional[str] = None, sbuild_output_to_terminal: bool = False, -) -> List[str]: +) -> ProDebPaths: """ Build the package through sbuild and store the debs into output_deb_dir @@ -141,25 +203,28 @@ def build_debs( architecture = get_dpkg_arch() deb_prefix = "{}-{}-{}-".format(series, architecture, repo_state_hash()) - tools_deb_name = "{}ubuntu-advantage-tools.deb".format(deb_prefix) - pro_deb_name = "{}ubuntu-advantage-pro.deb".format(deb_prefix) - l10n_deb_name = "{}ubuntu-pro-client-l10n.deb".format(deb_prefix) - tools_deb_cache_path = os.path.join(UA_DEB_BUILD_CACHE, tools_deb_name) - pro_deb_cache_path = os.path.join(UA_DEB_BUILD_CACHE, pro_deb_name) - l10n_deb_cache_path = os.path.join(UA_DEB_BUILD_CACHE, l10n_deb_name) + deb_paths = ProDebPaths( + ubuntu_pro_client=_create_deb_path(deb_prefix, "ubuntu-pro-client"), + ubuntu_pro_image_auto_attach=_create_deb_path( + deb_prefix, "ubuntu-pro-image-auto-attach" + ), + ubuntu_pro_client_l10n=_create_deb_path( + deb_prefix, "ubuntu-pro-client-l10n" + ), + ubuntu_advantage_tools=_create_deb_path( + deb_prefix, "ubuntu-advantage-tools" + ), + ubuntu_advantage_pro=_create_deb_path( + deb_prefix, "ubuntu-advantage-pro" + ), + ) if not os.path.exists(UA_DEB_BUILD_CACHE): os.makedirs(UA_DEB_BUILD_CACHE) - if os.path.exists(tools_deb_cache_path) and os.path.exists( - pro_deb_cache_path - ): - logging.info( - "--- Using debs in cache: {} and {} and {}".format( - tools_deb_cache_path, pro_deb_cache_path, l10n_deb_cache_path - ) - ) - return [tools_deb_cache_path, pro_deb_cache_path, l10n_deb_cache_path] + if os.path.exists(deb_paths.ubuntu_pro_client): + logging.info("--- Using debs in cache") + return deb_paths logging.info("--- Creating: {}".format(SOURCE_PR_TGZ)) @@ -248,17 +313,21 @@ def build_debs( for f in os.listdir(SBUILD_DIR): if f.endswith(".deb"): - if "l10n" in f: - dest = l10n_deb_cache_path - elif "pro" in f: - dest = pro_deb_cache_path - elif "tools" in f: - dest = tools_deb_cache_path + if "ubuntu-pro-client-l10n" in f: + dest = deb_paths.ubuntu_pro_client_l10n + elif "ubuntu-pro-client" in f: + dest = deb_paths.ubuntu_pro_client + elif "ubuntu-pro-image-auto-attach" in f: + dest = deb_paths.ubuntu_pro_image_auto_attach + elif "ubuntu-advantage-tools" in f: + dest = deb_paths.ubuntu_advantage_tools + elif "ubuntu-advantage-pro" in f: + dest = deb_paths.ubuntu_advantage_pro else: continue shutil.copy(os.path.join(SBUILD_DIR, f), dest) - return [tools_deb_cache_path, pro_deb_cache_path, l10n_deb_cache_path] + return deb_paths class SafeLoaderWithoutDatetime(yaml.SafeLoader): diff --git a/tools/build.py b/tools/build.py index c0d4aa098f..6f4a270824 100644 --- a/tools/build.py +++ b/tools/build.py @@ -39,7 +39,7 @@ def main(series=None, chroot=None, arch=None, quiet=False): chroot=chroot, sbuild_output_to_terminal=not quiet, architecture=arch, - ), + ).__dict__, } ) ) diff --git a/tools/test-in-lxd.sh b/tools/test-in-lxd.sh index b1e84b73b8..3ff732c521 100755 --- a/tools/test-in-lxd.sh +++ b/tools/test-in-lxd.sh @@ -13,8 +13,9 @@ SHELL_BEFORE=${SHELL_BEFORE:-0} series=${1:-jammy} build_out=$(./tools/build.sh "$series") hash=$(echo "$build_out" | jq -r .state_hash) -tools_deb=$(echo "$build_out" | jq -r '.debs[]' | grep tools) -l10n_deb=$(echo "$build_out" | jq -r '.debs[]' | grep l10n) +ubuntu_advantage_tools_deb=$(echo "$build_out" | jq -r '.debs.ubuntu_advantage_tools') +ubuntu_pro_client_deb=$(echo "$build_out" | jq -r '.debs.ubuntu_pro_client') +ubuntu_pro_client_l10n_deb=$(echo "$build_out" | jq -r '.debs.ubuntu_pro_client_l10n') name=ua-$series-$hash flags= @@ -29,8 +30,9 @@ if [[ "$VM" -ne 0 ]]; then echo "vms take a while before the agent is ready" sleep 30 fi -lxc file push "$tools_deb" "${name}/tmp/ua_tools.deb" -lxc file push "$l10n_deb" "${name}/tmp/ua_l10n.deb" +lxc file push "$ubuntu_advantage_tools_deb" "${name}/tmp/ua_tools.deb" +lxc file push "$ubuntu_pro_client_deb" "${name}/tmp/pro.deb" +lxc file push "$ubuntu_pro_client_l10n_deb" "${name}/tmp/pro_l10n.deb" if [[ "$SHELL_BEFORE" -ne 0 ]]; then set +x @@ -43,6 +45,5 @@ if [[ "$SHELL_BEFORE" -ne 0 ]]; then lxc exec "$name" bash fi -lxc exec "$name" -- dpkg -i /tmp/ua_tools.deb -lxc exec "$name" -- dpkg -i /tmp/ua_l10n.deb +lxc exec "$name" -- apt install /tmp/ua_tools.deb /tmp/pro.deb /tmp/pro_l10n.deb lxc shell "$name" diff --git a/tools/test-in-multipass.sh b/tools/test-in-multipass.sh index 18790d75c0..36dd315c27 100755 --- a/tools/test-in-multipass.sh +++ b/tools/test-in-multipass.sh @@ -11,18 +11,20 @@ SHELL_BEFORE=${SHELL_BEFORE:-0} series=${1:-jammy} build_out=$(./tools/build.sh "$series") hash=$(echo "$build_out" | jq -r .state_hash) -tools_deb=$(echo "$build_out" | jq -r '.debs[]' | grep tools) -l10n_deb=$(echo "$build_out" | jq -r '.debs[]' | grep l10n) +ubuntu_advantage_tools_deb=$(echo "$build_out" | jq -r '.debs.ubuntu_advantage_tools') +ubuntu_pro_client_deb=$(echo "$build_out" | jq -r '.debs.ubuntu_pro_client') +ubuntu_pro_client_l10n_deb=$(echo "$build_out" | jq -r '.debs.ubuntu_pro_client_l10n') name=ua-$series-$hash multipass delete "$name" --purge || true multipass launch "$series" --name "$name" -sleep 30 # Snaps won't access /tmp -cp "$tools_deb" ~/ua_tools.deb -cp "$l10n_deb" ~/ua_l10n.deb +cp "$ubuntu_advantage_tools_deb" ~/ua_tools.deb +cp "$ubuntu_pro_client_l10n_deb" ~/pro_l10n.deb +cp "$ubuntu_pro_client_deb" ~/pro.deb multipass transfer ~/ua_tools.deb "${name}:/tmp/ua_tools.deb" -multipass transfer ~/ua_l10n.deb "${name}:/tmp/ua_l10n.deb" +multipass transfer ~/pro_l10n.deb "${name}:/tmp/pro_l10n.deb" +multipass transfer ~/pro.deb "${name}:/tmp/pro.deb" rm -f ~/ua_tools.deb rm -f ~/ua_l10n.deb @@ -37,6 +39,5 @@ if [[ "$SHELL_BEFORE" -ne 0 ]]; then multipass exec "$name" bash fi -multipass exec "$name" -- sudo dpkg -i /tmp/ua_tools.deb -multipass exec "$name" -- sudo dpkg -i /tmp/ua_l10n.deb +multipass exec "$name" -- sudo apt install /tmp/ua_tools.deb /tmp/pro.deb /tmp/pro_l10n.deb multipass shell "$name"