diff --git a/cou/apps/auxiliary.py b/cou/apps/auxiliary.py index a0bb5a76..3e3dd7a2 100644 --- a/cou/apps/auxiliary.py +++ b/cou/apps/auxiliary.py @@ -324,12 +324,21 @@ def post_upgrade_steps( return steps def _check_auto_restarts(self) -> None: - """No-op, skip check auto restarts option. + """Check if enable-auto-restarts is enabled. - This method override the parent class's `_check_auto_restarts()` method - because the parent class's will raise an `ApplicationError` if - `enable-auto-restarts` is `True`. + Due to the charm [bug](https://bugs.launchpad.net/charm-rabbitmq-server/+bug/2046381), + if the `enable-auto-restarts` option is enabled, this check will raise an exception. + + :raises ApplicationError: When enable-auto-restarts is enabled. """ + if self.config["enable-auto-restarts"].get("value") is True: + raise ApplicationError( + "`enable-auto-restarts` must be `False` due to " + "https://bugs.launchpad.net/charm-rabbitmq-server/+bug/2046381 " + f"Please run `juju config {self.name} enable-auto-restarts=False` " + "before performing upgrades and rollback to original value after " + "upgrade is completed" + ) @AppFactory.register_application(["ceph-mon"]) diff --git a/docs/reference/known-issues.rst b/docs/reference/known-issues.rst index ec7ea39d..2d285918 100644 --- a/docs/reference/known-issues.rst +++ b/docs/reference/known-issues.rst @@ -23,6 +23,19 @@ See `bug 2060751`_ for details. If affected, refresh the manila-ganesha charm to its most recent version (within the same release channel) and re-run COU. +Rabbitmq Server +--------------- + +The rabbitmq-server charm must have `enable-auto-restarts=False` for **COU** to +work properly due to the known charm bug. + +See `bug 2046381`_ for details. + +We suggest that users should temporarily set `enable-auto-restarts=False` when +performing `cou upgrade`, and rollback to original setting after the upgrade is +completed. + .. LINKS: .. _Issues, charm procedures, and OpenStack upgrade notes: https://docs.openstack.org/charm-guide/latest/project/issues-and-procedures.html .. _bug 2060751: https://bugs.launchpad.net/charm-manila-ganesha/+bug/2060751 +.. _bug 2046381: https://bugs.launchpad.net/charm-rabbitmq-server/+bug/2046381 diff --git a/tests/mocked_plans/sample_plans/018346c5-f95c-46df-a34e-9a78bdec0018.yaml b/tests/mocked_plans/sample_plans/018346c5-f95c-46df-a34e-9a78bdec0018.yaml index 9f6d37d4..8d9f17ba 100644 --- a/tests/mocked_plans/sample_plans/018346c5-f95c-46df-a34e-9a78bdec0018.yaml +++ b/tests/mocked_plans/sample_plans/018346c5-f95c-46df-a34e-9a78bdec0018.yaml @@ -132,7 +132,17 @@ plan: | Ψ Upgrade software packages on unit 'rabbitmq-server/2' WARNING: Changing 'rabbitmq-server' channel from latest/stable to 3.9/stable. This may be a charm downgrade, which is generally not supported. Wait for up to 300s for app 'rabbitmq-server' to reach the idle state + Execute run-deferred-hooks for all 'rabbitmq-server' units to clear any leftover events + Execute run-deferred-hooks on unit: 'rabbitmq-server/0' + Execute run-deferred-hooks on unit: 'rabbitmq-server/1' + Execute run-deferred-hooks on unit: 'rabbitmq-server/2' + Wait for up to 2400s for app 'rabbitmq-server' to reach the idle state Change charm config of 'rabbitmq-server' 'source' to 'cloud:focal-victoria' + Wait for up to 2400s for app 'rabbitmq-server' to reach the idle state + Execute run-deferred-hooks for all 'rabbitmq-server' units to restart the service after upgrade + Execute run-deferred-hooks on unit: 'rabbitmq-server/0' + Execute run-deferred-hooks on unit: 'rabbitmq-server/1' + Execute run-deferred-hooks on unit: 'rabbitmq-server/2' Wait for up to 2400s for model '018346c5-f95c-46df-a34e-9a78bdec0018' to reach the idle state Verify that the workload of 'rabbitmq-server' has been upgraded on units: rabbitmq-server/0, rabbitmq-server/1, rabbitmq-server/2 Upgrade plan for 'ceph-mon' to 'victoria' @@ -8070,7 +8080,7 @@ applications: ' source: default type: boolean - value: true + value: false source: description: "Optional configuration to support use of additional sources such\ \ as:\n.\n - ppa:myteam/ppa\n - cloud:xenial-proposed/ocata\n - http://my.archive.com/ubuntu\ diff --git a/tests/mocked_plans/sample_plans/9eb9af6a-b919-4cf9-8f2f-9df16a1556be.yaml b/tests/mocked_plans/sample_plans/9eb9af6a-b919-4cf9-8f2f-9df16a1556be.yaml index f121aed2..1d527b5e 100644 --- a/tests/mocked_plans/sample_plans/9eb9af6a-b919-4cf9-8f2f-9df16a1556be.yaml +++ b/tests/mocked_plans/sample_plans/9eb9af6a-b919-4cf9-8f2f-9df16a1556be.yaml @@ -179,9 +179,19 @@ plan: | Ψ Upgrade software packages on unit 'rabbitmq-server/2' Refresh 'rabbitmq-server' to the latest revision of '3.8/stable' Wait for up to 300s for app 'rabbitmq-server' to reach the idle state + Execute run-deferred-hooks for all 'rabbitmq-server' units to clear any leftover events + Execute run-deferred-hooks on unit: 'rabbitmq-server/0' + Execute run-deferred-hooks on unit: 'rabbitmq-server/1' + Execute run-deferred-hooks on unit: 'rabbitmq-server/2' + Wait for up to 2400s for app 'rabbitmq-server' to reach the idle state Upgrade 'rabbitmq-server' from '3.8/stable' to the new channel: '3.9/stable' Wait for up to 300s for app 'rabbitmq-server' to reach the idle state Change charm config of 'rabbitmq-server' 'source' to 'cloud:focal-victoria' + Wait for up to 2400s for app 'rabbitmq-server' to reach the idle state + Execute run-deferred-hooks for all 'rabbitmq-server' units to restart the service after upgrade + Execute run-deferred-hooks on unit: 'rabbitmq-server/0' + Execute run-deferred-hooks on unit: 'rabbitmq-server/1' + Execute run-deferred-hooks on unit: 'rabbitmq-server/2' Wait for up to 2400s for model '9eb9af6a-b919-4cf9-8f2f-9df16a1556be' to reach the idle state Verify that the workload of 'rabbitmq-server' has been upgraded on units: rabbitmq-server/0, rabbitmq-server/1, rabbitmq-server/2 Upgrade plan for 'ceph-mon' to 'victoria' @@ -539,7 +549,7 @@ applications: ' source: user type: boolean - value: true + value: false source: description: "Optional configuration to support use of additional sources such\ \ as:\n.\n - ppa:myteam/ppa\n - cloud:xenial-proposed/ocata\n - http://my.archive.com/ubuntu\ diff --git a/tests/unit/apps/test_auxiliary.py b/tests/unit/apps/test_auxiliary.py index 92b41a4d..eab2b9e4 100644 --- a/tests/unit/apps/test_auxiliary.py +++ b/tests/unit/apps/test_auxiliary.py @@ -127,7 +127,7 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_change_channel(model): channel="3.8/stable", config={ "source": {"value": "distro"}, - "enable-auto-restarts": {"value": True}, + "enable-auto-restarts": {"value": False}, }, machines=machines, model=model, @@ -158,6 +158,50 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_change_channel(model): for unit in app.units.keys() ) + run_deferred_hooks_and_restart_pre_upgrades = PreUpgradeStep( + description=( + f"Execute run-deferred-hooks for all '{app.name}' units " + "to clear any leftover events" + ), + parallel=False, + ) + run_deferred_hooks_and_restart_pre_upgrades.add_steps( + [ + UnitUpgradeStep( + description=f"Execute run-deferred-hooks on unit: '{unit.name}'", + coro=model.run_action(unit.name, "run-deferred-hooks", raise_on_failure=True), + ) + for unit in app.units.values() + ] + ) + run_deferred_hooks_and_restart_pre_wait_step = PreUpgradeStep( + description=(f"Wait for up to 2400s for app '{app.name}'" " to reach the idle state"), + parallel=False, + coro=model.wait_for_idle(2400, apps=[app.name]), + ) + + run_deferred_hooks_and_restart_post_wait_step = PostUpgradeStep( + description=(f"Wait for up to 2400s for app '{app.name}'" " to reach the idle state"), + parallel=False, + coro=model.wait_for_idle(2400, apps=[app.name]), + ) + run_deferred_hooks_and_restart_post_upgrades = PostUpgradeStep( + description=( + f"Execute run-deferred-hooks for all '{app.name}' units " + "to restart the service after upgrade" + ), + parallel=False, + ) + run_deferred_hooks_and_restart_post_upgrades.add_steps( + [ + UnitUpgradeStep( + description=f"Execute run-deferred-hooks on unit: '{unit.name}'", + coro=model.run_action(unit.name, "run-deferred-hooks", raise_on_failure=True), + ) + for unit in app.units.values() + ] + ) + upgrade_steps = [ expected_upgrade_package_step, PreUpgradeStep( @@ -170,6 +214,8 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_change_channel(model): parallel=False, coro=model.wait_for_idle(300, apps=[app.name]), ), + run_deferred_hooks_and_restart_pre_upgrades, + run_deferred_hooks_and_restart_pre_wait_step, UpgradeStep( description=f"Upgrade '{app.name}' from '3.8/stable' to the new channel: '3.9/stable'", parallel=False, @@ -189,6 +235,8 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_change_channel(model): {f"{app.origin_setting}": "cloud:focal-victoria"}, ), ), + run_deferred_hooks_and_restart_post_wait_step, + run_deferred_hooks_and_restart_post_upgrades, PostUpgradeStep( description=f"Wait for up to 2400s for model '{model.name}' to reach the idle state", parallel=False, @@ -219,7 +267,7 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria(model): channel="3.9/stable", config={ "source": {"value": "distro"}, - "enable-auto-restarts": {"value": True}, + "enable-auto-restarts": {"value": False}, }, machines=machines, model=model, @@ -249,6 +297,50 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria(model): for unit in app.units.values() ) + run_deferred_hooks_and_restart_pre_upgrades = PreUpgradeStep( + description=( + f"Execute run-deferred-hooks for all '{app.name}' units " + "to clear any leftover events" + ), + parallel=False, + ) + run_deferred_hooks_and_restart_pre_upgrades.add_steps( + [ + UnitUpgradeStep( + description=f"Execute run-deferred-hooks on unit: '{unit.name}'", + coro=model.run_action(unit.name, "run-deferred-hooks", raise_on_failure=True), + ) + for unit in app.units.values() + ] + ) + run_deferred_hooks_and_restart_pre_wait_step = PreUpgradeStep( + description=(f"Wait for up to 2400s for app '{app.name}'" " to reach the idle state"), + parallel=False, + coro=model.wait_for_idle(2400, apps=[app.name]), + ) + + run_deferred_hooks_and_restart_post_wait_step = PostUpgradeStep( + description=(f"Wait for up to 2400s for app '{app.name}'" " to reach the idle state"), + parallel=False, + coro=model.wait_for_idle(2400, apps=[app.name]), + ) + run_deferred_hooks_and_restart_post_upgrades = PostUpgradeStep( + description=( + f"Execute run-deferred-hooks for all '{app.name}' units " + "to restart the service after upgrade" + ), + parallel=False, + ) + run_deferred_hooks_and_restart_post_upgrades.add_steps( + [ + UnitUpgradeStep( + description=f"Execute run-deferred-hooks on unit: '{unit.name}'", + coro=model.run_action(unit.name, "run-deferred-hooks", raise_on_failure=True), + ) + for unit in app.units.values() + ] + ) + upgrade_steps = [ upgrade_packages, PreUpgradeStep( @@ -260,6 +352,8 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria(model): parallel=False, coro=model.wait_for_idle(300, apps=[app.name]), ), + run_deferred_hooks_and_restart_pre_upgrades, + run_deferred_hooks_and_restart_pre_wait_step, UpgradeStep( description=f"Change charm config of '{app.name}' " f"'{app.origin_setting}' to 'cloud:focal-victoria'", @@ -268,6 +362,8 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria(model): app.name, {f"{app.origin_setting}": "cloud:focal-victoria"} ), ), + run_deferred_hooks_and_restart_post_wait_step, + run_deferred_hooks_and_restart_post_upgrades, PostUpgradeStep( description=(f"Wait for up to 2400s for model '{model.name}' to reach the idle state"), parallel=False, @@ -298,7 +394,7 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_ch_migration(model): channel="stable", config={ "source": {"value": "distro"}, - "enable-auto-restarts": {"value": True}, + "enable-auto-restarts": {"value": False}, }, machines=machines, model=model, @@ -328,6 +424,50 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_ch_migration(model): for unit in app.units.values() ) + run_deferred_hooks_and_restart_pre_upgrades = PreUpgradeStep( + description=( + f"Execute run-deferred-hooks for all '{app.name}' units " + "to clear any leftover events" + ), + parallel=False, + ) + run_deferred_hooks_and_restart_pre_upgrades.add_steps( + [ + UnitUpgradeStep( + description=f"Execute run-deferred-hooks on unit: '{unit.name}'", + coro=model.run_action(unit.name, "run-deferred-hooks", raise_on_failure=True), + ) + for unit in app.units.values() + ] + ) + run_deferred_hooks_and_restart_pre_wait_step = PreUpgradeStep( + description=(f"Wait for up to 2400s for app '{app.name}'" " to reach the idle state"), + parallel=False, + coro=model.wait_for_idle(2400, apps=[app.name]), + ) + + run_deferred_hooks_and_restart_post_wait_step = PostUpgradeStep( + description=(f"Wait for up to 2400s for app '{app.name}'" " to reach the idle state"), + parallel=False, + coro=model.wait_for_idle(2400, apps=[app.name]), + ) + run_deferred_hooks_and_restart_post_upgrades = PostUpgradeStep( + description=( + f"Execute run-deferred-hooks for all '{app.name}' units " + "to restart the service after upgrade" + ), + parallel=False, + ) + run_deferred_hooks_and_restart_post_upgrades.add_steps( + [ + UnitUpgradeStep( + description=f"Execute run-deferred-hooks on unit: '{unit.name}'", + coro=model.run_action(unit.name, "run-deferred-hooks", raise_on_failure=True), + ) + for unit in app.units.values() + ] + ) + upgrade_steps = [ upgrade_packages, PreUpgradeStep( @@ -339,6 +479,8 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_ch_migration(model): parallel=False, coro=model.wait_for_idle(300, apps=[app.name]), ), + run_deferred_hooks_and_restart_pre_upgrades, + run_deferred_hooks_and_restart_pre_wait_step, UpgradeStep( description=f"Change charm config of '{app.name}' " f"'{app.origin_setting}' to 'cloud:focal-victoria'", @@ -347,6 +489,8 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_ch_migration(model): app.name, {f"{app.origin_setting}": "cloud:focal-victoria"} ), ), + run_deferred_hooks_and_restart_post_wait_step, + run_deferred_hooks_and_restart_post_upgrades, PostUpgradeStep( description=(f"Wait for up to 2400s for model '{model.name}' to reach the idle state"), parallel=False, @@ -492,6 +636,39 @@ def test_rabbitmq_server_upgrade_plan_ussuri_to_victoria_auto_restart_False(mode assert_steps(upgrade_plan, expected_plan) +def test_rabbitmq_server_upgrade_plan_auto_restart_True(model): + """Test rabbitmq server upgrade with enable-auto-restarts=True.""" + target = OpenStackRelease("victoria") + machines = {"0": generate_cou_machine("0", "az-0")} + app = RabbitMQServer( + name="rabbitmq-server", + can_upgrade_to="3.9/stable", + charm="rabbitmq-server", + channel="3.9/stable", + config={ + "source": {"value": "distro"}, + "enable-auto-restarts": {"value": True}, + }, + machines=machines, + model=model, + origin="ch", + series="focal", + subordinate_to=[], + units={ + "rabbitmq-server/0": Unit( + name="rabbitmq-server/0", + workload_version="3.9", + machine=machines["0"], + ), + }, + workload_version="3.9", + ) + + exp_msg = "`enable-auto-restarts` must be `False`" + with pytest.raises(ApplicationError, match=exp_msg): + app.generate_upgrade_plan(target, False) + + def test_auxiliary_upgrade_plan_unknown_track(model): """Test auxiliary upgrade plan with unknown track.""" channel = "2.0/stable"