Require enable-auto-restarts=False for rabbitmq-server (#593)

cou/apps/auxiliary.py

@@ -324,12 +324,21 @@ def post_upgrade_steps(
         return steps
 
     def _check_auto_restarts(self) -> None:
-        """No-op, skip check auto restarts option.
+        """Check if enable-auto-restarts is enabled.
 
-        This method override the parent class's `_check_auto_restarts()` method
-        because the parent class's will raise an `ApplicationError` if
-        `enable-auto-restarts` is `True`.
+        Due to the charm [bug](https://bugs.launchpad.net/charm-rabbitmq-server/+bug/2046381),
+        if the `enable-auto-restarts` option is enabled, this check will raise an exception.
+
+        :raises ApplicationError: When enable-auto-restarts is enabled.
         """
+        if self.config["enable-auto-restarts"].get("value") is True:
+            raise ApplicationError(
+                "`enable-auto-restarts` must be `False` due to "
+                "https://bugs.launchpad.net/charm-rabbitmq-server/+bug/2046381 "
+                f"Please run `juju config {self.name} enable-auto-restarts=False` "
+                "before performing upgrades and rollback to original value after "
+                "upgrade is completed"
+            )
 
 
 @AppFactory.register_application(["ceph-mon"])

docs/reference/known-issues.rst

@@ -23,6 +23,19 @@ See `bug 2060751`_ for details.
 If affected, refresh the manila-ganesha charm to its most recent version (within the same release
 channel) and re-run COU.
 
+Rabbitmq Server
+---------------
+
+The rabbitmq-server charm must have `enable-auto-restarts=False` for **COU** to
+work properly due to the known charm bug.
+
+See `bug 2046381`_ for details.
+
+We suggest that users should temporarily set `enable-auto-restarts=False` when
+performing `cou upgrade`, and rollback to original setting after the upgrade is
+completed.
+
 .. LINKS:
 .. _Issues, charm procedures, and OpenStack upgrade notes: https://docs.openstack.org/charm-guide/latest/project/issues-and-procedures.html
 .. _bug 2060751: https://bugs.launchpad.net/charm-manila-ganesha/+bug/2060751
+.. _bug 2046381: https://bugs.launchpad.net/charm-rabbitmq-server/+bug/2046381

tests/mocked_plans/sample_plans/018346c5-f95c-46df-a34e-9a78bdec0018.yaml

@@ -132,7 +132,17 @@ plan: |
                     Ψ Upgrade software packages on unit 'rabbitmq-server/2'
                 WARNING: Changing 'rabbitmq-server' channel from latest/stable to 3.9/stable. This may be a charm downgrade, which is generally not supported.
                 Wait for up to 300s for app 'rabbitmq-server' to reach the idle state
+                Execute run-deferred-hooks for all 'rabbitmq-server' units to clear any leftover events
+                    Execute run-deferred-hooks on unit: 'rabbitmq-server/0'
+                    Execute run-deferred-hooks on unit: 'rabbitmq-server/1'
+                    Execute run-deferred-hooks on unit: 'rabbitmq-server/2'
+                Wait for up to 2400s for app 'rabbitmq-server' to reach the idle state
                 Change charm config of 'rabbitmq-server' 'source' to 'cloud:focal-victoria'
+                Wait for up to 2400s for app 'rabbitmq-server' to reach the idle state
+                Execute run-deferred-hooks for all 'rabbitmq-server' units to restart the service after upgrade
+                    Execute run-deferred-hooks on unit: 'rabbitmq-server/0'
+                    Execute run-deferred-hooks on unit: 'rabbitmq-server/1'
+                    Execute run-deferred-hooks on unit: 'rabbitmq-server/2'
                 Wait for up to 2400s for model '018346c5-f95c-46df-a34e-9a78bdec0018' to reach the idle state
                 Verify that the workload of 'rabbitmq-server' has been upgraded on units: rabbitmq-server/0, rabbitmq-server/1, rabbitmq-server/2
             Upgrade plan for 'ceph-mon' to 'victoria'
@@ -8070,7 +8080,7 @@ applications:
           '
         source: default
         type: boolean
-        value: true
+        value: false
       source:
         description: "Optional configuration to support use of additional sources such\
           \ as:\n.\n  - ppa:myteam/ppa\n  - cloud:xenial-proposed/ocata\n  - http://my.archive.com/ubuntu\

tests/mocked_plans/sample_plans/9eb9af6a-b919-4cf9-8f2f-9df16a1556be.yaml

@@ -179,9 +179,19 @@ plan: |
                     Ψ Upgrade software packages on unit 'rabbitmq-server/2'
                 Refresh 'rabbitmq-server' to the latest revision of '3.8/stable'
                 Wait for up to 300s for app 'rabbitmq-server' to reach the idle state
+                Execute run-deferred-hooks for all 'rabbitmq-server' units to clear any leftover events
+                    Execute run-deferred-hooks on unit: 'rabbitmq-server/0'
+                    Execute run-deferred-hooks on unit: 'rabbitmq-server/1'
+                    Execute run-deferred-hooks on unit: 'rabbitmq-server/2'
+                Wait for up to 2400s for app 'rabbitmq-server' to reach the idle state
                 Upgrade 'rabbitmq-server' from '3.8/stable' to the new channel: '3.9/stable'
                 Wait for up to 300s for app 'rabbitmq-server' to reach the idle state
                 Change charm config of 'rabbitmq-server' 'source' to 'cloud:focal-victoria'
+                Wait for up to 2400s for app 'rabbitmq-server' to reach the idle state
+                Execute run-deferred-hooks for all 'rabbitmq-server' units to restart the service after upgrade
+                    Execute run-deferred-hooks on unit: 'rabbitmq-server/0'
+                    Execute run-deferred-hooks on unit: 'rabbitmq-server/1'
+                    Execute run-deferred-hooks on unit: 'rabbitmq-server/2'
                 Wait for up to 2400s for model '9eb9af6a-b919-4cf9-8f2f-9df16a1556be' to reach the idle state
                 Verify that the workload of 'rabbitmq-server' has been upgraded on units: rabbitmq-server/0, rabbitmq-server/1, rabbitmq-server/2
             Upgrade plan for 'ceph-mon' to 'victoria'
@@ -539,7 +549,7 @@ applications:
           '
         source: user
         type: boolean
-        value: true
+        value: false
       source:
         description: "Optional configuration to support use of additional sources such\
           \ as:\n.\n  - ppa:myteam/ppa\n  - cloud:xenial-proposed/ocata\n  - http://my.archive.com/ubuntu\

tests/unit/apps/test_auxiliary.py

@@ -127,7 +127,7 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_change_channel(model):
         channel="3.8/stable",
         config={
             "source": {"value": "distro"},
-            "enable-auto-restarts": {"value": True},
+            "enable-auto-restarts": {"value": False},
         },
         machines=machines,
         model=model,
@@ -158,6 +158,50 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_change_channel(model):
         for unit in app.units.keys()
     )
 
+    run_deferred_hooks_and_restart_pre_upgrades = PreUpgradeStep(
+        description=(
+            f"Execute run-deferred-hooks for all '{app.name}' units "
+            "to clear any leftover events"
+        ),
+        parallel=False,
+    )
+    run_deferred_hooks_and_restart_pre_upgrades.add_steps(
+        [
+            UnitUpgradeStep(
+                description=f"Execute run-deferred-hooks on unit: '{unit.name}'",
+                coro=model.run_action(unit.name, "run-deferred-hooks", raise_on_failure=True),
+            )
+            for unit in app.units.values()
+        ]
+    )
+    run_deferred_hooks_and_restart_pre_wait_step = PreUpgradeStep(
+        description=(f"Wait for up to 2400s for app '{app.name}'" " to reach the idle state"),
+        parallel=False,
+        coro=model.wait_for_idle(2400, apps=[app.name]),
+    )
+
+    run_deferred_hooks_and_restart_post_wait_step = PostUpgradeStep(
+        description=(f"Wait for up to 2400s for app '{app.name}'" " to reach the idle state"),
+        parallel=False,
+        coro=model.wait_for_idle(2400, apps=[app.name]),
+    )
+    run_deferred_hooks_and_restart_post_upgrades = PostUpgradeStep(
+        description=(
+            f"Execute run-deferred-hooks for all '{app.name}' units "
+            "to restart the service after upgrade"
+        ),
+        parallel=False,
+    )
+    run_deferred_hooks_and_restart_post_upgrades.add_steps(
+        [
+            UnitUpgradeStep(
+                description=f"Execute run-deferred-hooks on unit: '{unit.name}'",
+                coro=model.run_action(unit.name, "run-deferred-hooks", raise_on_failure=True),
+            )
+            for unit in app.units.values()
+        ]
+    )
+
     upgrade_steps = [
         expected_upgrade_package_step,
         PreUpgradeStep(
@@ -170,6 +214,8 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_change_channel(model):
             parallel=False,
             coro=model.wait_for_idle(300, apps=[app.name]),
         ),
+        run_deferred_hooks_and_restart_pre_upgrades,
+        run_deferred_hooks_and_restart_pre_wait_step,
         UpgradeStep(
             description=f"Upgrade '{app.name}' from '3.8/stable' to the new channel: '3.9/stable'",
             parallel=False,
@@ -189,6 +235,8 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_change_channel(model):
                 {f"{app.origin_setting}": "cloud:focal-victoria"},
             ),
         ),
+        run_deferred_hooks_and_restart_post_wait_step,
+        run_deferred_hooks_and_restart_post_upgrades,
         PostUpgradeStep(
             description=f"Wait for up to 2400s for model '{model.name}' to reach the idle state",
             parallel=False,
@@ -219,7 +267,7 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria(model):
         channel="3.9/stable",
         config={
             "source": {"value": "distro"},
-            "enable-auto-restarts": {"value": True},
+            "enable-auto-restarts": {"value": False},
         },
         machines=machines,
         model=model,
@@ -249,6 +297,50 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria(model):
         for unit in app.units.values()
     )
 
+    run_deferred_hooks_and_restart_pre_upgrades = PreUpgradeStep(
+        description=(
+            f"Execute run-deferred-hooks for all '{app.name}' units "
+            "to clear any leftover events"
+        ),
+        parallel=False,
+    )
+    run_deferred_hooks_and_restart_pre_upgrades.add_steps(
+        [
+            UnitUpgradeStep(
+                description=f"Execute run-deferred-hooks on unit: '{unit.name}'",
+                coro=model.run_action(unit.name, "run-deferred-hooks", raise_on_failure=True),
+            )
+            for unit in app.units.values()
+        ]
+    )
+    run_deferred_hooks_and_restart_pre_wait_step = PreUpgradeStep(
+        description=(f"Wait for up to 2400s for app '{app.name}'" " to reach the idle state"),
+        parallel=False,
+        coro=model.wait_for_idle(2400, apps=[app.name]),
+    )
+
+    run_deferred_hooks_and_restart_post_wait_step = PostUpgradeStep(
+        description=(f"Wait for up to 2400s for app '{app.name}'" " to reach the idle state"),
+        parallel=False,
+        coro=model.wait_for_idle(2400, apps=[app.name]),
+    )
+    run_deferred_hooks_and_restart_post_upgrades = PostUpgradeStep(
+        description=(
+            f"Execute run-deferred-hooks for all '{app.name}' units "
+            "to restart the service after upgrade"
+        ),
+        parallel=False,
+    )
+    run_deferred_hooks_and_restart_post_upgrades.add_steps(
+        [
+            UnitUpgradeStep(
+                description=f"Execute run-deferred-hooks on unit: '{unit.name}'",
+                coro=model.run_action(unit.name, "run-deferred-hooks", raise_on_failure=True),
+            )
+            for unit in app.units.values()
+        ]
+    )
+
     upgrade_steps = [
         upgrade_packages,
         PreUpgradeStep(
@@ -260,6 +352,8 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria(model):
             parallel=False,
             coro=model.wait_for_idle(300, apps=[app.name]),
         ),
+        run_deferred_hooks_and_restart_pre_upgrades,
+        run_deferred_hooks_and_restart_pre_wait_step,
         UpgradeStep(
             description=f"Change charm config of '{app.name}' "
             f"'{app.origin_setting}' to 'cloud:focal-victoria'",
@@ -268,6 +362,8 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria(model):
                 app.name, {f"{app.origin_setting}": "cloud:focal-victoria"}
             ),
         ),
+        run_deferred_hooks_and_restart_post_wait_step,
+        run_deferred_hooks_and_restart_post_upgrades,
         PostUpgradeStep(
             description=(f"Wait for up to 2400s for model '{model.name}' to reach the idle state"),
             parallel=False,
@@ -298,7 +394,7 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_ch_migration(model):
         channel="stable",
         config={
             "source": {"value": "distro"},
-            "enable-auto-restarts": {"value": True},
+            "enable-auto-restarts": {"value": False},
         },
         machines=machines,
         model=model,
@@ -328,6 +424,50 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_ch_migration(model):
         for unit in app.units.values()
     )
 
+    run_deferred_hooks_and_restart_pre_upgrades = PreUpgradeStep(
+        description=(
+            f"Execute run-deferred-hooks for all '{app.name}' units "
+            "to clear any leftover events"
+        ),
+        parallel=False,
+    )
+    run_deferred_hooks_and_restart_pre_upgrades.add_steps(
+        [
+            UnitUpgradeStep(
+                description=f"Execute run-deferred-hooks on unit: '{unit.name}'",
+                coro=model.run_action(unit.name, "run-deferred-hooks", raise_on_failure=True),
+            )
+            for unit in app.units.values()
+        ]
+    )
+    run_deferred_hooks_and_restart_pre_wait_step = PreUpgradeStep(
+        description=(f"Wait for up to 2400s for app '{app.name}'" " to reach the idle state"),
+        parallel=False,
+        coro=model.wait_for_idle(2400, apps=[app.name]),
+    )
+
+    run_deferred_hooks_and_restart_post_wait_step = PostUpgradeStep(
+        description=(f"Wait for up to 2400s for app '{app.name}'" " to reach the idle state"),
+        parallel=False,
+        coro=model.wait_for_idle(2400, apps=[app.name]),
+    )
+    run_deferred_hooks_and_restart_post_upgrades = PostUpgradeStep(
+        description=(
+            f"Execute run-deferred-hooks for all '{app.name}' units "
+            "to restart the service after upgrade"
+        ),
+        parallel=False,
+    )
+    run_deferred_hooks_and_restart_post_upgrades.add_steps(
+        [
+            UnitUpgradeStep(
+                description=f"Execute run-deferred-hooks on unit: '{unit.name}'",
+                coro=model.run_action(unit.name, "run-deferred-hooks", raise_on_failure=True),
+            )
+            for unit in app.units.values()
+        ]
+    )
+
     upgrade_steps = [
         upgrade_packages,
         PreUpgradeStep(
@@ -339,6 +479,8 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_ch_migration(model):
             parallel=False,
             coro=model.wait_for_idle(300, apps=[app.name]),
         ),
+        run_deferred_hooks_and_restart_pre_upgrades,
+        run_deferred_hooks_and_restart_pre_wait_step,
         UpgradeStep(
             description=f"Change charm config of '{app.name}' "
             f"'{app.origin_setting}' to 'cloud:focal-victoria'",
@@ -347,6 +489,8 @@ def test_auxiliary_upgrade_plan_ussuri_to_victoria_ch_migration(model):
                 app.name, {f"{app.origin_setting}": "cloud:focal-victoria"}
             ),
         ),
+        run_deferred_hooks_and_restart_post_wait_step,
+        run_deferred_hooks_and_restart_post_upgrades,
         PostUpgradeStep(
             description=(f"Wait for up to 2400s for model '{model.name}' to reach the idle state"),
             parallel=False,
@@ -492,6 +636,39 @@ def test_rabbitmq_server_upgrade_plan_ussuri_to_victoria_auto_restart_False(mode
     assert_steps(upgrade_plan, expected_plan)
 
 
+def test_rabbitmq_server_upgrade_plan_auto_restart_True(model):
+    """Test rabbitmq server upgrade with enable-auto-restarts=True."""
+    target = OpenStackRelease("victoria")
+    machines = {"0": generate_cou_machine("0", "az-0")}
+    app = RabbitMQServer(
+        name="rabbitmq-server",
+        can_upgrade_to="3.9/stable",
+        charm="rabbitmq-server",
+        channel="3.9/stable",
+        config={
+            "source": {"value": "distro"},
+            "enable-auto-restarts": {"value": True},
+        },
+        machines=machines,
+        model=model,
+        origin="ch",
+        series="focal",
+        subordinate_to=[],
+        units={
+            "rabbitmq-server/0": Unit(
+                name="rabbitmq-server/0",
+                workload_version="3.9",
+                machine=machines["0"],
+            ),
+        },
+        workload_version="3.9",
+    )
+
+    exp_msg = "`enable-auto-restarts` must be `False`"
+    with pytest.raises(ApplicationError, match=exp_msg):
+        app.generate_upgrade_plan(target, False)
+
+
 def test_auxiliary_upgrade_plan_unknown_track(model):
     """Test auxiliary upgrade plan with unknown track."""
     channel = "2.0/stable"