diff --git a/cou/steps/ceph.py b/cou/steps/ceph.py index 616d4e1b..b09f866d 100644 --- a/cou/steps/ceph.py +++ b/cou/steps/ceph.py @@ -319,5 +319,5 @@ async def get_versions(model: Model, ceph_mon_unit_name: str) -> dict[str, dict[ :return: ceph version information :rtype: dict[str, dict[str, int]] """ - result = await model.run_on_unit(ceph_mon_unit_name, "ceph versions") + result = await model.run_on_unit(ceph_mon_unit_name, "ceph versions -f json") return json.loads(result["stdout"]) diff --git a/cou/steps/plan.py b/cou/steps/plan.py index eb884793..d51ac13f 100644 --- a/cou/steps/plan.py +++ b/cou/steps/plan.py @@ -173,8 +173,8 @@ async def post_upgrade_sanity_checks(analysis_result: Analysis) -> None: version_data = await ceph.get_versions(analysis_result.model, units[0].name) if len(version_data["overall"]) > 1: messages.append( - "Ceph mon sees mismatched versions in ceph daemons:\n" - "\n{json.dumps(version_data, indent=2)}\n\n" + f"Ceph mon ({units[0].name}) sees mismatched versions in ceph daemons:\n" + f"\n{json.dumps(version_data, indent=2)}\n\n" "This is unexpected: at the end of a clean upgrade, " "all ceph applications should be running the same version." ) diff --git a/tests/unit/steps/test_ceph.py b/tests/unit/steps/test_ceph.py index 5bad0aa2..243b2c84 100644 --- a/tests/unit/steps/test_ceph.py +++ b/tests/unit/steps/test_ceph.py @@ -309,3 +309,37 @@ async def test_get_current_osd_release_unsuccessful(model, osd_release_output, e command="ceph versions -f json", timeout=600, ) + + +@pytest.mark.asyncio +async def test_get_versions(model): + check_output = """ + { + "mon": { + "ceph version 15.2.17 (8a82819d84cf884bd39c17e3236e0632) octopus (stable)": 1 + }, + "mgr": { + "ceph version 15.2.17 (8a82819d84cf884bd39c17e3236e0632) octopus (stable)": 1 + }, + "osd": { + "ceph version 15.2.17 (8a82819d84cf884bd39c17e3236e0632) octopus (stable)": 3 + }, + "mds": {}, + "overall": { + "ceph version 15.2.17 (8a82819d84cf884bd39c17e3236e0632) octopus (stable)": 5 + } + } + """ + model.run_on_unit.return_value = {"return-code": 0, "stdout": check_output} + + versions = await ceph.get_versions(model, "my-ceph-mon/0") + + model.run_on_unit.assert_called_once_with( + "my-ceph-mon/0", + "ceph versions -f json", + ) + + assert ( + versions["mon"]["ceph version 15.2.17 (8a82819d84cf884bd39c17e3236e0632) octopus (stable)"] + == 1 + ) diff --git a/tests/unit/steps/test_plan.py b/tests/unit/steps/test_plan.py index f53d1450..9904941c 100644 --- a/tests/unit/steps/test_plan.py +++ b/tests/unit/steps/test_plan.py @@ -1767,6 +1767,49 @@ async def test_verify_model_idle_failed(mock_verify_model_idle, mock_analysis, m mock_plan_status.add_message.assert_called_once() +@pytest.mark.asyncio +@patch("cou.steps.plan.PlanStatus", autospec=True) +@patch("cou.steps.analyze.Analysis") +@patch("cou.steps.plan.ceph.get_versions", autospec=True) +async def test_verify_ceph_running_versions_consistent( + mock_get_versions, mock_analysis, mock_plan_status +): + """Test _verify_ceph_running_versions_consistent (normal success case).""" + # the scenario: two ceph clouds, both with consistent running versions + mock_analysis.apps_control_plane = [ + MagicMock(charm="ceph-mon", units={"first/0": MagicMock(name="first/0")}), + MagicMock(charm="ceph-mon", units={"second/0": MagicMock(name="second/0")}), + ] + mock_get_versions.side_effect = [ + { + "mon": { + # hashes truncated for brevity in tests + "ceph version 16.2.15 (618f440) pacific (stable)": 1, + }, + "osd": { + "ceph version 16.2.15 (618f440) pacific (stable)": 18, + }, + "overall": { + "ceph version 16.2.15 (618f440) pacific (stable)": 20, + }, + }, + { + "mon": {"ceph version 15.2.17 (8a82819) octopus (stable)": 1}, + "osd": { + "ceph version 15.2.17 (8a82819) octopus (stable)": 2, + }, + "overall": { + "ceph version 15.2.17 (8a82819) octopus (stable)": 2, + }, + }, + ] + + await cou_plan._verify_ceph_running_versions_consistent(mock_analysis) + + # verify that for this scenario, no warnings were generated + mock_plan_status.add_message.assert_not_called() + + @pytest.mark.asyncio @patch("cou.steps.plan.PlanStatus", autospec=True) @patch("cou.steps.analyze.Analysis") @@ -1839,3 +1882,118 @@ async def test_verify_ceph_running_versions_no_units( await cou_plan._verify_ceph_running_versions_consistent(mock_analysis) mock_plan_status.add_message.assert_not_called() + + +@pytest.mark.asyncio +@patch("cou.steps.plan.print_and_debug") +@patch("cou.steps.analyze.Analysis") +@patch("cou.steps.plan.ceph.assert_osd_noout_state", AsyncMock()) +async def test_post_upgrade_sanity_checks_ceph_mon_versions_no_units( + mock_analysis, mock_print_and_debug +): + """Test post_upgrade_sanity_checks with exception.""" + mock_analysis.apps_control_plane = [ + MagicMock(charm="ceph-mon", units={}), + ] + + await cou_plan.post_upgrade_sanity_checks(mock_analysis) + + mock_print_and_debug.assert_not_called() + + +@pytest.mark.asyncio +@patch("cou.steps.plan.print_and_debug") +@patch("cou.steps.analyze.Analysis") +@patch("cou.steps.plan.ceph.assert_osd_noout_state", AsyncMock()) +@patch("cou.steps.plan.ceph.get_versions", autospec=True) +async def test_post_upgrade_sanity_checks_ceph_mon_versions_inconsistent( + mock_get_versions, mock_analysis, mock_print_and_debug +): + """Test post_upgrade_sanity_checks with exception.""" + # the scenario: two ceph clouds, both with inconsistent running versions + mock_analysis.apps_control_plane = [ + MagicMock(charm="ceph-mon", units={"first/0": MagicMock(name="first/0")}), + MagicMock(charm="ceph-mon", units={"second/0": MagicMock(name="second/0")}), + ] + mock_get_versions.side_effect = [ + { + "mon": { + # hashes truncated for brevity in tests + "ceph version 16.2.14 (238ba60) pacific (stable)": 2, + "ceph version 16.2.15 (618f440) pacific (stable)": 1, + }, + "osd": { + "ceph version 15.2.17 (8a82819) octopus (stable)": 2, + "ceph version 16.2.15 (618f440) pacific (stable)": 18, + }, + "overall": { + "ceph version 15.2.17 (8a82819) octopus (stable)": 2, + "ceph version 16.2.14 (238ba60) pacific (stable)": 5, + "ceph version 16.2.15 (618f440) pacific (stable)": 20, + }, + }, + { + "mon": {"ceph version 16.2.15 (618f440) pacific (stable)": 1}, + "osd": { + "ceph version 15.2.17 (8a82819) octopus (stable)": 2, + "ceph version 16.2.15 (618f440) pacific (stable)": 18, + }, + "overall": { + "ceph version 15.2.17 (8a82819) octopus (stable)": 2, + "ceph version 16.2.15 (618f440) pacific (stable)": 20, + }, + }, + ] + + await cou_plan.post_upgrade_sanity_checks(mock_analysis) + + assert mock_print_and_debug.call_count == 2 + assert "first" in mock_print_and_debug.call_args_list[0].args[0] + assert "mismatched versions" in mock_print_and_debug.call_args_list[0].args[0] + assert "pacific" in mock_print_and_debug.call_args_list[0].args[0] + assert "second" in mock_print_and_debug.call_args_list[1].args[0] + assert "mismatched versions" in mock_print_and_debug.call_args_list[1].args[0] + assert "pacific" in mock_print_and_debug.call_args_list[1].args[0] + + +@pytest.mark.asyncio +@patch("cou.steps.plan.print_and_debug") +@patch("cou.steps.analyze.Analysis") +@patch("cou.steps.plan.ceph.assert_osd_noout_state", AsyncMock()) +@patch("cou.steps.plan.ceph.get_versions", autospec=True) +async def test_post_upgrade_sanity_checks_ceph_mon_versions_consistent( + mock_get_versions, mock_analysis, mock_print_and_debug +): + """Test post_upgrade_sanity_checks with exception.""" + # the scenario: two ceph clouds, both with consistent running versions + mock_analysis.apps_control_plane = [ + MagicMock(charm="ceph-mon", units={"first/0": MagicMock(name="first/0")}), + MagicMock(charm="ceph-mon", units={"second/0": MagicMock(name="second/0")}), + ] + mock_get_versions.side_effect = [ + { + "mon": { + # hashes truncated for brevity in tests + "ceph version 16.2.15 (618f440) pacific (stable)": 1, + }, + "osd": { + "ceph version 16.2.15 (618f440) pacific (stable)": 18, + }, + "overall": { + "ceph version 16.2.15 (618f440) pacific (stable)": 20, + }, + }, + { + "mon": {"ceph version 15.2.17 (8a82819) octopus (stable)": 1}, + "osd": { + "ceph version 15.2.17 (8a82819) octopus (stable)": 2, + }, + "overall": { + "ceph version 15.2.17 (8a82819) octopus (stable)": 2, + }, + }, + ] + + await cou_plan.post_upgrade_sanity_checks(mock_analysis) + + mock_print_and_debug.assert_not_called()