Add relevant commands to upgrade kube-prometheus-stack if the monitor…

…ing module is enabled.

src/_nebari/upgrade.py

@@ -791,6 +791,55 @@ class Upgrade_2024_6_1(UpgradeStep):
     def _version_specific_upgrade(
         self, config, start_version, config_filename: Path, *args, **kwargs
     ):
+        # Prompt users to manually update kube-prometheus-stack CRDs if monitoring is enabled
+        if config.get("monitoring", {}).get("enabled", True):
+            rich.print(
+                "\n ⚠️  Warning ⚠️"
+                "\n-> [red bold]Nebari version 2024.6.1 comes with a new version of Grafana. Any custom dashboards that you created will be deleted after upgrading Nebari. Make sure to [link=https://grafana.com/docs/grafana/latest/dashboards/share-dashboards-panels/#export-a-dashboard-as-json]export them as JSON[/link] so you can [link=https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/#import-a-dashboard]import them[/link] again afterwards.[/red bold]"
+                "\n-> [red bold]Before upgrading, you need to manually delete the prometheus-node-exporter daemonset and update the kube-prometheus-stack CRDs. To do that, please run the following commands.[/red bold]"
+            )
+
+            # We're upgrading from version 30.1.0 to 58.4.0. This is a major upgrade and requires manual intervention.
+            # See https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/README.md#upgrading-chart
+            # for more information on why the following commands are necessary.
+            commands = textwrap.dedent(
+                f"""
+                [cyan bold]
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+                kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml
+                kubectl delete daemonset -l app=prometheus-node-exporter --namespace {config['namespace']}
+                [/cyan bold]
+            """
+            )
+
+            # By default, rich wraps lines by splitting them into multiple lines. This is
+            # far from ideal, as users copy-pasting the commands will get errors when running them.
+            # To avoid this, we use a rich console with a larger width to print the entire commands
+            # and let the terminal wrap them if needed.
+            Prompt.ask("Hit enter to show the commands")
+            console = rich.console.Console(width=220)
+            console.print(commands)
+
+            Prompt.ask("Hit enter to continue")
+            continue_ = Prompt.ask(
+                "Have you backed up your custom dashboards (if necessary), deleted the prometheus-node-exporter daemonset and updated the kube-prometheus-stack CRDs? \[y/N] ",
+                default="N",
+            )
+            if not continue_ == "y":
+                rich.print(
+                    f"[red bold]You must back up your custom dashboards (if necessary), delete the prometheus-node-exporter daemonset and update the kube-prometheus-stack CRDs before upgrading to [green]{self.version}[/green] (or later).[/bold red]"
+                )
+                exit()
+
+        # Prompt users to upgrade to the new default node groups for GCP
         if (provider := config.get("provider", "")) == ProviderEnum.gcp.value:
             provider_full_name = provider_enum_name_map[provider]
             if not config.get(provider_full_name, {}).get("node_groups", {}):