diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py index e35d7ea309..bd9c3d6788 100644 --- a/src/_nebari/upgrade.py +++ b/src/_nebari/upgrade.py @@ -791,6 +791,55 @@ class Upgrade_2024_6_1(UpgradeStep): def _version_specific_upgrade( self, config, start_version, config_filename: Path, *args, **kwargs ): + # Prompt users to manually update kube-prometheus-stack CRDs if monitoring is enabled + if config.get("monitoring", {}).get("enabled", True): + rich.print( + "\n ⚠️ Warning ⚠️" + "\n-> [red bold]Nebari version 2024.6.1 comes with a new version of Grafana. Any custom dashboards that you created will be deleted after upgrading Nebari. Make sure to [link=https://grafana.com/docs/grafana/latest/dashboards/share-dashboards-panels/#export-a-dashboard-as-json]export them as JSON[/link] so you can [link=https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/#import-a-dashboard]import them[/link] again afterwards.[/red bold]" + "\n-> [red bold]Before upgrading, you need to manually delete the prometheus-node-exporter daemonset and update the kube-prometheus-stack CRDs. To do that, please run the following commands.[/red bold]" + ) + + # We're upgrading from version 30.1.0 to 58.4.0. This is a major upgrade and requires manual intervention. + # See https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/README.md#upgrading-chart + # for more information on why the following commands are necessary. + commands = textwrap.dedent( + f""" + [cyan bold] + kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml + kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml + kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml + kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml + kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml + kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml + kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml + kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml + kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml + kubectl apply --server-side --force-conflicts -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml + kubectl delete daemonset -l app=prometheus-node-exporter --namespace {config['namespace']} + [/cyan bold] + """ + ) + + # By default, rich wraps lines by splitting them into multiple lines. This is + # far from ideal, as users copy-pasting the commands will get errors when running them. + # To avoid this, we use a rich console with a larger width to print the entire commands + # and let the terminal wrap them if needed. + Prompt.ask("Hit enter to show the commands") + console = rich.console.Console(width=220) + console.print(commands) + + Prompt.ask("Hit enter to continue") + continue_ = Prompt.ask( + "Have you backed up your custom dashboards (if necessary), deleted the prometheus-node-exporter daemonset and updated the kube-prometheus-stack CRDs? \[y/N] ", + default="N", + ) + if not continue_ == "y": + rich.print( + f"[red bold]You must back up your custom dashboards (if necessary), delete the prometheus-node-exporter daemonset and update the kube-prometheus-stack CRDs before upgrading to [green]{self.version}[/green] (or later).[/bold red]" + ) + exit() + + # Prompt users to upgrade to the new default node groups for GCP if (provider := config.get("provider", "")) == ProviderEnum.gcp.value: provider_full_name = provider_enum_name_map[provider] if not config.get(provider_full_name, {}).get("node_groups", {}):