Delete deployment label selector is outdated (#2369) (#2372)

Prior, if the coredns deployment was using the old label, "kube-dns",
it would not be updated during the coredns manifests apply. This was
because label selectors are immutable. Now, if the label is outdated
on the deployment, the deployment is deleted prior to the coredns
manifests apply.

(cherry picked from commit 595cf00)

Co-authored-by: Ricardo Weir <[email protected]>

pkg/setup/controllers.go

@@ -175,7 +175,23 @@ func StartControllers(controllerContext *synccontext.ControllerContext, syncers
 
 func ApplyCoreDNS(controllerContext *synccontext.ControllerContext) {
 	_ = wait.ExponentialBackoffWithContext(controllerContext.Context, wait.Backoff{Duration: time.Second, Factor: 1.5, Cap: time.Minute, Steps: math.MaxInt32}, func(ctx context.Context) (bool, error) {
-		err := coredns.ApplyManifest(ctx, controllerContext.Config.ControlPlane.Advanced.DefaultImageRegistry, controllerContext.VirtualManager.GetConfig(), controllerContext.VirtualClusterVersion)
+		dnsDeployment := &appsv1.Deployment{}
+		err := controllerContext.VirtualManager.GetClient().Get(controllerContext.Context, types.NamespacedName{Namespace: "kube-system", Name: "coredns"}, dnsDeployment)
+		if err != nil && !kerrors.IsNotFound(err) {
+			return false, err
+		}
+		if err == nil {
+			// dns pod labels were changed to avoid conflict with apps running in the host cluster that select for the "kube-dns" label, e.g. cilium.
+			// If the deployment already exists with a label selector that is not "vcluster-kube-dns" then it needs to be deleted because the selector field is immutable.
+			// Otherwise, dns will break because the dns service will target the updated label but not match any deployments.
+			if dnsDeployment.Spec.Selector.MatchLabels["k8s-app"] != "vcluster-kube-dns" {
+				err = controllerContext.VirtualManager.GetClient().Delete(controllerContext.Context, dnsDeployment)
+				if err != nil && !kerrors.IsNotFound(err) {
+					return false, err
+				}
+			}
+		}
+		err = coredns.ApplyManifest(ctx, controllerContext.Config.ControlPlane.Advanced.DefaultImageRegistry, controllerContext.VirtualManager.GetConfig(), controllerContext.VirtualClusterVersion)
 		if err != nil {
 			if errors.Is(err, coredns.ErrNoCoreDNSManifests) {
 				klog.Infof("No CoreDNS manifests found, skipping CoreDNS configuration")