Update e2e for upgrade cluster manager

Signed-off-by: Jian Qiu <[email protected]>

pkg/cmd/init/scenario/init/operator.yaml

@@ -41,7 +41,7 @@ spec:
       - args:
         - /registration-operator
         - hub       
-        image: {{ .Hub.Registry }}/registration-operator:{{ .BundleVersion.RegistrationImageVersion }} 
+        image: {{ .Hub.Registry }}/registration-operator:{{ .BundleVersion.OperatorImageVersion }}
         imagePullPolicy: IfNotPresent
         livenessProbe:
           httpGet:

pkg/cmd/upgrade/clustermanager/exec.go

@@ -2,11 +2,16 @@
 package clustermanager
 
 import (
+	"context"
 	"fmt"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	operatorclient "open-cluster-management.io/api/client/operator/clientset/versioned"
+	clusteradminit "open-cluster-management.io/clusteradm/pkg/cmd/init"
 	"open-cluster-management.io/clusteradm/pkg/helpers/reader"
 
 	"github.com/spf13/cobra"
-	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/klog/v2"
 	init_scenario "open-cluster-management.io/clusteradm/pkg/cmd/init/scenario"
 	"open-cluster-management.io/clusteradm/pkg/helpers"
@@ -16,54 +21,60 @@ import (
 
 func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
 	klog.V(1).InfoS("init options:", "dry-run", o.ClusteradmFlags.DryRun)
-	o.values = Values{
-		Hub: Hub{
-			Registry: o.registry,
-		},
-	}
-
-	versionBundle, err := version.GetVersionBundle(o.bundleVersion)
-
-	if err != nil {
-		klog.Errorf("unable to retrieve version ", err)
-		return err
-	}
-
-	o.values.BundleVersion = BundleVersion{
-		RegistrationImageVersion: versionBundle.Registration,
-		PlacementImageVersion:    versionBundle.Placement,
-		WorkImageVersion:         versionBundle.Work,
-		OperatorImageVersion:     versionBundle.Operator,
-	}
 
 	f := o.ClusteradmFlags.KubectlFactory
 	o.builder = f.NewBuilder()
 
-	return nil
-}
-
-func (o *Options) validate() (err error) {
-	err = o.ClusteradmFlags.ValidateHub()
+	restConfig, err := f.ToRESTConfig()
 	if err != nil {
 		return err
 	}
 
-	restConfig, err := o.ClusteradmFlags.KubectlFactory.ToRESTConfig()
+	operatorClient, err := operatorclient.NewForConfig(restConfig)
 	if err != nil {
 		return err
 	}
-
-	apiExtensionsClient, err := apiextensionsclient.NewForConfig(restConfig)
+	cm, err := operatorClient.OperatorV1().ClusterManagers().Get(context.TODO(), "cluster-manager", metav1.GetOptions{})
+	if errors.IsNotFound(err) {
+		return fmt.Errorf("clustermanager is not installed")
+	}
 	if err != nil {
 		return err
 	}
-	installed, err := helpers.IsClusterManagerInstalled(apiExtensionsClient)
+
+	versionBundle, err := version.GetVersionBundle(o.bundleVersion)
 	if err != nil {
+		klog.Errorf("unable to retrieve version ", err)
 		return err
 	}
 
-	if !installed {
-		return fmt.Errorf("clustermanager is not installed")
+	o.values = clusteradminit.Values{
+		Hub: clusteradminit.Hub{
+			Registry: o.registry,
+		},
+		// reconstruct values from the cluster manager CR.
+		RegistrationFeatures: cm.Spec.RegistrationConfiguration.FeatureGates,
+		WorkFeatures:         cm.Spec.WorkConfiguration.FeatureGates,
+		AddonFeatures:        cm.Spec.AddOnManagerConfiguration.FeatureGates,
+		BundleVersion: clusteradminit.BundleVersion{
+			RegistrationImageVersion: versionBundle.Registration,
+			PlacementImageVersion:    versionBundle.Placement,
+			WorkImageVersion:         versionBundle.Work,
+			OperatorImageVersion:     versionBundle.Operator,
+		},
+	}
+
+	if len(cm.Spec.RegistrationConfiguration.AutoApproveUsers) > 0 {
+		o.values.AutoApprove = true
+	}
+
+	return nil
+}
+
+func (o *Options) validate() (err error) {
+	err = o.ClusteradmFlags.ValidateHub()
+	if err != nil {
+		return err
 	}
 
 	//TODO check desired version is greater then current version

pkg/cmd/upgrade/clustermanager/options.go

@@ -4,6 +4,7 @@ package clustermanager
 import (
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/cli-runtime/pkg/resource"
+	clusteradminit "open-cluster-management.io/clusteradm/pkg/cmd/init"
 	genericclioptionsclusteradm "open-cluster-management.io/clusteradm/pkg/genericclioptions"
 )
 
@@ -12,7 +13,7 @@ type Options struct {
 	//ClusteradmFlags: The generic options from the clusteradm cli-runtime.
 	ClusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags
 
-	values Values
+	values clusteradminit.Values
 	//The file to output the resources will be sent to the file.
 	registry string
 	//version of predefined compatible image versions
@@ -25,36 +26,6 @@ type Options struct {
 	builder *resource.Builder
 }
 
-type BundleVersion struct {
-	// registration image version
-	RegistrationImageVersion string
-	// placement image version
-	PlacementImageVersion string
-	// work image version
-	WorkImageVersion string
-	// operator image version
-	OperatorImageVersion string
-}
-
-// Values: The values used in the template
-type Values struct {
-	//The values related to the hub
-	Registry string `json:"registry"`
-	//bundle version
-	BundleVersion BundleVersion
-	//Hub: Hub information
-	Hub Hub
-}
-
-type Hub struct {
-	//APIServer: The API Server external URL
-	APIServer string
-	//KubeConfig: The kubeconfig of the bootstrap secret to connect to the hub
-	KubeConfig string
-	//image registry
-	Registry string
-}
-
 func newOptions(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, streams genericclioptions.IOStreams) *Options {
 	return &Options{
 		ClusteradmFlags: clusteradmFlags,

pkg/cmd/upgrade/klusterlet/exec.go

@@ -10,6 +10,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog/v2"
 	operatorclient "open-cluster-management.io/api/client/operator/clientset/versioned"
+	"open-cluster-management.io/clusteradm/pkg/cmd/join"
 	join_scenario "open-cluster-management.io/clusteradm/pkg/cmd/join/scenario"
 	"open-cluster-management.io/clusteradm/pkg/helpers"
 	"open-cluster-management.io/clusteradm/pkg/helpers/reader"
@@ -47,31 +48,26 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
 		return err
 	}
 
-	klog.V(1).InfoS("init options:", "dry-run", o.ClusteradmFlags.DryRun)
-	o.values = Values{
-		ClusterName: k.Spec.ClusterName,
-		Hub: Hub{
-			Registry: o.registry,
-		},
-	}
-
 	versionBundle, err := version.GetVersionBundle(o.bundleVersion)
-
 	if err != nil {
 		klog.Errorf("unable to retrieve version ", err)
 		return err
 	}
 
-	o.values.BundleVersion = BundleVersion{
-		RegistrationImageVersion: versionBundle.Registration,
-		PlacementImageVersion:    versionBundle.Placement,
-		WorkImageVersion:         versionBundle.Work,
-		OperatorImageVersion:     versionBundle.Operator,
+	klog.V(1).InfoS("init options:", "dry-run", o.ClusteradmFlags.DryRun)
+	o.values = join.Values{
+		Registry:             o.registry,
+		ClusterName:          k.Spec.ClusterName,
+		RegistrationFeatures: k.Spec.RegistrationConfiguration.FeatureGates,
+		WorkFeatures:         k.Spec.WorkConfiguration.FeatureGates,
+		BundleVersion: join.BundleVersion{
+			RegistrationImageVersion: versionBundle.Registration,
+			PlacementImageVersion:    versionBundle.Placement,
+			WorkImageVersion:         versionBundle.Work,
+			OperatorImageVersion:     versionBundle.Operator,
+		},
 	}
 
-	f := o.ClusteradmFlags.KubectlFactory
-	o.builder = f.NewBuilder()
-
 	return nil
 }
 

pkg/cmd/upgrade/klusterlet/options.go

@@ -4,6 +4,7 @@ package klusterlet
 import (
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/cli-runtime/pkg/resource"
+	"open-cluster-management.io/clusteradm/pkg/cmd/join"
 	genericclioptionsclusteradm "open-cluster-management.io/clusteradm/pkg/genericclioptions"
 )
 
@@ -12,7 +13,7 @@ type Options struct {
 	//ClusteradmFlags: The generic options from the clusteradm cli-runtime.
 	ClusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags
 
-	values Values
+	values join.Values
 	//The file to output the resources will be sent to the file.
 	registry string
 	//version of predefined compatible image versions

test/e2e/clusteradm/e2e_suite_test.go

@@ -77,7 +77,7 @@ var _ = ginkgo.BeforeSuite(func() {
 	operatorClient, err = operatorclient.NewForConfig(hubConfig)
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
-	err = clusterv1.AddToScheme(scheme.Scheme)
+	err = clusterv1.Install(scheme.Scheme)
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 	restConfig = hubConfig

test/e2e/clusteradm/upgrade_test.go

@@ -2,8 +2,14 @@
 package clusteradme2e
 
 import (
+	"context"
+	"fmt"
+	"time"
+
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"open-cluster-management.io/clusteradm/pkg/helpers/version"
 )
 
 var _ = ginkgo.Describe("test clusteradm upgrade clustermanager and Klusterlets", ginkgo.Ordered, func() {
@@ -12,28 +18,67 @@ var _ = ginkgo.Describe("test clusteradm upgrade clustermanager and Klusterlets"
 		ginkgo.By("reset e2e environment...")
 		err := e2e.ClearEnv()
 		gomega.Expect(err).NotTo(gomega.HaveOccurred())
-		err = e2e.ResetEnv()
-		gomega.Expect(err).NotTo(gomega.HaveOccurred())
 	})
 
 	var err error
 
 	ginkgo.It("run cluster manager upgrade version latest ", func() {
-		ginkgo.Skip("Upgrade is skipped due to flaky when destroying control plane. Need to revisit it after fix cleanup issue")
-		err = e2e.Clusteradm().Upgrade(
-			"clustermanager",
-			"--bundle-version", "latest",
+		ginkgo.By("init hub with service account")
+		err = e2e.Clusteradm().Init(
 			"--context", e2e.Cluster().Hub().Context(),
 		)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm init error")
 
-		gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm upgrade error")
+		ginkgo.By("Check the version of operator and agent")
+		gomega.Eventually(func() error {
+			operator, err := kubeClient.AppsV1().Deployments("open-cluster-management").Get(context.TODO(), "cluster-manager", metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			if operator.Spec.Template.Spec.Containers[0].Image != "quay.io/open-cluster-management/registration-operator:v"+version.GetDefaultBundleVersion() {
+				return fmt.Errorf("version of the operator is not correct, get %s", operator.Spec.Template.Spec.Containers[0].Image)
+			}
+			as, _ := kubeClient.AppsV1().Deployments("open-cluster-management-hub").List(context.TODO(), metav1.ListOptions{})
+			for _, a := range as.Items {
+				ginkgo.By(fmt.Sprintf("deployment is %s\n", a.Name))
+			}
+			registration, err := kubeClient.AppsV1().Deployments("open-cluster-management-hub").Get(
+				context.TODO(), "cluster-manager-registration-controller", metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			if registration.Spec.Template.Spec.Containers[0].Image != "quay.io/open-cluster-management/registration:v"+version.GetDefaultBundleVersion() {
+				return fmt.Errorf("version of the registration agent is not correct, get %s", operator.Spec.Template.Spec.Containers[0].Image)
+			}
+			return nil
+		}, 5*time.Second, 30*time.Second).Should(gomega.Succeed())
 
 		err = e2e.Clusteradm().Upgrade(
-			"klusterlet",
-			"--bundle-version", "latest",
-			"--context", e2e.Cluster().ManagedCluster1().Context(),
+			"clustermanager",
+			"--bundle-version=latest",
+			"--context", e2e.Cluster().Hub().Context(),
 		)
 
-		gomega.Expect(err).NotTo(gomega.HaveOccurred(), "klusterlet upgrade error")
+		gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm upgrade error")
+
+		ginkgo.By("Upgrade to the latest version")
+		gomega.Eventually(func() error {
+			operator, err := kubeClient.AppsV1().Deployments("open-cluster-management").Get(context.TODO(), "cluster-manager", metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			if operator.Spec.Template.Spec.Containers[0].Image != "quay.io/open-cluster-management/registration-operator:latest" {
+				return fmt.Errorf("version of the operator is not correct, get %s", operator.Spec.Template.Spec.Containers[0].Image)
+			}
+			registration, err := kubeClient.AppsV1().Deployments("open-cluster-management-hub").Get(
+				context.TODO(), "cluster-manager-registration-controller", metav1.GetOptions{})
+			if err != nil {
+				return err
+			}
+			if registration.Spec.Template.Spec.Containers[0].Image != "quay.io/open-cluster-management/registration:latest" {
+				return fmt.Errorf("version of the operator is not correct, get %s", operator.Spec.Template.Spec.Containers[0].Image)
+			}
+			return nil
+		}, 5*time.Second, 30*time.Second).Should(gomega.Succeed())
 	})
 })