🌱 Update e2e for upgrade cluster manager (#394)

* Update e2e for upgrade cluster manager Signed-off-by: Jian Qiu <[email protected]> * Fix nil pointer panic in upgrade Signed-off-by: Jian Qiu <[email protected]> --------- Signed-off-by: Jian Qiu <[email protected]>
open-cluster-management-io · Nov 20, 2023 · 4d7d077 · 4d7d077
1 parent 8dfb382
commit 4d7d077
Show file tree

Hide file tree

Showing 12 changed files with 236 additions and 153 deletions.
diff --git a/pkg/cmd/init/scenario/init/operator.yaml b/pkg/cmd/init/scenario/init/operator.yaml
@@ -41,7 +41,7 @@ spec:
       - args:
         - /registration-operator
         - hub       
-        image: {{ .Hub.Registry }}/registration-operator:{{ .BundleVersion.RegistrationImageVersion }} 
+        image: {{ .Hub.Registry }}/registration-operator:{{ .BundleVersion.OperatorImageVersion }}
         imagePullPolicy: IfNotPresent
         livenessProbe:
           httpGet:

diff --git a/pkg/cmd/unjoin/exec.go b/pkg/cmd/unjoin/exec.go
@@ -187,7 +187,7 @@ func (o *Options) purgeKlusterlet(kubeClient kubernetes.Interface, klusterletCli
 	}
 
 	b := retry.DefaultBackoff
-	b.Duration = 1 * time.Second
+	b.Duration = 5 * time.Second
 	err = WaitResourceToBeDelete(context.Background(), klusterletClient, o.values.KlusterletName, b)
 	if err != nil {
 		return err

diff --git a/pkg/cmd/upgrade/clustermanager/exec.go b/pkg/cmd/upgrade/clustermanager/exec.go
@@ -2,11 +2,16 @@
 package clustermanager
 
 import (
+	"context"
 	"fmt"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	operatorclient "open-cluster-management.io/api/client/operator/clientset/versioned"
+	clusteradminit "open-cluster-management.io/clusteradm/pkg/cmd/init"
 	"open-cluster-management.io/clusteradm/pkg/helpers/reader"
 
 	"github.com/spf13/cobra"
-	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/klog/v2"
 	init_scenario "open-cluster-management.io/clusteradm/pkg/cmd/init/scenario"
 	"open-cluster-management.io/clusteradm/pkg/helpers"
@@ -16,54 +21,66 @@ import (
 
 func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
 	klog.V(1).InfoS("init options:", "dry-run", o.ClusteradmFlags.DryRun)
-	o.values = Values{
-		Hub: Hub{
-			Registry: o.registry,
-		},
-	}
-
-	versionBundle, err := version.GetVersionBundle(o.bundleVersion)
-
-	if err != nil {
-		klog.Errorf("unable to retrieve version ", err)
-		return err
-	}
-
-	o.values.BundleVersion = BundleVersion{
-		RegistrationImageVersion: versionBundle.Registration,
-		PlacementImageVersion:    versionBundle.Placement,
-		WorkImageVersion:         versionBundle.Work,
-		OperatorImageVersion:     versionBundle.Operator,
-	}
 
 	f := o.ClusteradmFlags.KubectlFactory
 	o.builder = f.NewBuilder()
 
-	return nil
-}
-
-func (o *Options) validate() (err error) {
-	err = o.ClusteradmFlags.ValidateHub()
+	restConfig, err := f.ToRESTConfig()
 	if err != nil {
 		return err
 	}
 
-	restConfig, err := o.ClusteradmFlags.KubectlFactory.ToRESTConfig()
+	operatorClient, err := operatorclient.NewForConfig(restConfig)
 	if err != nil {
 		return err
 	}
-
-	apiExtensionsClient, err := apiextensionsclient.NewForConfig(restConfig)
+	cm, err := operatorClient.OperatorV1().ClusterManagers().Get(context.TODO(), "cluster-manager", metav1.GetOptions{})
+	if errors.IsNotFound(err) {
+		return fmt.Errorf("clustermanager is not installed")
+	}
 	if err != nil {
 		return err
 	}
-	installed, err := helpers.IsClusterManagerInstalled(apiExtensionsClient)
+
+	versionBundle, err := version.GetVersionBundle(o.bundleVersion)
 	if err != nil {
+		klog.Errorf("unable to retrieve version ", err)
 		return err
 	}
 
-	if !installed {
-		return fmt.Errorf("clustermanager is not installed")
+	o.values = clusteradminit.Values{
+		Hub: clusteradminit.Hub{
+			Registry: o.registry,
+		},
+		BundleVersion: clusteradminit.BundleVersion{
+			RegistrationImageVersion: versionBundle.Registration,
+			PlacementImageVersion:    versionBundle.Placement,
+			WorkImageVersion:         versionBundle.Work,
+			OperatorImageVersion:     versionBundle.Operator,
+		},
+	}
+
+	// reconstruct values from the cluster manager CR.
+	if cm.Spec.RegistrationConfiguration != nil {
+		o.values.RegistrationFeatures = cm.Spec.RegistrationConfiguration.FeatureGates
+		if len(cm.Spec.RegistrationConfiguration.AutoApproveUsers) > 0 {
+			o.values.AutoApprove = true
+		}
+	}
+	if cm.Spec.WorkConfiguration != nil {
+		o.values.WorkFeatures = cm.Spec.WorkConfiguration.FeatureGates
+	}
+	if cm.Spec.AddOnManagerConfiguration != nil {
+		o.values.AddonFeatures = cm.Spec.AddOnManagerConfiguration.FeatureGates
+	}
+
+	return nil
+}
+
+func (o *Options) validate() (err error) {
+	err = o.ClusteradmFlags.ValidateHub()
+	if err != nil {
+		return err
 	}
 
 	//TODO check desired version is greater then current version

diff --git a/pkg/cmd/upgrade/clustermanager/options.go b/pkg/cmd/upgrade/clustermanager/options.go
@@ -4,6 +4,7 @@ package clustermanager
 import (
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/cli-runtime/pkg/resource"
+	clusteradminit "open-cluster-management.io/clusteradm/pkg/cmd/init"
 	genericclioptionsclusteradm "open-cluster-management.io/clusteradm/pkg/genericclioptions"
 )
 
@@ -12,7 +13,7 @@ type Options struct {
 	//ClusteradmFlags: The generic options from the clusteradm cli-runtime.
 	ClusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags
 
-	values Values
+	values clusteradminit.Values
 	//The file to output the resources will be sent to the file.
 	registry string
 	//version of predefined compatible image versions
@@ -25,36 +26,6 @@ type Options struct {
 	builder *resource.Builder
 }
 
-type BundleVersion struct {
-	// registration image version
-	RegistrationImageVersion string
-	// placement image version
-	PlacementImageVersion string
-	// work image version
-	WorkImageVersion string
-	// operator image version
-	OperatorImageVersion string
-}
-
-// Values: The values used in the template
-type Values struct {
-	//The values related to the hub
-	Registry string `json:"registry"`
-	//bundle version
-	BundleVersion BundleVersion
-	//Hub: Hub information
-	Hub Hub
-}
-
-type Hub struct {
-	//APIServer: The API Server external URL
-	APIServer string
-	//KubeConfig: The kubeconfig of the bootstrap secret to connect to the hub
-	KubeConfig string
-	//image registry
-	Registry string
-}
-
 func newOptions(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, streams genericclioptions.IOStreams) *Options {
 	return &Options{
 		ClusteradmFlags: clusteradmFlags,

diff --git a/pkg/cmd/upgrade/klusterlet/exec.go b/pkg/cmd/upgrade/klusterlet/exec.go
@@ -10,6 +10,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog/v2"
 	operatorclient "open-cluster-management.io/api/client/operator/clientset/versioned"
+	"open-cluster-management.io/clusteradm/pkg/cmd/join"
 	join_scenario "open-cluster-management.io/clusteradm/pkg/cmd/join/scenario"
 	"open-cluster-management.io/clusteradm/pkg/helpers"
 	"open-cluster-management.io/clusteradm/pkg/helpers/reader"
@@ -19,11 +20,7 @@ import (
 
 //nolint:deadcode,varcheck
 const (
-	klusterletName                 = "klusterlet"
-	registrationOperatorNamespace  = "open-cluster-management"
-	klusterletCRD                  = "klusterlets.operator.open-cluster-management.io"
-	componentNameRegistrationAgent = "klusterlet-registration-agent"
-	componentNameWorkAgent         = "klusterlet-work-agent"
+	klusterletName = "klusterlet"
 )
 
 func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
@@ -32,7 +29,10 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
 		return err
 	}
 
-	cfg, err := o.ClusteradmFlags.KubectlFactory.ToRESTConfig()
+	f := o.ClusteradmFlags.KubectlFactory
+	o.builder = f.NewBuilder()
+
+	cfg, err := f.ToRESTConfig()
 	if err != nil {
 		return err
 	}
@@ -47,30 +47,36 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
 		return err
 	}
 
-	klog.V(1).InfoS("init options:", "dry-run", o.ClusteradmFlags.DryRun)
-	o.values = Values{
-		ClusterName: k.Spec.ClusterName,
-		Hub: Hub{
-			Registry: o.registry,
-		},
-	}
-
 	versionBundle, err := version.GetVersionBundle(o.bundleVersion)
-
 	if err != nil {
 		klog.Errorf("unable to retrieve version ", err)
 		return err
 	}
 
-	o.values.BundleVersion = BundleVersion{
-		RegistrationImageVersion: versionBundle.Registration,
-		PlacementImageVersion:    versionBundle.Placement,
-		WorkImageVersion:         versionBundle.Work,
-		OperatorImageVersion:     versionBundle.Operator,
+	klog.V(1).InfoS("init options:", "dry-run", o.ClusteradmFlags.DryRun)
+	o.values = join.Values{
+		Registry:    o.registry,
+		ClusterName: k.Spec.ClusterName,
+		Klusterlet: join.Klusterlet{
+			Name:                k.Name,
+			Mode:                string(k.Spec.DeployOption.Mode),
+			KlusterletNamespace: k.Spec.Namespace,
+		},
+		BundleVersion: join.BundleVersion{
+			RegistrationImageVersion: versionBundle.Registration,
+			PlacementImageVersion:    versionBundle.Placement,
+			WorkImageVersion:         versionBundle.Work,
+			OperatorImageVersion:     versionBundle.Operator,
+		},
 	}
 
-	f := o.ClusteradmFlags.KubectlFactory
-	o.builder = f.NewBuilder()
+	// reconstruct values from the klusterlet CR.
+	if k.Spec.RegistrationConfiguration != nil {
+		o.values.RegistrationFeatures = k.Spec.RegistrationConfiguration.FeatureGates
+	}
+	if k.Spec.WorkConfiguration != nil {
+		o.values.WorkFeatures = k.Spec.WorkConfiguration.FeatureGates
+	}
 
 	return nil
 }
@@ -108,8 +114,6 @@ func (o *Options) run() error {
 	}
 
 	files := []string{
-		"join/namespace_agent.yaml",
-		"join/namespace_addon.yaml",
 		"join/namespace.yaml",
 		"join/cluster_role.yaml",
 		"join/cluster_role_binding.yaml",
@@ -128,7 +132,7 @@ func (o *Options) run() error {
 	}
 
 	if !o.ClusteradmFlags.DryRun {
-		if err := wait.WaitUntilCRDReady(apiExtensionsClient, "clustermanagers.operator.open-cluster-management.io", o.wait); err != nil {
+		if err := wait.WaitUntilCRDReady(apiExtensionsClient, "klusterlets.operator.open-cluster-management.io", o.wait); err != nil {
 			return err
 		}
 	}

diff --git a/pkg/cmd/upgrade/klusterlet/options.go b/pkg/cmd/upgrade/klusterlet/options.go
@@ -4,6 +4,7 @@ package klusterlet
 import (
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/cli-runtime/pkg/resource"
+	"open-cluster-management.io/clusteradm/pkg/cmd/join"
 	genericclioptionsclusteradm "open-cluster-management.io/clusteradm/pkg/genericclioptions"
 )
 
@@ -12,7 +13,7 @@ type Options struct {
 	//ClusteradmFlags: The generic options from the clusteradm cli-runtime.
 	ClusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags
 
-	values Values
+	values join.Values
 	//The file to output the resources will be sent to the file.
 	registry string
 	//version of predefined compatible image versions

diff --git a/test/e2e/clusteradm/e2e_suite_test.go b/test/e2e/clusteradm/e2e_suite_test.go
@@ -77,7 +77,7 @@ var _ = ginkgo.BeforeSuite(func() {
 	operatorClient, err = operatorclient.NewForConfig(hubConfig)
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
-	err = clusterv1.AddToScheme(scheme.Scheme)
+	err = clusterv1.Install(scheme.Scheme)
 	gomega.Expect(err).NotTo(gomega.HaveOccurred())
 
 	restConfig = hubConfig