diff --git a/pkg/cmd/init/scenario/init/operator.yaml b/pkg/cmd/init/scenario/init/operator.yaml index 4ecd6f91a..675468380 100644 --- a/pkg/cmd/init/scenario/init/operator.yaml +++ b/pkg/cmd/init/scenario/init/operator.yaml @@ -41,7 +41,7 @@ spec: - args: - /registration-operator - hub - image: {{ .Hub.Registry }}/registration-operator:{{ .BundleVersion.RegistrationImageVersion }} + image: {{ .Hub.Registry }}/registration-operator:{{ .BundleVersion.OperatorImageVersion }} imagePullPolicy: IfNotPresent livenessProbe: httpGet: diff --git a/pkg/cmd/upgrade/clustermanager/exec.go b/pkg/cmd/upgrade/clustermanager/exec.go index 687191874..d3f8c1ca7 100644 --- a/pkg/cmd/upgrade/clustermanager/exec.go +++ b/pkg/cmd/upgrade/clustermanager/exec.go @@ -2,11 +2,16 @@ package clustermanager import ( + "context" "fmt" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + operatorclient "open-cluster-management.io/api/client/operator/clientset/versioned" + clusteradminit "open-cluster-management.io/clusteradm/pkg/cmd/init" "open-cluster-management.io/clusteradm/pkg/helpers/reader" "github.com/spf13/cobra" - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/klog/v2" init_scenario "open-cluster-management.io/clusteradm/pkg/cmd/init/scenario" "open-cluster-management.io/clusteradm/pkg/helpers" @@ -16,54 +21,60 @@ import ( func (o *Options) complete(cmd *cobra.Command, args []string) (err error) { klog.V(1).InfoS("init options:", "dry-run", o.ClusteradmFlags.DryRun) - o.values = Values{ - Hub: Hub{ - Registry: o.registry, - }, - } - - versionBundle, err := version.GetVersionBundle(o.bundleVersion) - - if err != nil { - klog.Errorf("unable to retrieve version ", err) - return err - } - - o.values.BundleVersion = BundleVersion{ - RegistrationImageVersion: versionBundle.Registration, - PlacementImageVersion: versionBundle.Placement, - WorkImageVersion: versionBundle.Work, - OperatorImageVersion: versionBundle.Operator, - } f := o.ClusteradmFlags.KubectlFactory o.builder = f.NewBuilder() - return nil -} - -func (o *Options) validate() (err error) { - err = o.ClusteradmFlags.ValidateHub() + restConfig, err := f.ToRESTConfig() if err != nil { return err } - restConfig, err := o.ClusteradmFlags.KubectlFactory.ToRESTConfig() + operatorClient, err := operatorclient.NewForConfig(restConfig) if err != nil { return err } - - apiExtensionsClient, err := apiextensionsclient.NewForConfig(restConfig) + cm, err := operatorClient.OperatorV1().ClusterManagers().Get(context.TODO(), "cluster-manager", metav1.GetOptions{}) + if errors.IsNotFound(err) { + return fmt.Errorf("clustermanager is not installed") + } if err != nil { return err } - installed, err := helpers.IsClusterManagerInstalled(apiExtensionsClient) + + versionBundle, err := version.GetVersionBundle(o.bundleVersion) if err != nil { + klog.Errorf("unable to retrieve version ", err) return err } - if !installed { - return fmt.Errorf("clustermanager is not installed") + o.values = clusteradminit.Values{ + Hub: clusteradminit.Hub{ + Registry: o.registry, + }, + // reconstruct values from the cluster manager CR. + RegistrationFeatures: cm.Spec.RegistrationConfiguration.FeatureGates, + WorkFeatures: cm.Spec.WorkConfiguration.FeatureGates, + AddonFeatures: cm.Spec.AddOnManagerConfiguration.FeatureGates, + BundleVersion: clusteradminit.BundleVersion{ + RegistrationImageVersion: versionBundle.Registration, + PlacementImageVersion: versionBundle.Placement, + WorkImageVersion: versionBundle.Work, + OperatorImageVersion: versionBundle.Operator, + }, + } + + if len(cm.Spec.RegistrationConfiguration.AutoApproveUsers) > 0 { + o.values.AutoApprove = true + } + + return nil +} + +func (o *Options) validate() (err error) { + err = o.ClusteradmFlags.ValidateHub() + if err != nil { + return err } //TODO check desired version is greater then current version diff --git a/pkg/cmd/upgrade/clustermanager/options.go b/pkg/cmd/upgrade/clustermanager/options.go index 7258fb332..aebf45c91 100644 --- a/pkg/cmd/upgrade/clustermanager/options.go +++ b/pkg/cmd/upgrade/clustermanager/options.go @@ -4,6 +4,7 @@ package clustermanager import ( "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/cli-runtime/pkg/resource" + clusteradminit "open-cluster-management.io/clusteradm/pkg/cmd/init" genericclioptionsclusteradm "open-cluster-management.io/clusteradm/pkg/genericclioptions" ) @@ -12,7 +13,7 @@ type Options struct { //ClusteradmFlags: The generic options from the clusteradm cli-runtime. ClusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags - values Values + values clusteradminit.Values //The file to output the resources will be sent to the file. registry string //version of predefined compatible image versions @@ -25,36 +26,6 @@ type Options struct { builder *resource.Builder } -type BundleVersion struct { - // registration image version - RegistrationImageVersion string - // placement image version - PlacementImageVersion string - // work image version - WorkImageVersion string - // operator image version - OperatorImageVersion string -} - -// Values: The values used in the template -type Values struct { - //The values related to the hub - Registry string `json:"registry"` - //bundle version - BundleVersion BundleVersion - //Hub: Hub information - Hub Hub -} - -type Hub struct { - //APIServer: The API Server external URL - APIServer string - //KubeConfig: The kubeconfig of the bootstrap secret to connect to the hub - KubeConfig string - //image registry - Registry string -} - func newOptions(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, streams genericclioptions.IOStreams) *Options { return &Options{ ClusteradmFlags: clusteradmFlags, diff --git a/pkg/cmd/upgrade/klusterlet/exec.go b/pkg/cmd/upgrade/klusterlet/exec.go index 379fb1140..36815a0ae 100644 --- a/pkg/cmd/upgrade/klusterlet/exec.go +++ b/pkg/cmd/upgrade/klusterlet/exec.go @@ -10,6 +10,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" operatorclient "open-cluster-management.io/api/client/operator/clientset/versioned" + "open-cluster-management.io/clusteradm/pkg/cmd/join" join_scenario "open-cluster-management.io/clusteradm/pkg/cmd/join/scenario" "open-cluster-management.io/clusteradm/pkg/helpers" "open-cluster-management.io/clusteradm/pkg/helpers/reader" @@ -47,31 +48,26 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) { return err } - klog.V(1).InfoS("init options:", "dry-run", o.ClusteradmFlags.DryRun) - o.values = Values{ - ClusterName: k.Spec.ClusterName, - Hub: Hub{ - Registry: o.registry, - }, - } - versionBundle, err := version.GetVersionBundle(o.bundleVersion) - if err != nil { klog.Errorf("unable to retrieve version ", err) return err } - o.values.BundleVersion = BundleVersion{ - RegistrationImageVersion: versionBundle.Registration, - PlacementImageVersion: versionBundle.Placement, - WorkImageVersion: versionBundle.Work, - OperatorImageVersion: versionBundle.Operator, + klog.V(1).InfoS("init options:", "dry-run", o.ClusteradmFlags.DryRun) + o.values = join.Values{ + Registry: o.registry, + ClusterName: k.Spec.ClusterName, + RegistrationFeatures: k.Spec.RegistrationConfiguration.FeatureGates, + WorkFeatures: k.Spec.WorkConfiguration.FeatureGates, + BundleVersion: join.BundleVersion{ + RegistrationImageVersion: versionBundle.Registration, + PlacementImageVersion: versionBundle.Placement, + WorkImageVersion: versionBundle.Work, + OperatorImageVersion: versionBundle.Operator, + }, } - f := o.ClusteradmFlags.KubectlFactory - o.builder = f.NewBuilder() - return nil } diff --git a/pkg/cmd/upgrade/klusterlet/options.go b/pkg/cmd/upgrade/klusterlet/options.go index 86e31e90f..0c3788f73 100644 --- a/pkg/cmd/upgrade/klusterlet/options.go +++ b/pkg/cmd/upgrade/klusterlet/options.go @@ -4,6 +4,7 @@ package klusterlet import ( "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/cli-runtime/pkg/resource" + "open-cluster-management.io/clusteradm/pkg/cmd/join" genericclioptionsclusteradm "open-cluster-management.io/clusteradm/pkg/genericclioptions" ) @@ -12,7 +13,7 @@ type Options struct { //ClusteradmFlags: The generic options from the clusteradm cli-runtime. ClusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags - values Values + values join.Values //The file to output the resources will be sent to the file. registry string //version of predefined compatible image versions diff --git a/test/e2e/clusteradm/e2e_suite_test.go b/test/e2e/clusteradm/e2e_suite_test.go index 317ededd2..6204b3f18 100644 --- a/test/e2e/clusteradm/e2e_suite_test.go +++ b/test/e2e/clusteradm/e2e_suite_test.go @@ -77,7 +77,7 @@ var _ = ginkgo.BeforeSuite(func() { operatorClient, err = operatorclient.NewForConfig(hubConfig) gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = clusterv1.AddToScheme(scheme.Scheme) + err = clusterv1.Install(scheme.Scheme) gomega.Expect(err).NotTo(gomega.HaveOccurred()) restConfig = hubConfig diff --git a/test/e2e/clusteradm/upgrade_test.go b/test/e2e/clusteradm/upgrade_test.go index 53352f7d5..6a2904734 100644 --- a/test/e2e/clusteradm/upgrade_test.go +++ b/test/e2e/clusteradm/upgrade_test.go @@ -2,8 +2,14 @@ package clusteradme2e import ( + "context" + "fmt" + "time" + "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "open-cluster-management.io/clusteradm/pkg/helpers/version" ) var _ = ginkgo.Describe("test clusteradm upgrade clustermanager and Klusterlets", ginkgo.Ordered, func() { @@ -12,28 +18,67 @@ var _ = ginkgo.Describe("test clusteradm upgrade clustermanager and Klusterlets" ginkgo.By("reset e2e environment...") err := e2e.ClearEnv() gomega.Expect(err).NotTo(gomega.HaveOccurred()) - err = e2e.ResetEnv() - gomega.Expect(err).NotTo(gomega.HaveOccurred()) }) var err error ginkgo.It("run cluster manager upgrade version latest ", func() { - ginkgo.Skip("Upgrade is skipped due to flaky when destroying control plane. Need to revisit it after fix cleanup issue") - err = e2e.Clusteradm().Upgrade( - "clustermanager", - "--bundle-version", "latest", + ginkgo.By("init hub with service account") + err = e2e.Clusteradm().Init( "--context", e2e.Cluster().Hub().Context(), ) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm init error") - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm upgrade error") + ginkgo.By("Check the version of operator and agent") + gomega.Eventually(func() error { + operator, err := kubeClient.AppsV1().Deployments("open-cluster-management").Get(context.TODO(), "cluster-manager", metav1.GetOptions{}) + if err != nil { + return err + } + if operator.Spec.Template.Spec.Containers[0].Image != "quay.io/open-cluster-management/registration-operator:v"+version.GetDefaultBundleVersion() { + return fmt.Errorf("version of the operator is not correct, get %s", operator.Spec.Template.Spec.Containers[0].Image) + } + as, _ := kubeClient.AppsV1().Deployments("open-cluster-management-hub").List(context.TODO(), metav1.ListOptions{}) + for _, a := range as.Items { + ginkgo.By(fmt.Sprintf("deployment is %s\n", a.Name)) + } + registration, err := kubeClient.AppsV1().Deployments("open-cluster-management-hub").Get( + context.TODO(), "cluster-manager-registration-controller", metav1.GetOptions{}) + if err != nil { + return err + } + if registration.Spec.Template.Spec.Containers[0].Image != "quay.io/open-cluster-management/registration:v"+version.GetDefaultBundleVersion() { + return fmt.Errorf("version of the registration agent is not correct, get %s", operator.Spec.Template.Spec.Containers[0].Image) + } + return nil + }, 5*time.Second, 30*time.Second).Should(gomega.Succeed()) err = e2e.Clusteradm().Upgrade( - "klusterlet", - "--bundle-version", "latest", - "--context", e2e.Cluster().ManagedCluster1().Context(), + "clustermanager", + "--bundle-version=latest", + "--context", e2e.Cluster().Hub().Context(), ) - gomega.Expect(err).NotTo(gomega.HaveOccurred(), "klusterlet upgrade error") + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm upgrade error") + + ginkgo.By("Upgrade to the latest version") + gomega.Eventually(func() error { + operator, err := kubeClient.AppsV1().Deployments("open-cluster-management").Get(context.TODO(), "cluster-manager", metav1.GetOptions{}) + if err != nil { + return err + } + if operator.Spec.Template.Spec.Containers[0].Image != "quay.io/open-cluster-management/registration-operator:latest" { + return fmt.Errorf("version of the operator is not correct, get %s", operator.Spec.Template.Spec.Containers[0].Image) + } + registration, err := kubeClient.AppsV1().Deployments("open-cluster-management-hub").Get( + context.TODO(), "cluster-manager-registration-controller", metav1.GetOptions{}) + if err != nil { + return err + } + if registration.Spec.Template.Spec.Containers[0].Image != "quay.io/open-cluster-management/registration:latest" { + return fmt.Errorf("version of the operator is not correct, get %s", operator.Spec.Template.Spec.Containers[0].Image) + } + return nil + }, 5*time.Second, 30*time.Second).Should(gomega.Succeed()) }) })