WIP:Add integration test for agent in detached mode

test/integration/integration_suite_test.go

@@ -58,6 +58,15 @@ var testNamespace string
 
 var authn *util.TestAuthn
 
+// detachedTestEnv, detachedCfg and detachedKubeClient will be used in Detached mode.
+// for klusterlet, it represents the managed cluster.
+var (
+	detachedTestEnv             *envtest.Environment
+	detachedCfg                 *rest.Config
+	detachedKubeClient          kubernetes.Interface
+	detachedSpokeKubeconfigFile string
+)
+
 func TestIntegration(t *testing.T) {
 	gomega.RegisterFailHandler(ginkgo.Fail)
 	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Integration Suite", []ginkgo.Reporter{printer.NewlineReporter{}})
@@ -134,6 +143,32 @@ var _ = ginkgo.BeforeSuite(func(done ginkgo.Done) {
 	gomega.Expect(err).ToNot(gomega.HaveOccurred())
 	gomega.Expect(clusterClient).ToNot(gomega.BeNil())
 
+	// prepare cluster for detached mode
+	{
+		// start a local kube-apiserver as the hub/managed cluster for Detached mode.
+		detachedTestEnv = &envtest.Environment{
+			ErrorIfCRDPathMissing: true,
+			CRDDirectoryPaths: []string{
+				// filepath.Join(".", "deploy", "hub"),
+				filepath.Join(".", "deploy", "spoke"),
+			},
+		}
+		detachedConfig, err := detachedTestEnv.Start()
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+		gomega.Expect(detachedConfig).ToNot(gomega.BeNil())
+
+		detachedCfg = detachedConfig
+		gomega.Expect(detachedCfg).ToNot(gomega.BeNil())
+
+		detachedKubeClient, err = kubernetes.NewForConfig(detachedCfg)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+		gomega.Expect(detachedKubeClient).ToNot(gomega.BeNil())
+
+		detachedSpokeKubeconfigFile = path.Join(util.TestDir, "detached", "spoke-kubeconfig")
+		util.TranslateKubeConfiguration(detachedCfg, detachedSpokeKubeconfigFile)
+		gomega.Expect(err).ToNot(gomega.HaveOccurred())
+	}
+
 	// prepare test namespace
 	nsBytes, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
 	if err != nil {
@@ -162,6 +197,9 @@ var _ = ginkgo.AfterSuite(func() {
 	err := testEnv.Stop()
 	gomega.Expect(err).ToNot(gomega.HaveOccurred())
 
+	err = detachedTestEnv.Stop()
+	gomega.Expect(err).ToNot(gomega.HaveOccurred())
+
 	err = os.RemoveAll(util.TestDir)
 	gomega.Expect(err).ToNot(gomega.HaveOccurred())
 })

test/integration/spokeagent_detached_test.go

@@ -0,0 +1,107 @@
+package integration_test
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"path"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+
+	clusterv1 "open-cluster-management.io/api/cluster/v1"
+	"open-cluster-management.io/registration/pkg/spoke"
+	"open-cluster-management.io/registration/test/integration/util"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+)
+
+var _ = ginkgo.Describe("Agent detached mode", func() {
+
+	ginkgo.It("spoke agent runs outside of the managed cluster, detached mode", func() {
+		var err error
+		managedClusterName := "detached-test-cluster1"
+
+		hubKubeconfigSecret := "detached-test-hub-kubeconfig-secret"
+		hubKubeconfigDir := path.Join(util.TestDir, "detached-agent-test", "hub-kubeconfig")
+		bootstrapFile := path.Join(util.TestDir, "detached-agent-test", "kubeconfig")
+
+		ginkgo.By("Create bootstrap kubeconfig")
+		err = authn.CreateBootstrapKubeConfigWithCertAge(bootstrapFile, serverCertFile, securePort, 20*time.Second)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		ginkgo.By("run registration agent")
+		agentOptions := spoke.SpokeAgentOptions{
+			ClusterName:              managedClusterName,
+			BootstrapKubeconfig:      bootstrapFile,
+			HubKubeconfigSecret:      hubKubeconfigSecret,
+			HubKubeconfigDir:         hubKubeconfigDir,
+			SpokeKubeconfig:          detachedSpokeKubeconfigFile,
+			ClusterHealthCheckPeriod: 1 * time.Minute,
+			// set the SpokeExternalServerURL with detach cluster URL
+			SpokeExternalServerURLs: []string{detachedCfg.Host},
+		}
+
+		stopAgent := util.RunAgent("detached-test", agentOptions, spokeCfg)
+		defer stopAgent()
+
+		ginkgo.By("Check existence of csr and ManagedCluster")
+		// the csr should be created
+		gomega.Eventually(func() bool {
+			_, err := util.FindUnapprovedSpokeCSR(kubeClient, managedClusterName)
+			return err == nil
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		// the spoke cluster should be created
+		gomega.Eventually(func() bool {
+			if _, err := util.GetManagedCluster(clusterClient, managedClusterName); err != nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		ginkgo.By("Accept ManagedCluster and approve csr")
+		err = util.AcceptManagedCluster(clusterClient, managedClusterName)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		err = authn.ApproveSpokeClusterCSR(kubeClient, managedClusterName, time.Second*20)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+		ginkgo.By("Check if hub kubeconfig secret is updated")
+		// the hub kubeconfig secret should be filled after the csr is approved
+		gomega.Eventually(func() bool {
+			if _, err := util.GetFilledHubKubeConfigSecret(kubeClient, testNamespace, hubKubeconfigSecret); err != nil {
+				return false
+			}
+			return true
+		}, eventuallyTimeout, eventuallyInterval).Should(gomega.BeTrue())
+
+		ginkgo.By("Check if ManagedCluster joins the hub")
+		// the spoke cluster should have joined condition finally
+		gomega.Eventually(func() error {
+			spokeCluster, err := util.GetManagedCluster(clusterClient, managedClusterName)
+			if err != nil {
+				return err
+			}
+			if !meta.IsStatusConditionTrue(spokeCluster.Status.Conditions, clusterv1.ManagedClusterConditionJoined) {
+				return fmt.Errorf("cluster should be joined")
+			}
+			return nil
+		}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
+
+		// check whether the managed cluster is detached cluster by client config CABundle.
+		gomega.Eventually(func() error {
+			spokeCluster, err := util.GetManagedCluster(clusterClient, managedClusterName)
+			if err != nil {
+				return err
+			}
+			for _, config := range spokeCluster.Spec.ManagedClusterClientConfigs {
+				if config.URL == detachedCfg.Host && bytes.Equal(config.CABundle, detachedCfg.CAData) {
+					return nil
+				}
+			}
+			return errors.New("managed cluster client config does not match")
+		}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
+	})
+})

test/integration/util/util.go

@@ -609,3 +609,39 @@ func RunAgent(name string, opt spoke.SpokeAgentOptions, cfg *rest.Config) contex
 
 	return cancel
 }
+
+// TranslateKubeConfiguration translates the rest.Config to kubeconfig file,
+// and write it to file.
+func TranslateKubeConfiguration(c *rest.Config, filePath string) error {
+	namespace := "default"
+
+	clusters := make(map[string]*clientcmdapi.Cluster)
+	clusters["default-cluster"] = &clientcmdapi.Cluster{
+		Server:                   c.Host,
+		CertificateAuthorityData: c.CAData,
+	}
+
+	contexts := make(map[string]*clientcmdapi.Context)
+	contexts["default-context"] = &clientcmdapi.Context{
+		Cluster:   "default-cluster",
+		Namespace: namespace,
+		AuthInfo:  namespace,
+	}
+
+	authinfos := make(map[string]*clientcmdapi.AuthInfo)
+	authinfos[namespace] = &clientcmdapi.AuthInfo{
+		Token:                 c.BearerToken,
+		ClientCertificateData: c.CertData,
+		ClientKeyData:         c.KeyData,
+	}
+
+	clientConfig := clientcmdapi.Config{
+		Kind:           "Config",
+		APIVersion:     "v1",
+		Clusters:       clusters,
+		Contexts:       contexts,
+		CurrentContext: "default-context",
+		AuthInfos:      authinfos,
+	}
+	return clientcmd.WriteToFile(clientConfig, filePath)
+}