Merge pull request kubesphere#3334 from xyz-li/app

manage openpitrix with crd

Makefile

@@ -88,7 +88,7 @@ docker-build-no-test: ks-apiserver ks-controller-manager
 
 # Run tests
 test: fmt vet
-	export KUBEBUILDER_CONTROLPLANE_START_TIMEOUT=1m; go test ./pkg/... ./cmd/... -covermode=atomic -coverprofile=coverage.txt
+	export KUBEBUILDER_CONTROLPLANE_START_TIMEOUT=2m; go test ./pkg/... ./cmd/... -covermode=atomic -coverprofile=coverage.txt
 
 .PHONY: clean
 clean:

build/ks-apiserver/Dockerfile

@@ -1,10 +1,12 @@
 # Copyright 2020 The KubeSphere Authors. All rights reserved.
 # Use of this source code is governed by an Apache license
 # that can be found in the LICENSE file.
-FROM alpine:3.11
+FROM alpine/helm:3.4.2 as helm-base
 
-RUN apk add --no-cache ca-certificates 
+FROM alpine:3.11
 
+RUN apk add --no-cache ca-certificates
+COPY --from=helm-base /usr/bin/helm /usr/bin/helm
 # To speed up building process, we copy binary directly from make
 # result instead of building it again, so make sure you run the 
 # following command first before building docker image

build/ks-controller-manager/Dockerfile

@@ -1,10 +1,12 @@
 # Copyright 2020 The KubeSphere Authors. All rights reserved.
 # Use of this source code is governed by an Apache license
 # that can be found in the LICENSE file.
+FROM alpine/helm:3.4.2 as helm-base
+
 FROM alpine:3.11
 
 RUN apk add --no-cache ca-certificates
-
+COPY --from=helm-base /usr/bin/helm /usr/bin/helm
 COPY  /bin/cmd/controller-manager /usr/local/bin/
 
 EXPOSE 8443 8080

cmd/controller-manager/app/controllers.go

@@ -51,7 +51,6 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/multicluster"
 	"kubesphere.io/kubesphere/pkg/simple/client/network"
 	ippoolclient "kubesphere.io/kubesphere/pkg/simple/client/network/ippool"
-	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/kubefed/pkg/controller/util"
@@ -66,7 +65,6 @@ func addControllers(
 	ldapClient ldapclient.Interface,
 	options *k8s.KubernetesOptions,
 	authenticationOptions *authoptions.AuthenticationOptions,
-	openpitrixClient openpitrix.Client,
 	multiClusterOptions *multicluster.Options,
 	networkOptions *network.Options,
 	serviceMeshEnabled bool,
@@ -233,7 +231,6 @@ func addControllers(
 			client.Config(),
 			kubesphereInformer.Cluster().V1alpha1().Clusters(),
 			client.KubeSphere().ClusterV1alpha1().Clusters(),
-			openpitrixClient,
 			multiClusterOptions.ClusterControllerResyncSecond)
 	}
 

cmd/controller-manager/app/server.go

@@ -18,21 +18,23 @@ package app
 
 import (
 	"fmt"
+	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
-	"kubesphere.io/kubesphere/pkg/controller/application"
-	"os"
-
-	"github.com/spf13/cobra"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/klog"
 	"k8s.io/klog/klogr"
 	"kubesphere.io/kubesphere/cmd/controller-manager/app/options"
 	"kubesphere.io/kubesphere/pkg/apis"
 	controllerconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
+	"kubesphere.io/kubesphere/pkg/controller/application"
 	"kubesphere.io/kubesphere/pkg/controller/namespace"
 	"kubesphere.io/kubesphere/pkg/controller/network/webhooks"
+	"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmapplication"
+	"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmcategory"
+	"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmrelease"
+	"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmrepo"
 	"kubesphere.io/kubesphere/pkg/controller/quota"
 	"kubesphere.io/kubesphere/pkg/controller/serviceaccount"
 	"kubesphere.io/kubesphere/pkg/controller/user"
@@ -45,10 +47,10 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
 	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
 	ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap"
-	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
 	"kubesphere.io/kubesphere/pkg/utils/metrics"
 	"kubesphere.io/kubesphere/pkg/utils/term"
+	"os"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
@@ -142,14 +144,6 @@ func run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{})
 		klog.Warning("ks-controller-manager starts without ldap provided, it will not sync user into ldap")
 	}
 
-	var openpitrixClient openpitrix.Client
-	if s.OpenPitrixOptions != nil && !s.OpenPitrixOptions.IsEmpty() {
-		openpitrixClient, err = openpitrix.NewClient(s.OpenPitrixOptions)
-		if err != nil {
-			return fmt.Errorf("failed to connect to openpitrix, please check openpitrix status, error: %v", err)
-		}
-	}
-
 	var s3Client s3.Interface
 	if s.S3Options != nil && len(s.S3Options.Endpoint) != 0 {
 		s3Client, err = s3.NewS3Client(s.S3Options)
@@ -224,6 +218,41 @@ func run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{})
 		klog.Fatalf("Unable to create namespace controller: %v", err)
 	}
 
+	err = helmrepo.Add(mgr)
+	if err != nil {
+		klog.Fatal("Unable to create helm repo controller")
+	}
+
+	err = helmcategory.Add(mgr)
+	if err != nil {
+		klog.Fatal("Unable to create helm category controller")
+	}
+
+	if !s.OpenPitrixOptions.IsEmpty() {
+		storageClient, err := s3.NewS3Client(s.OpenPitrixOptions.S3Options)
+		if err != nil {
+			klog.Fatalf("failed to connect to s3, please check openpitrix s3 service status, error: %v", err)
+		}
+		err = (&helmapplication.ReconcileHelmApplication{}).SetupWithManager(mgr)
+		if err != nil {
+			klog.Fatalf("Unable to create helm application controller, error: %s", err)
+		}
+
+		err = (&helmapplication.ReconcileHelmApplicationVersion{}).SetupWithManager(mgr)
+		if err != nil {
+			klog.Fatalf("Unable to create helm application version controller, error: %s ", err)
+		}
+
+		err = (&helmrelease.ReconcileHelmRelease{
+			StorageClient: storageClient,
+			KsFactory:     informerFactory.KubeSphereSharedInformerFactory(),
+		}).SetupWithManager(mgr)
+
+		if err != nil {
+			klog.Fatalf("Unable to create helm release controller, error: %s", err)
+		}
+	}
+
 	selector, _ := labels.Parse(s.ApplicationSelector)
 	applicationReconciler := &application.ApplicationReconciler{
 		Scheme:              mgr.GetScheme(),
@@ -255,7 +284,6 @@ func run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{})
 		ldapClient,
 		s.KubernetesOptions,
 		s.AuthenticationOptions,
-		openpitrixClient,
 		s.MultiClusterOptions,
 		s.NetworkOptions,
 		servicemeshEnabled,
@@ -283,7 +311,9 @@ func run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{})
 	hookServer.Register("/validate-quota-kubesphere-io-v1alpha2", &webhook.Admission{Handler: resourceQuotaAdmission})
 
 	klog.V(2).Info("registering metrics to the webhook server")
-	hookServer.Register("/metrics", metrics.Handler())
+	// Add an extra metric endpoint, so we can use the the same metric definition with ks-apiserver
+	// /kapis/metrics is independent of controller-manager's built-in /metrics
+	mgr.AddMetricsExtraHandler("/kapis/metrics", metrics.Handler())
 
 	klog.V(0).Info("Starting the controllers.")
 	if err = mgr.Start(stopCh); err != nil {

cmd/ks-apiserver/app/options/options.go

@@ -39,7 +39,6 @@ import (
 	esclient "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring/metricsserver"
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus"
-	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
 	fakes3 "kubesphere.io/kubesphere/pkg/simple/client/s3/fake"
 	"kubesphere.io/kubesphere/pkg/simple/client/sonarqube"
@@ -196,14 +195,6 @@ func (s *ServerRunOptions) NewAPIServer(stopCh <-chan struct{}) (*apiserver.APIS
 		apiServer.AuditingClient = auditingClient
 	}
 
-	if s.OpenPitrixOptions != nil && !s.OpenPitrixOptions.IsEmpty() {
-		opClient, err := openpitrix.NewClient(s.OpenPitrixOptions)
-		if err != nil {
-			return nil, fmt.Errorf("failed to connect to openpitrix, please check openpitrix status, error: %v", err)
-		}
-		apiServer.OpenpitrixClient = opClient
-	}
-
 	if s.AlertingOptions != nil && (s.AlertingOptions.PrometheusEndpoint != "" || s.AlertingOptions.ThanosRulerEndpoint != "") {
 		alertingClient, err := alerting.NewRuleClient(s.AlertingOptions)
 		if err != nil {