feat. ingress mutate webhook support annotations flags. (#4293)

* feat. ingress mutate webhook support annotations flags. Signed-off-by: yy <[email protected]> * fix init annotations logic Signed-off-by: yy <[email protected]> * fix deploy.yaml.tmpl Signed-off-by: yy <[email protected]> --------- Signed-off-by: yy <[email protected]>
labring · Nov 14, 2023 · a2f3ca0 · a2f3ca0
1 parent ea37dc9
commit a2f3ca0
Show file tree

Hide file tree

Showing 10 changed files with 221 additions and 56 deletions.
diff --git a/controllers/admission/api/v1/ingress_webhook.go b/controllers/admission/api/v1/ingress_webhook.go
@@ -47,19 +47,38 @@ var ilog = logf.Log.WithName("ingress-validating-webhook")
 
 type IngressMutator struct {
 	client.Client
+	domain             string
+	IngressAnnotations map[string]string
 }
 
 func (m *IngressMutator) SetupWithManager(mgr ctrl.Manager) error {
+	m.Client = mgr.GetClient()
+	m.domain = os.Getenv("DOMAIN")
 	return builder.WebhookManagedBy(mgr).
 		For(&netv1.Ingress{}).
-		WithDefaulter(&IngressMutator{Client: mgr.GetClient()}).
+		WithDefaulter(m).
 		Complete()
 }
 
-func (m *IngressMutator) Default(_ context.Context, _ runtime.Object) error {
+func (m *IngressMutator) Default(_ context.Context, obj runtime.Object) error {
+	i, ok := obj.(*netv1.Ingress)
+	if !ok {
+		return errors.New("obj convert Ingress is error")
+	}
+	if isUserNamespace(i.Namespace) && hasSubDomain(i, m.domain) {
+		ilog.Info("mutating ingress in user ns", "ingress namespace", i.Namespace, "ingress name", i.Name)
+		m.mutateUserIngressAnnotations(i)
+	}
 	return nil
 }
 
+func (m *IngressMutator) mutateUserIngressAnnotations(i *netv1.Ingress) {
+	initAnnotationAndLabels(&i.ObjectMeta)
+	for k, v := range m.IngressAnnotations {
+		i.Annotations[k] = v
+	}
+}
+
 //+kubebuilder:object:generate=false
 
 type IngressValidator struct {
@@ -160,6 +179,11 @@ func (v *IngressValidator) validate(ctx context.Context, i *netv1.Ingress) error
 		return nil
 	}
 
+	if !isUserNamespace(i.Namespace) {
+		ilog.Info("namespace is system namespace, skip validate")
+		return nil
+	}
+
 	checks := []func(*netv1.Ingress, *netv1.IngressRule) error{
 		v.checkCname,
 		v.checkOwner,

diff --git a/controllers/admission/api/v1/namespace_webhook.go b/controllers/admission/api/v1/namespace_webhook.go
@@ -45,7 +45,7 @@ func (m *NamespaceMutator) Default(_ context.Context, obj runtime.Object) error
 		return errors.New("obj convert to Namespace error")
 	}
 	nlog.Info("mutating create/update", "name", i.Name)
-	i.ObjectMeta = initAnnotationAndLabels(i.ObjectMeta)
+	initAnnotationAndLabels(&i.ObjectMeta)
 
 	// add sealos.io/namespace annotation
 	i.Annotations["sealos.io/namespace"] = i.Name

diff --git a/controllers/admission/api/v1/validate.go → controllers/admission/api/v1/utils.go b/controllers/admission/api/v1/validate.go → controllers/admission/api/v1/utils.go
@@ -17,21 +17,39 @@ package v1
 import (
 	"strings"
 
+	"github.com/miekg/dns"
+
+	netv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-const userServiceAccountPrefix = "system:serviceaccount:ns-"
+const (
+	userServiceAccountPrefix = "system:serviceaccount:ns-"
+	userNamespacePrefix      = "ns-"
+)
 
 func isUserServiceAccount(sa string) bool {
 	return strings.HasPrefix(sa, userServiceAccountPrefix)
 }
 
-func initAnnotationAndLabels(meta metav1.ObjectMeta) metav1.ObjectMeta {
+func isUserNamespace(ns string) bool {
+	return strings.HasPrefix(ns, userNamespacePrefix)
+}
+
+func hasSubDomain(i *netv1.Ingress, domain string) bool {
+	for _, r := range i.Spec.Rules {
+		if dns.IsSubDomain(domain, r.Host) {
+			return true
+		}
+	}
+	return false
+}
+
+func initAnnotationAndLabels(meta *metav1.ObjectMeta) {
 	if meta.Annotations == nil {
 		meta.Annotations = make(map[string]string)
 	}
 	if meta.Labels == nil {
 		meta.Labels = make(map[string]string)
 	}
-	return meta
 }
diff --git a/controllers/admission/cmd/main.go b/controllers/admission/cmd/main.go
@@ -18,8 +18,8 @@ package main
 
 import (
 	"flag"
-
 	"os"
+	"strings"
 
 	v1 "github.com/labring/sealos/controllers/admission/api/v1"
 
@@ -52,17 +52,35 @@ func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
 	var probeAddr string
+	var ingressAnnotationString string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
+	flag.StringVar(&ingressAnnotationString, "ingress-mutating-annotations", "", "Ingress annotations: 'key1=value1,key2=value2'")
+
 	opts := zap.Options{
 		Development: true,
 	}
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
 
+	setupLog.Info("ingress annotations:", "annotation", ingressAnnotationString)
+	ingressAnnotations := make(map[string]string)
+	kvs := strings.Split(ingressAnnotationString, ",")
+	for _, kv := range kvs {
+		parts := strings.Split(kv, "=")
+		if len(parts) == 2 {
+			key := parts[0]
+			value := parts[1]
+			ingressAnnotations[key] = value
+		} else {
+			setupLog.Error(nil, "ingress annotation format error", "annotation", kv)
+			os.Exit(1)
+		}
+	}
+
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
 
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
@@ -94,7 +112,9 @@ func main() {
 		os.Exit(1)
 	}
 
-	if (&v1.IngressMutator{}).SetupWithManager(mgr) != nil {
+	if (&v1.IngressMutator{
+		IngressAnnotations: ingressAnnotations,
+	}).SetupWithManager(mgr) != nil {
 		setupLog.Error(err, "unable to create ingress mutator webhook")
 		os.Exit(1)
 	}

diff --git a/controllers/admission/config/default/manager_auth_proxy_patch.yaml b/controllers/admission/config/default/manager_auth_proxy_patch.yaml
@@ -51,3 +51,4 @@ spec:
         - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=127.0.0.1:8080"
         - "--leader-elect"
+        - "--ingress-mutate-annotations='{{ .ingressMutateAnnotations }}'"
diff --git a/controllers/admission/config/manager/manager.yaml b/controllers/admission/config/manager/manager.yaml
@@ -84,6 +84,7 @@ spec:
         - /manager
         args:
         - --leader-elect
+        - --ingress-mutate-annotations='{{ .ingressMutatingAnnotations }}'
         image: controller:latest
         env:
         - name: DOMAIN

diff --git a/controllers/admission/deploy/Kubefile b/controllers/admission/deploy/Kubefile
@@ -7,6 +7,10 @@ COPY manifests manifests
 
 ENV cloudDomain="cloud.sealos.io"
 
+# Example for ingressMutatingAnnotations
+# ENV ingressMutatingAnnotations="nginx.ingress.kubernetes.io/limit-connections=10,nginx.ingress.kubernetes.io/limit-rate-after=10m,nginx.ingress.kubernetes.io/limit-rate=100k,nginx.ingress.kubernetes.io/proxy-buffering=on"
+
+ENV ingressMutatingAnnotations=""
 ENV ingressWebhookEnabled="true"
 ENV ingressWebhookFailurePolicy="Fail"
 ENV icpEnabled="false"

diff --git a/controllers/admission/deploy/manifests/deploy.yaml.tmpl b/controllers/admission/deploy/manifests/deploy.yaml.tmpl
@@ -291,6 +291,7 @@ spec:
         - --health-probe-bind-address=:8081
         - --metrics-bind-address=127.0.0.1:8080
         - --leader-elect
+        - --ingress-mutating-annotations={{ .ingressMutatingAnnotations }}
         command:
         - /manager
         env: