Merge pull request #4460 from cyclinder/0.8_cp/multus4.0

bump multus to v4 & fix multus fails to reach api server when the old service account is out of data

charts/spiderpool/README.md

@@ -200,7 +200,7 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \
 | `multus.multusCNI.image.repository`           | the multus-CNI image repository                                                                                                                                                                                                                                                                                                                                                                  | `k8snetworkplumbingwg/multus-cni` |
 | `multus.multusCNI.image.pullPolicy`           | the multus-CNI image pullPolicy                                                                                                                                                                                                                                                                                                                                                                  | `IfNotPresent`                    |
 | `multus.multusCNI.image.digest`               | the multus-CNI image digest                                                                                                                                                                                                                                                                                                                                                                      | `""`                              |
-| `multus.multusCNI.image.tag`                  | the multus-CNI image tag                                                                                                                                                                                                                                                                                                                                                                         | `v3.9.3`                          |
+| `multus.multusCNI.image.tag`                  | the multus-CNI image tag                                                                                                                                                                                                                                                                                                                                                                         | `v4.1.4`                          |
 | `multus.multusCNI.image.imagePullSecrets`     | the multus-CNI image imagePullSecrets                                                                                                                                                                                                                                                                                                                                                            | `[]`                              |
 | `multus.multusCNI.defaultCniCRName`           | if this value is empty, multus will automatically get default CNI according to the existed CNI conf file in /etc/cni/net.d/, if no cni files found in /etc/cni/net.d, A Spidermultusconfig CR named default will be created, please update the related SpiderMultusConfig for default CNI after installation. The namespace of defaultCniCRName follows with the release namespace of spdierpool | `""`                              |
 | `multus.multusCNI.securityContext.privileged` | the securityContext privileged of multus-CNI daemonset pod                                                                                                                                                                                                                                                                                                                                       | `true`                            |

charts/spiderpool/templates/configmap.yaml

@@ -13,6 +13,7 @@ metadata:
     {{- include "tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }}
   {{- end }}
 data:
+  clusterNetwork: {{ .Values.multus.multusCNI.defaultCniCRName | quote }}
   conf.yml: |
     ipamUnixSocketPath: {{ .Values.global.ipamUNIXSocketHostPath }}
     enableIPv4: {{ .Values.ipam.enableIPv4 }}
@@ -31,31 +32,168 @@ data:
 kind: ConfigMap
 apiVersion: v1
 metadata:
-  name: {{ .Values.multus.multusCNI.name | trunc 63 | trimSuffix "-" }}
+  name: {{ .Values.multus.multusCNI.name | trunc 63 | trimSuffix "-" }}-entrypoint
   namespace: {{ .Release.Namespace | quote }}
   labels:
     {{- include "spiderpool.multus.labels" . | nindent 4 }}
       {{- if .Values.global.commonLabels }}
       {{- include "tplvalues.render" ( dict "value" .Values.global.commonLabels "context" $ ) | nindent 4 }}
       {{- end }}
 data:
-  cni-conf.json: |
+  entrypoint.sh: |
+    #!/bin/bash
+    set -e
+
+    function log(){
+        echo "INFO: $(date --iso-8601=seconds) ${1}"
+    }
+    function error(){
+        log "ERR:  {$1}"
+    }
+    function warn(){
+        log "WARN: {$1}"
+    }
+
+    function generateKubeConfig {
+      # Check if we're running as a k8s pod.
+      if [ -f "$SERVICE_ACCOUNT_TOKEN_PATH" ]; then
+        # We're running as a k8d pod - expect some variables.
+        if [ -z ${KUBERNETES_SERVICE_HOST} ]; then
+          error "KUBERNETES_SERVICE_HOST not set"; exit 1;
+        fi
+        if [ -z ${KUBERNETES_SERVICE_PORT} ]; then
+          error "KUBERNETES_SERVICE_PORT not set"; exit 1;
+        fi
+
+        if [ "$SKIP_TLS_VERIFY" == "true" ]; then
+          TLS_CFG="insecure-skip-tls-verify: true"
+        elif [ -f "$KUBE_CA_FILE" ]; then
+          TLS_CFG="certificate-authority-data: $(cat $KUBE_CA_FILE | base64 | tr -d '\n')"
+        fi
+
+        # Get the contents of service account token.
+        SERVICEACCOUNT_TOKEN=$(cat $SERVICE_ACCOUNT_TOKEN_PATH)
+
+        SKIP_TLS_VERIFY=${SKIP_TLS_VERIFY:-false}
+
+        # Write a kubeconfig file for the CNI plugin.  Do this
+        # to skip TLS verification for now.  We should eventually support
+        # writing more complete kubeconfig files. This is only used
+        # if the provided CNI network config references it.
+        touch $MULTUS_TEMP_KUBECONFIG
+        chmod ${KUBECONFIG_MODE:-600} $MULTUS_TEMP_KUBECONFIG
+        # Write the kubeconfig to a temp file first.
+        timenow=$(date)
+        cat > $MULTUS_TEMP_KUBECONFIG <<EOF
+    # Kubeconfig file for Multus CNI plugin.
+    # Generated at ${timenow}
+    apiVersion: v1
+    kind: Config
+    clusters:
+    - name: local
+      cluster:
+        server: ${KUBERNETES_SERVICE_PROTOCOL:-https}://[${KUBERNETES_SERVICE_HOST}]:${KUBERNETES_SERVICE_PORT}
+        $TLS_CFG
+    users:
+    - name: multus
+      user:
+        token: "${SERVICEACCOUNT_TOKEN}"
+    contexts:
+    - name: multus-context
+      context:
+        cluster: local
+        user: multus
+    current-context: multus-context
+    EOF
+
+        # Atomically move the temp kubeconfig to its permanent home.
+        mv -f $MULTUS_TEMP_KUBECONFIG $MULTUS_KUBECONFIG
+
+        # Keep track of the md5sum
+        LAST_SERVICEACCOUNT_MD5SUM=$(md5sum $SERVICE_ACCOUNT_TOKEN_PATH | awk '{print $1}')
+        LAST_KUBE_CA_FILE_MD5SUM=$(md5sum $KUBE_CA_FILE | awk '{print $1}')
+      else
+        warn "Doesn't look like we're running in a kubernetes environment (no serviceaccount token)"
+      fi
+    }
+
+    CNI_CONF_DIR="/host/etc/cni/net.d"
+    MULTUS_TEMP_CONFIG="/tmp/00-multus.conf"
+    touch $MULTUS_TEMP_CONFIG
+    MULTUS_TEMP_KUBECONFIG="/tmp/multus.kubeconfig"
+    mkdir -p $CNI_CONF_DIR/multus.d
+    MULTUS_KUBECONFIG=$CNI_CONF_DIR/multus.d/multus.kubeconfig
+    SERVICE_ACCOUNT_PATH=/var/run/secrets/kubernetes.io/serviceaccount
+    SERVICE_ACCOUNT_TOKEN_PATH=$SERVICE_ACCOUNT_PATH/token
+    KUBE_CA_FILE=${KUBE_CA_FILE:-$SERVICE_ACCOUNT_PATH/ca.crt}
+
+    LAST_SERVICEACCOUNT_MD5SUM=""
+    LAST_KUBE_CA_FILE_MD5SUM=""
+
+    cat >  $MULTUS_TEMP_CONFIG << EOF
     {
       "cniVersion": "0.3.1",
       "name": "multus-cni-network",
       "type": "multus",
       "confDir": "/etc/cni/net.d/" ,
-      "logLevel": "{{ .Values.multus.multusCNI.log.logLevel }}",
-      "logFile": "{{ .Values.multus.multusCNI.log.logFile }}",
+      "logLevel": "debug",
+      "logFile": "/var/log/multus.log",
       "capabilities": {
         "portMappings": true,
         "bandwidth": true
       },
       "namespaceIsolation": false,
-      "clusterNetwork": "{{ .Values.multus.multusCNI.defaultCniCRName }}",
+      "clusterNetwork": "$MULTUS_CLUSTER_NETWORK",
       "defaultNetworks": [],
-      "multusNamespace": "{{ .Release.Namespace }}",
+      "multusNamespace": "$MULTUS_NAMESPACE",
       "systemNamespaces": [],
       "kubeconfig": "/etc/cni/net.d/multus.d/multus.kubeconfig"
     }
-{{- end }}
+    EOF
+
+    if [ -z "${MULTUS_CLUSTER_NETWORK}" ]; then
+    log "ENV MULTUS_CLUSTER_NETWORK is empty, Detecting default cni in the ${CNI_CONF_DIR}"
+        DEFAULT_CNI_FILEPATH=$(ls -l ${CNI_CONF_DIR} | grep ^- | grep -v -i multus | awk '{print $9}' | grep -E '(*\.conf|*\.conflist|*\.json)'  | head -n 1)
+        if [ -z "$DEFAULT_CNI_FILEPATH" ] ; then
+          error "No default cni file found in ${CNI_CONF_DIR}, please install your default cni in the cluster first" && exit 1
+        fi
+
+        log "Found the default-cni file: ${DEFAULT_CNI_FILEPATH}"
+        log "cat /host/etc/cni/net.d/${DEFAULT_CNI_FILEPATH}:"
+        cat /host/etc/cni/net.d/${DEFAULT_CNI_FILEPATH}
+
+        echo ""
+        DEFAULT_CNI_NAME=$(grep '"name":' ${CNI_CONF_DIR}/${DEFAULT_CNI_FILEPATH} | awk '{print $2}' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'  | tr -d ',' | tr -d '"')
+        if [ -z "$DEFAULT_CNI_NAME" ] ; then
+          error "The name fleid shouldn't be empty, please check the default cni: ${DEFAULT_CNI_FILEPATH}" && exit 1
+        fi
+
+        log "Updating the clusterNetwork of the multus-cni config to $DEFAULT_CNI_NAME"
+        sed -i "s?\"clusterNetwork\": \"\"?\"clusterNetwork\": \"${DEFAULT_CNI_NAME}\"?g" /tmp/00-multus.conf
+    else
+      log "User set multus ClusterNetwork: $MULTUS_CLUSTER_NETWORK"
+    fi
+
+    generateKubeConfig
+    log "multus kubeconfig is generated."
+
+    cp $MULTUS_TEMP_CONFIG /host/etc/cni/net.d
+    log "multus config file ${MULTUS_TEMP_CONFIG} is copied to ${CNI_CONF_DIR}."
+    log "cat ${CNI_CONF_DIR}/00-multus.conf"
+    cat ${CNI_CONF_DIR}/00-multus.conf
+
+    log "Entering watch loop..."
+    while true; do
+
+      # Check the md5sum of the service account token and ca.
+      svcaccountsum=$(md5sum $SERVICE_ACCOUNT_TOKEN_PATH | awk '{print $1}')
+      casum=$(md5sum $KUBE_CA_FILE | awk '{print $1}')
+      if [ "$svcaccountsum" != "$LAST_SERVICEACCOUNT_MD5SUM" ] || [ "$casum" != "$LAST_KUBE_CA_FILE_MD5SUM" ]; then
+        log "Detected service account or CA file change, regenerating kubeconfig..."
+        generateKubeConfig
+      fi
+
+      # todo: watch the default cni file is changed.
+      sleep 10
+    done
+{{- end }}

charts/spiderpool/templates/daemonset.yaml

@@ -100,6 +100,22 @@ spec:
             - name: cni-bin-path
               mountPath: /host/opt/cni/bin
         {{- end }}
+        {{- if .Values.multus.multusCNI.install }}
+        - name: install-multus-binary
+          image: {{ include "spiderpool.multus.image" . | quote }}
+          imagePullPolicy: IfNotPresent
+          command:
+          - /install_multus
+          args:
+          - --type
+          - thin
+          securityContext:
+            privileged: true
+          volumeMounts:
+          - mountPath: /host/opt/cni/bin
+            mountPropagation: Bidirectional
+            name: cni-bin-path
+        {{- end }}
       containers:
       - name: {{ .Values.spiderpoolAgent.name | trunc 63 | trimSuffix "-" }}
         image: {{ include "spiderpool.spiderpoolAgent.image" . | quote }}
@@ -223,47 +239,53 @@ spec:
         {{- end }}
      {{- if .Values.multus.multusCNI.install }}
       - name: multus-cni
-        imagePullPolicy: {{ .Values.multus.multusCNI.image.pullPolicy }}
-        image: {{ include "spiderpool.multus.image" . | quote }}
+        image: {{ include "spiderpool.spiderpoolAgent.image" . | quote }}
+        imagePullPolicy: {{ .Values.spiderpoolAgent.image.pullPolicy }}
         command:
-          - "/bin/sh"
-          - "-c"
-          - |
-            ITEM="multus"
-            rm -f /host/opt/cni/bin/${ITEM}.old || true
-            ( [ -f "/host/opt/cni/bin/${ITEM}" ] && mv /host/opt/cni/bin/${ITEM} /host/opt/cni/bin/${ITEM}.old ) || true
-            cp /usr/src/multus-cni/bin/${ITEM} /host/opt/cni/bin/${ITEM}
-            rm -f /host/opt/cni/bin/${ITEM}.old &>/dev/null  || true
-            ./entrypoint.sh --multus-conf-file=/tmp/multus-conf/00-multus.conf \
-              --cni-version=0.3.1
+        - "/home/entrypoint.sh"
         securityContext:
           privileged: true
-        {{- if .Values.multus.multusCNI.uninstall }}        
+        env:
+        - name: MULTUS_CLUSTER_NETWORK
+          valueFrom:
+            configMapKeyRef:
+              key: clusterNetwork
+              name: spiderpool-conf
+        - name: MULTUS_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        resources:
+          limits:
+            cpu: 100m
+            memory: 50Mi
+          requests:
+            cpu: 100m
+            memory: 50Mi
+        {{- if .Values.multus.multusCNI.uninstall }}
         lifecycle:
           preStop:
             exec:
               command:
-              - "/bin/sh"
-              - "-c"
-              - | 
-                rm -f /host/opt/cni/bin/multus || true
-                rm -rf /host/etc/cni/net.d/multus.d || true
-                rm -f /host/etc/cni/net.d/00-multus.conf || true
+                - "/bin/sh"
+                - "-c"
+                - |
+                  rm -f /host/opt/cni/bin/multus || true
+                  rm -rf /host/etc/cni/net.d/multus.d || true
+                  rm -f /host/etc/cni/net.d/00-multus.conf || true
         {{- end }}
         volumeMounts:
           - name: cni
             mountPath: /host/etc/cni/net.d
-          - name: cni-bin-path
-            mountPath: /host/opt/cni/bin
-            mountPropagation: Bidirectional
-          - name: multus-cfg
-            mountPath: /tmp/multus-conf
-          {{- if .Values.multus.multusCNI.extraVolumes }}
-          {{- include "tplvalues.render" ( dict "value" .Values.multus.multusCNI.extraVolumeMounts "context" $ ) | nindent 12 }}
-          {{- end }}
+          - mountPath: /home
+            name: multus-entrypoint
+        {{- if .Values.multus.multusCNI.extraVolumes }}
+        {{- include "tplvalues.render" ( dict "value" .Values.multus.multusCNI.extraVolumeMounts "context" $ ) | nindent 10 }}
+        {{- end }}
       {{- end }}
       volumes:
-        # To read the configuration from the config map
+      # To read the configuration from the config map
       - name: config-path
         configMap:
           defaultMode: 0400
@@ -282,16 +304,17 @@ spec:
       - name: cni
         hostPath:
           path: /etc/cni/net.d
-      - name: multus-cfg
+      - name: multus-entrypoint
         configMap:
-          name: {{ .Values.multus.multusCNI.name | trunc 63 | trimSuffix "-" }}
+          name: {{ .Values.multus.multusCNI.name | trunc 63 | trimSuffix "-" }}-entrypoint
+          defaultMode: 511
           items:
-            - key: cni-conf.json
-              path: 00-multus.conf
-      {{- end }}
-      {{- if .Values.spiderpoolAgent.extraVolumeMounts }}
-      {{- include "tplvalues.render" ( dict "value" .Values.spiderpoolAgent.extraVolumeMounts "context" $ ) | nindent 6 }}
-      {{- end }}
-      {{- if .Values.multus.multusCNI.extraVolumeMounts }}
-      {{- include "tplvalues.render" ( dict "value" .Values.multus.multusCNI.extraVolumeMounts "context" $ ) | nindent 8 }}
+            - key: entrypoint.sh
+              path: entrypoint.sh
       {{- end }}
+    {{- if .Values.spiderpoolAgent.extraVolumeMounts }}
+    {{- include "tplvalues.render" ( dict "value" .Values.spiderpoolAgent.extraVolumeMounts "context" $ ) | nindent 6 }}
+    {{- end }}
+    {{- if .Values.multus.multusCNI.extraVolumeMounts }}
+    {{- include "tplvalues.render" ( dict "value" .Values.multus.multusCNI.extraVolumeMounts "context" $ ) | nindent 6 }}
+    {{- end }}

charts/spiderpool/templates/pod.yaml

@@ -80,19 +80,13 @@ spec:
         {{- end }}
         - name: SPIDERPOOL_INIT_ENABLE_MULTUS_CONFIG
           value: {{ .Values.multus.enableMultusConfig | quote }}
-        - name: SPIDERPOOL_INIT_INSTALL_MULTUS
-          value: {{ .Values.multus.multusCNI.install | quote }}
         - name: SPIDERPOOL_INIT_DEFAULT_CNI_NAME
           value: {{ .Values.multus.multusCNI.defaultCniCRName | quote }}
         - name: SPIDERPOOL_INIT_DEFAULT_CNI_NAMESPACE
           value: {{ .Release.Namespace | quote }}
-        - name: SPIDERPOOL_INIT_MULTUS_CONFIGMAP
-          value: {{ .Values.multus.multusCNI.name | trunc 63 | trimSuffix "-" | quote }}
         {{- if eq .Values.multus.multusCNI.defaultCniCRName "" }}
         - name: SPIDERPOOL_INIT_DEFAULT_CNI_DIR
           value: {{ .Values.global.cniConfHostPath | quote }}
-        - name: SPIDERPOOL_INIT_READINESS_FILE
-          value: "/etc/spiderpool/ready"
         volumeMounts:
           - name: cni
             mountPath: {{ .Values.global.cniConfHostPath }}

charts/spiderpool/values.yaml

@@ -227,8 +227,7 @@ multus:
       digest: ""
 
       ## @param multus.multusCNI.image.tag the multus-CNI image tag
-      tag: v3.9.3
-      # tag: v4.0.2-thick
+      tag: v4.1.4
 
       ## @param multus.multusCNI.image.imagePullSecrets the multus-CNI image imagePullSecrets
       imagePullSecrets: []

cmd/spiderpool-controller/cmd/crd_manager.go

@@ -94,7 +94,7 @@ type _webhookHealthCheck struct{}
 func (*_webhookHealthCheck) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
 	if request.Method == http.MethodGet {
 		writer.WriteHeader(http.StatusOK)
-		logger.Info("Webhook health check successful")
+		logger.Debug("Webhook health check successful")
 	}
 }
 

cmd/spiderpool-init/cmd/config.go

@@ -91,7 +91,6 @@ type InitDefaultConfig struct {
 
 	// multuscniconfig
 	enableMultusConfig  bool
-	installMultusCNI    bool
 	DefaultCNIDir       string
 	DefaultCNIName      string
 	DefaultCNINamespace string
@@ -281,12 +280,6 @@ func parseENVAsDefault() InitDefaultConfig {
 		logger.Sugar().Fatalf("ENV %s: %s invalid: %v", ENVEnableMultusConfig, enableMultusConfig, err)
 	}
 
-	installMultusCNI := strings.ReplaceAll(os.Getenv(ENVInstallMultusCNI), "\"", "")
-	config.installMultusCNI, err = strconv.ParseBool(installMultusCNI)
-	if err != nil {
-		logger.Sugar().Fatalf("ENV %s: %s invalid: %v", ENVInstallMultusCNI, installMultusCNI, err)
-	}
-
 	config.DefaultCNIDir = strings.ReplaceAll(os.Getenv(ENVDefaultCNIDir), "\"", "")
 	if config.DefaultCNIDir != "" {
 		_, err = os.ReadDir(config.DefaultCNIDir)