feat: Bump modulectl version in E2E tests (kyma-project#2017)

* Bump modulectl version

* add new required fields for the moduleconfig

* remove the ModuleTemplate name overwrite in E2E tests

* rename module-config-file to config-file

* Add Https server

* Debugging ModuleTemplate creation

* Remove pattern and min length validation from ModuleTemplate spec.channel to allow nullable channels

* fix test

* Add channel to old ModuleTemplates

* Write channel for old ModuleTemplates and remove it from ModuleConfig

* fix status decoupling with statefulset e2e test

* remove unneeded make build-manifests

* re add new validation pattern

* re add test

* try sudo -S

* try sudo -s

* try export SSL_CERT_FILE

* try export SSL_CERT_FILE

* try mitmproxy

* try mitmproxy

* try custom cert directory

* try custom ca cert

* try custom ca cert

* fix comments

* use default cr with localhost

* use golang server

* fix script

* fix linting

.github/actions/deploy-template-operator-with-modulereleasemeta/action.yml

@@ -3,6 +3,10 @@ description: Deploys a test-specific template-operator and corresponding ModuleR
 runs:
   using: composite
   steps:
+    - name: Setup HTTPS server for the manifest file
+      uses: ./lifecycle-manager/.github/actions/setup-https-server
+      with:
+        directory_name: ../template-operator
     - name: Create Template Operator Module and apply ModuleTemplate and ModuleReleaseMeta
       working-directory: template-operator
       if: ${{ matrix.e2e-test == 'kyma-metrics' ||
@@ -19,10 +23,8 @@ runs:
         }}
       shell: bash
       run: |
-        make build-manifests
-        modulectl create --module-config-file ./module-config.yaml --registry http://localhost:5111 --insecure
+        modulectl create --config-file ./module-config.yaml --registry http://localhost:5111 --insecure
         sed -i 's/localhost:5111/k3d-kcp-registry.localhost:5000/g' ./template.yaml
-        sed -i 's/template-operator-regular/template-operator-1.0.1/g' ./template.yaml
         kubectl get crds
         kubectl apply -f template.yaml
         kubectl apply -f module-release-meta.yaml
@@ -38,7 +40,7 @@ runs:
         sed -i 's/template-operator-fast/template-operator-2.4.2-e2e-test/g' tests/moduletemplates/moduletemplate_template_operator_v2_fast.yaml
         sed -i 's/template-operator-regular/template-operator-1.1.1-e2e-test/g' tests/moduletemplates/moduletemplate_template_operator_v1_regular.yaml
         sed -i 's/template-operator-regular/template-operator-2.4.2-e2e-test/g' tests/moduletemplates/moduletemplate_template_operator_v2_regular_new_version.yaml
-
+        
         kubectl apply -f tests/moduletemplates/moduletemplate_template_operator_v2_fast.yaml
         kubectl apply -f tests/moduletemplates/moduletemplate_template_operator_v1_regular.yaml
 
@@ -80,6 +82,7 @@ runs:
       shell: bash
       run: |
         sed -i 's/template-operator-fast/template-operator-2.4.2-e2e-test/g' tests/moduletemplates/moduletemplate_template_operator_v2_fast.yaml
+        
         kubectl apply -f tests/moduletemplates/moduletemplate_template_operator_v2_fast.yaml
         cat <<EOF > mrm.yaml
         apiVersion: operator.kyma-project.io/v1beta2
@@ -114,16 +117,17 @@ runs:
         popd
         make build-manifests
         echo "name: kyma-project.io/module/template-operator
-        channel: regular
         version: 1.0.1
         security: sec-scanners-config.yaml
-        defaultCR: ./config/samples/default-sample-cr.yaml
-        manifest: template-operator.yaml
-        annotations:
-          operator.kyma-project.io/doc-url: https://kyma-project.io" >> warning-module-config.yaml
-        modulectl create --module-config-file ./warning-module-config.yaml --registry http://localhost:5111 --insecure
+        defaultCR: https://localhost:8080//config/samples/default-sample-cr.yaml
+        manifest: https://localhost:8080/template-operator.yaml
+        repository: https://github.com/kyma-project/template-operator
+        documentation: https://github.com/kyma-project/template-operator/blob/main/README.md
+        icons:
+        - name: module-icon
+          link: https://github.com/kyma-project/template-operator/blob/main/docs/assets/logo.png" >> warning-module-config.yaml
+        modulectl create --config-file ./warning-module-config.yaml --registry http://localhost:5111 --insecure
         sed -i 's/localhost:5111/k3d-kcp-registry.localhost:5000/g' ./template.yaml
-        sed -i 's/template-operator-regular/template-operator-1.0.1/g' ./template.yaml
         kubectl get crds
         kubectl apply -f template.yaml
         kubectl apply -f module-release-meta.yaml
@@ -145,16 +149,17 @@ runs:
         popd
         make build-statefulset-manifests
         echo "name: kyma-project.io/module/template-operator
-        channel: regular
         version: 1.0.1
         security: sec-scanners-config.yaml
-        defaultCR: ./config/samples/default-sample-cr.yaml
-        manifest: template-operator.yaml
-        annotations:
-          operator.kyma-project.io/doc-url: https://kyma-project.io" >> warning-module-config.yaml
-        modulectl create --module-config-file ./warning-module-config.yaml --registry http://localhost:5111 --insecure
+        defaultCR: https://localhost:8080//config/samples/default-sample-cr.yaml
+        manifest: https://localhost:8080/template-operator.yaml
+        repository: https://github.com/kyma-project/template-operator
+        documentation: https://github.com/kyma-project/template-operator/blob/main/README.md
+        icons:
+        - name: module-icon
+          link: https://github.com/kyma-project/template-operator/blob/main/docs/assets/logo.png" >> warning-module-config.yaml
+        modulectl create --config-file ./warning-module-config.yaml --registry http://localhost:5111 --insecure
         sed -i 's/localhost:5111/k3d-kcp-registry.localhost:5000/g' ./template.yaml
-        sed -i 's/template-operator-regular/template-operator-1.0.1/g' ./template.yaml
         kubectl get crds
         kubectl apply -f template.yaml
         kubectl apply -f module-release-meta.yaml
@@ -165,13 +170,15 @@ runs:
       shell: bash
       run: |
         echo "name: kyma-project.io/module/template-operator-misconfigured
-        channel: regular
         version: 1.1.1
-        manifest: template-operator.yaml
+        manifest: https://localhost:8080/template-operator.yaml
         security: sec-scanners-config.yaml
-        defaultCR: ./config/samples/default-sample-cr.yaml
-        annotations:
-          operator.kyma-project.io/doc-url: https://kyma-project.io" >> misconfigured-module-config.yaml
+        defaultCR: https://localhost:8080//config/samples/default-sample-cr.yaml
+        repository: https://github.com/kyma-project/template-operator
+        documentation: https://github.com/kyma-project/template-operator/blob/main/README.md
+        icons:
+        - name: module-icon
+          link: https://github.com/kyma-project/template-operator/blob/main/docs/assets/logo.png" >> misconfigured-module-config.yaml
         
         cat <<EOF > mrm.yaml
         apiVersion: operator.kyma-project.io/v1beta2
@@ -201,9 +208,8 @@ runs:
         kustomize edit add patch --path image_patch.yaml --kind Deployment
         popd
         make build-manifests
-        modulectl create --module-config-file ./misconfigured-module-config.yaml --registry http://localhost:5111 --insecure
+        modulectl create --config-file ./misconfigured-module-config.yaml --registry http://localhost:5111 --insecure
         sed -i 's/localhost:5111/k3d-kcp-registry.localhost:5000/g' ./template.yaml
-        sed -i 's/template-operator-misconfigured-regular/template-operator-misconfigured-1.1.1/g' ./template.yaml
         kubectl get crds
         kubectl apply -f template.yaml
     - name: Create Template Operator Module with StatefulSet, with non-working image and apply
@@ -220,9 +226,8 @@ runs:
         kustomize edit add patch --path image_patch.yaml --kind StatefulSet
         popd
         make build-statefulset-manifests
-        modulectl create --module-config-file ./misconfigured-module-config.yaml --registry http://localhost:5111 --insecure
+        modulectl create --config-file ./misconfigured-module-config.yaml --registry http://localhost:5111 --insecure
         sed -i 's/localhost:5111/k3d-kcp-registry.localhost:5000/g' ./template.yaml
-        sed -i 's/template-operator-misconfigured-regular/template-operator-misconfigured-1.1.1/g' ./template.yaml
         kubectl get crds
         kubectl apply -f template.yaml
 
@@ -233,15 +238,16 @@ runs:
       run: |
         make build-manifests
         echo "name: kyma-project.io/module/template-operator
-        channel: regular
         version: 1.0.0
-        manifest: template-operator.yaml
+        manifest: https://localhost:8080/template-operator.yaml
         security: sec-scanners-config.yaml
-        annotations:
-          operator.kyma-project.io/doc-url: https://kyma-project.io" >> module-config-no-cr.yaml
-        modulectl create --module-config-file ./module-config-no-cr.yaml --registry http://localhost:5111 --insecure
+        repository: https://github.com/kyma-project/template-operator
+        documentation: https://github.com/kyma-project/template-operator/blob/main/README.md
+        icons:
+        - name: module-icon
+          link: https://github.com/kyma-project/template-operator/blob/main/docs/assets/logo.png" >> module-config-no-cr.yaml
+        modulectl create --config-file ./module-config-no-cr.yaml --registry http://localhost:5111 --insecure
         sed -i 's/localhost:5111/k3d-kcp-registry.localhost:5000/g' ./template.yaml
-        sed -i 's/template-operator-regular/template-operator-1.0.0/g' ./template.yaml
         kubectl get crds
         kubectl apply -f template.yaml
         cat <<EOF > mrm.yaml
@@ -286,8 +292,8 @@ runs:
       if: ${{ matrix.e2e-test == 'modulereleasemeta-with-obsolete-moduletemplate' }}
       shell: bash
       run: |
-        make build-manifests
-        modulectl create --module-config-file ./module-config.yaml --registry http://localhost:5111 --insecure
+        modulectl create --config-file ./module-config.yaml --registry http://localhost:5111 --insecure
         sed -i 's/localhost:5111/k3d-kcp-registry.localhost:5000/g' ./template.yaml
+        sed -i 's/template-operator-1.0.1/template-operator-regular/g' ./template.yaml
         kubectl apply -f template.yaml
         kubectl apply -f module-release-meta.yaml

.github/actions/deploy-template-operator/action.yml

@@ -3,6 +3,10 @@ description: Deploys a test-specific template-operator.
 runs:
   using: composite
   steps:
+    - name: Setup HTTPS server for the manifest file
+      uses: ./lifecycle-manager/.github/actions/setup-https-server
+      with:
+        directory_name: ../template-operator
     - name: Create Template Operator Module and apply
       working-directory: template-operator
       if: ${{ matrix.e2e-test == 'kyma-metrics' ||
@@ -18,9 +22,9 @@ runs:
         }}
       shell: bash
       run: |
-        make build-manifests
-        modulectl create --module-config-file ./module-config.yaml --registry http://localhost:5111 --insecure
+        modulectl create --config-file ./module-config.yaml --registry http://localhost:5111 --insecure
         sed -i 's/localhost:5111/k3d-kcp-registry.localhost:5000/g' ./template.yaml
+        sed -i '/^spec:/a\  channel: regular\n' ./template.yaml
         kubectl get crds
         kubectl apply -f template.yaml
     - name: Create Template Operator Module for regular and fast channels
@@ -73,14 +77,17 @@ runs:
         popd
         make build-manifests
         echo "name: kyma-project.io/module/template-operator
-        channel: regular
         version: 1.0.1
         security: sec-scanners-config.yaml
-        defaultCR: ./config/samples/default-sample-cr.yaml
-        manifest: template-operator.yaml
-        annotations:
-          operator.kyma-project.io/doc-url: https://kyma-project.io" >> warning-module-config.yaml
-        modulectl create --module-config-file ./warning-module-config.yaml --registry http://localhost:5111 --insecure
+        defaultCR: https://github.com/kyma-project/template-operator/releases/download/1.0.1/default-sample-cr.yaml
+        manifest: https://localhost:8080/template-operator.yaml
+        repository: https://github.com/kyma-project/template-operator
+        documentation: https://github.com/kyma-project/template-operator/blob/main/README.md
+        icons:
+        - name: module-icon
+          link: https://github.com/kyma-project/template-operator/blob/main/docs/assets/logo.png" >> warning-module-config.yaml
+        modulectl create --config-file ./warning-module-config.yaml --registry http://localhost:5111 --insecure
+        sed -i '/^spec:/a\  channel: regular\n' ./template.yaml
         sed -i 's/localhost:5111/k3d-kcp-registry.localhost:5000/g' ./template.yaml
         kubectl get crds
         kubectl apply -f template.yaml
@@ -102,14 +109,17 @@ runs:
         popd
         make build-statefulset-manifests
         echo "name: kyma-project.io/module/template-operator
-        channel: regular
         version: 1.0.1
         security: sec-scanners-config.yaml
-        defaultCR: ./config/samples/default-sample-cr.yaml
-        manifest: template-operator.yaml
-        annotations:
-          operator.kyma-project.io/doc-url: https://kyma-project.io" >> warning-module-config.yaml
-        modulectl create --module-config-file ./warning-module-config.yaml --registry http://localhost:5111 --insecure
+        defaultCR: https://github.com/kyma-project/template-operator/releases/download/1.0.1/default-sample-cr.yaml
+        manifest: https://localhost:8080/template-operator.yaml
+        repository: https://github.com/kyma-project/template-operator
+        documentation: https://github.com/kyma-project/template-operator/blob/main/README.md
+        icons:
+        - name: module-icon
+          link: https://github.com/kyma-project/template-operator/blob/main/docs/assets/logo.png" >> warning-module-config.yaml
+        modulectl create --config-file ./warning-module-config.yaml --registry http://localhost:5111 --insecure
+        sed -i '/^spec:/a\  channel: regular\n' ./template.yaml
         sed -i 's/localhost:5111/k3d-kcp-registry.localhost:5000/g' ./template.yaml
         kubectl get crds
         kubectl apply -f template.yaml
@@ -120,13 +130,15 @@ runs:
       shell: bash
       run: |
         echo "name: kyma-project.io/module/template-operator-misconfigured
-        channel: regular
         version: 1.1.1
-        manifest: template-operator.yaml
+        manifest: https://localhost:8080/template-operator.yaml
         security: sec-scanners-config.yaml
-        defaultCR: ./config/samples/default-sample-cr.yaml
-        annotations:
-          operator.kyma-project.io/doc-url: https://kyma-project.io" >> misconfigured-module-config.yaml
+        defaultCR: https://github.com/kyma-project/template-operator/releases/download/1.0.1/default-sample-cr.yaml
+        repository: https://github.com/kyma-project/template-operator
+        documentation: https://github.com/kyma-project/template-operator/blob/main/README.md
+        icons:
+        - name: module-icon
+          link: https://github.com/kyma-project/template-operator/blob/main/docs/assets/logo.png" >> misconfigured-module-config.yaml
     - name: Create Template Operator Module with Deployment, with non-working image and apply
       working-directory: template-operator
       if: ${{ matrix.e2e-test == 'module-status-decoupling-with-deployment'}}
@@ -141,7 +153,8 @@ runs:
         kustomize edit add patch --path image_patch.yaml --kind Deployment
         popd
         make build-manifests
-        modulectl create --module-config-file ./misconfigured-module-config.yaml --registry http://localhost:5111 --insecure
+        modulectl create --config-file ./misconfigured-module-config.yaml --registry http://localhost:5111 --insecure
+        sed -i '/^spec:/a\  channel: regular\n' ./template.yaml
         sed -i 's/localhost:5111/k3d-kcp-registry.localhost:5000/g' ./template.yaml
         kubectl get crds
         kubectl apply -f template.yaml
@@ -159,7 +172,8 @@ runs:
         kustomize edit add patch --path image_patch.yaml --kind StatefulSet
         popd
         make build-statefulset-manifests
-        modulectl create --module-config-file ./misconfigured-module-config.yaml --registry http://localhost:5111 --insecure
+        modulectl create --config-file ./misconfigured-module-config.yaml --registry http://localhost:5111 --insecure
+        sed -i '/^spec:/a\  channel: regular\n' ./template.yaml
         sed -i 's/localhost:5111/k3d-kcp-registry.localhost:5000/g' ./template.yaml
         kubectl get crds
         kubectl apply -f template.yaml
@@ -170,13 +184,16 @@ runs:
       run: |
         make build-manifests
         echo "name: kyma-project.io/module/template-operator
-        channel: regular
         version: 1.0.0
-        manifest: template-operator.yaml
+        manifest: https://localhost:8080/template-operator.yaml
         security: sec-scanners-config.yaml
-        annotations:
-          operator.kyma-project.io/doc-url: https://kyma-project.io" >> module-config-no-cr.yaml
-        modulectl create --module-config-file ./module-config-no-cr.yaml --registry http://localhost:5111 --insecure
+        repository: https://github.com/kyma-project/template-operator
+        documentation: https://github.com/kyma-project/template-operator/blob/main/README.md
+        icons:
+        - name: module-icon
+          link: https://github.com/kyma-project/template-operator/blob/main/docs/assets/logo.png" >> module-config-no-cr.yaml
+        modulectl create --config-file ./module-config-no-cr.yaml --registry http://localhost:5111 --insecure
+        sed -i '/^spec:/a\  channel: regular\n' ./template.yaml
         sed -i 's/localhost:5111/k3d-kcp-registry.localhost:5000/g' ./template.yaml
         kubectl get crds
         kubectl apply -f template.yaml

.github/actions/get-configuration/action.yml

@@ -32,7 +32,7 @@ runs:
         echo "k8s_version=${{ github.event.inputs.k8s_version || '1.30.3' }}" >> $GITHUB_OUTPUT
         echo "istio_version=1.20.3" >> $GITHUB_OUTPUT
         echo "k3d_version=5.7.4" >> $GITHUB_OUTPUT
-        echo "modulectl_version=1.0.0" >> $GITHUB_OUTPUT
+        echo "modulectl_version=1.1.1" >> $GITHUB_OUTPUT
         echo "cert_manager_version=1.15.0" >> $GITHUB_OUTPUT
         echo "klm_version_tag=PR-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
         echo "klm_image_repo=dev" >> $GITHUB_OUTPUT

.github/actions/install-modulectl/action.yml

@@ -13,4 +13,4 @@ runs:
         wget https://github.com/kyma-project/modulectl/releases/download/${{ inputs.modulectl_version }}/modulectl-linux -O modulectl-linux
         chmod +x modulectl-linux
         mv modulectl-linux /usr/local/bin/modulectl
-        echo "PATH=/usr/local/bin/modulectl" >> $GITHUB_OUTPUT
+        echo "PATH=/usr/local/bin/modulectl" >> $GITHUB_OUTPUT

.github/actions/setup-https-server/action.yml

@@ -0,0 +1,14 @@
+name: Setup HTTPS server
+description: Setup an HTTPS server to serve the template operator manifest file.
+inputs:
+  directory_name:
+    description: The name of the directory which contains the manifest file to serve.
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Serve HTTPs server
+      working-directory: ./lifecycle-manager
+      shell: bash
+      run: |
+        ./scripts/tests/setup_https_server.sh ${{ inputs.directory_name }}

docs/contributor/04-local-test-setup.md

@@ -105,7 +105,7 @@ This setup is deployed with the following security features enabled:
    make build-manifests
    
    # create the a ModuleTemplate CR and save it to the template.yaml file
-   modulectl create --module-config-file ./module-config.yaml --registry http://k3d-registry.localhost:5111 --insecure 
+   modulectl create --config-file ./module-config.yaml --registry http://k3d-registry.localhost:5111 --insecure 
    ```
 
 6. Verify images pushed to the local registry:

scripts/tests/https_server.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"crypto/tls"
+	"flag"
+	"log"
+	"net/http"
+	"time"
+)
+
+const headerTimeout = 10 * time.Second
+
+func main() {
+	directory := flag.String("dir", ".", "Directory to serve files from")
+	certFile := flag.String("certfile", "cert.crt", "SSL certificate file")
+	keyFile := flag.String("keyfile", "key.crt", "SSL key file")
+	port := flag.String("port", "8080", "Port to run the server on")
+	flag.Parse()
+
+	mux := http.NewServeMux()
+	fs := http.FileServer(http.Dir(*directory))
+	mux.Handle("/", fs)
+
+	addr := ":" + *port
+	httpsServer := &http.Server{
+		Addr:    addr,
+		Handler: mux,
+		TLSConfig: &tls.Config{
+			MinVersion: tls.VersionTLS12,
+		},
+		ReadHeaderTimeout: headerTimeout,
+	}
+
+	err := httpsServer.ListenAndServeTLS(*certFile, *keyFile)
+	if err != nil {
+		log.Fatal("failed to setup http server:", err)
+	}
+}