Uses viper for reading configurations for controller/watcher

.github/workflows/e2e-config.yaml

@@ -0,0 +1,30 @@
+apiVersion: v1
+data:
+  config.yaml: |
+    application-name: "Pipelines as Code CI"
+    secret-auto-create: "true"
+    secret-github-app-token-scoped: "true"
+    secret-github-app-scope-extra-repos: ""
+    hub-url: "https://api.hub.tekton.dev/v1"
+    hub-catalog-name: "tekton"
+    remote-tasks: "true"
+    tekton-dashboard-url: ""
+    error-log-snippet: "true"
+    error-detection-from-container-logs: "true"
+    error-detection-max-number-of-lines: "50"
+    error-detection-simple-regexp: |
+      ^(?P<filename>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+)?([ ]*)?(?P<error>.*)
+    bitbucket-cloud-check-source-ip: "false"
+    bitbucket-cloud-additional-source-ip: ""
+    max-keep-run-upper-limit: ""
+    default-max-keep-runs: ""
+    auto-configure-new-github-repo: "false"
+    auto-configure-repo-namespace-template: ""
+    remember-ok-to-test: "true"
+kind: ConfigMap
+metadata:
+  name: pipelines-as-code
+  namespace: pipelines-as-code
+  labels:
+    app.kubernetes.io/version: "devel"
+    app.kubernetes.io/part-of: pipelines-as-code

.github/workflows/kind-e2e-tests.yaml

@@ -66,11 +66,11 @@ jobs:
             --from-literal github-application-id=${{ secrets.APPLICATION_ID }} \
             --from-literal webhook.secret=${{ secrets.WEBHOOK_SECRET }}
           # Disable Bitbucket Cloud Source IP check, since we should be god here.
-          kubectl patch configmap -n pipelines-as-code -p "{\"data\":{\"bitbucket-cloud-check-source-ip\": \"false\"}}" \
-          --type merge pipelines-as-code
+          kubectl apply -f ./.github/workflows/e2e-config.yaml
 
           # restart controller
           kubectl -n pipelines-as-code delete pod -l app.kubernetes.io/name=controller
+          kubectl -n pipelines-as-code delete pod -l app.kubernetes.io/name=watcher
 
           # wait for controller to start
           i=0

cmd/pipelines-as-code-controller/main.go

@@ -26,7 +26,9 @@ func main() {
 	if err != nil {
 		log.Fatal("failed to init clients : ", err)
 	}
-
+	if err := run.InitConfig(ctx); err != nil {
+		log.Fatal("failed to init config : ", err)
+	}
 	kinteract, err := kubeinteraction.NewKubernetesInteraction(run)
 	if err != nil {
 		log.Fatal("failed to init kinit client : ", err)

config/302-pac-configmap.yaml

@@ -17,128 +17,129 @@
 
 apiVersion: v1
 data:
-  # The application name, you can customize this label. If using the Github App you will need to customize the label on the github app setting as well.
-  application-name: "Pipelines as Code CI"
-
-  # Whether to automatically create a secret with the token to be use by git-clone
-  secret-auto-create: "true"
-
-  # By default we only generate token scoped to the repository from where the
-  # payload come from.
-  # We do this because if the github apps is installed on an github organisation
-  #
-  # and there is a mix of public and private repositories in there
-  # where some users on that org does not have access.
-  #
-  # If you trust every users on your organisations to access any repos there or
-  # not planning to install your github application globally on a Github Organisation
-  # then you can safely set this option to false.
-  secret-github-app-token-scoped: "true"
-
-  # If you don't want to completely disable the scoping of the token, but still
-  # wants some other repos (on the same installation id) available from the
-  # token, then you can add an extra owner/repo here.
-  #
-  # You can have multiple owner/repositories separated by commas:
-  # i.e: "owner/private-repo1, org/repo2"
-  secret-github-app-scope-extra-repos: ""
-
-  # Tekton HUB API urls
-  hub-url: "https://api.hub.tekton.dev/v1"
-
-  # Tekton HUB catalog name
-  hub-catalog-name: "tekton"
-
-  # Additional Hub Catalogs is supported, for example:
-  #
-  # catalog-1-id: anotherhub
-  # catalog-1-name: tekton
-  # catalog-1-url: https://api.other.com/v1
-  #
-  # this configuration will have a new catalog named anotherhub on  https://api.other.com/v1 endpoint and catalog name tekton
-  # to be used by a user in their templates like this:
-  # pipelinesascode.tekton.dev/task: "anotherhub://task"
-  #
-  # Increase the number of the catalog to add more of them
-
-  # Allow fetching remote tasks
-  remote-tasks: "true"
-
-  # Using the URL of the Tekton dashboard, Pipelines-as-Code generates a URL to the
-  # PipelineRun on the Tekton dashboard
-  tekton-dashboard-url: ""
-
-  # Enable or disable the feature to show a log snippet of the failed task when there is
-  # an error in a Pipeline
-  #
-  # It will show the last 3 lines of the first container of the first task
-  # that has error in the pipeline.
-  #
-  # you may want to disable this if you think your pipeline may leak some value
-  error-log-snippet: "true"
-
-  # Enable or disable the inspection of container logs to detect error message
-  # and expose them as annotations on Pull Request. Only Github apps is supported
-  error-detection-from-container-logs: "true"
-
-  # How many lines to grab from the container when inspecting the
-  # logs for error-detection. Increasing this value may increase the watcher
-  # memory usage.  Use -1 for unlimited lines.
-  error-detection-max-number-of-lines: "50"
-
-  # The default regexp used when we use the simple error detection
-  error-detection-simple-regexp: |
-    ^(?P<filename>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+)?([ ]*)?(?P<error>.*)
-
-  # Since public bitbucket doesn't have the concept of Secret, we need to be
-  # able to secure the request by querying https://ip-ranges.atlassian.com/,
-  # this only happen for public bitbucket (ie: when provider.url is not set in
-  # repository spec). If you want to override this, you need to bear in mind
-  # this could be a security issue, a malicious user can send a PR to your repo
-  # with a modification to your PipelineRun that would grab secrets, tunnel or
-  # others and then send a malicious webhook payload to the controller which
-  # look like a authorized owner has send the PR to run it..
-  bitbucket-cloud-check-source-ip: "true"
-
-  # Add extra IPS (ie: 127.0.0.1) or networks (127.0.0.0/16) separated by commas.
-  bitbucket-cloud-additional-source-ip: ""
-
-  # max-keep-run-upper-limit defines the upper limit for max-keep-run annotation
-  # value which a user can set on pipelineRun. the value set on annotation
-  # should be less than or equal to the upper limit otherwise the upper limit
-  # will be used while cleaning up
-  max-keep-run-upper-limit: ""
-
-  # if defined then applies to all pipelineRun who doesn't have max-keep-runs annotation
-  default-max-keep-runs: ""
-
-  # Whether to auto configure newly created repositories, this will create a new
-  # namespace and repository CR, supported only with GitHub App
-  auto-configure-new-github-repo: "false"
-
-  # add a template to generate name for namespace for your auto configured
-  # github repo supported fields are repo_owner, repo_name eg. if defined as
-  # `{{repo_owner}}-{{repo_name}}-ci`, then namespace generated for repository
-  # https://github.com/owner/repo will be `owner-repo-ci`
-  auto-configure-repo-namespace-template: ""
-
-  # Enable or disable the feature to rerun the CI if push event happens on
-  # a pull request
-  #
-  # By default it is true and CI will be re-run in case of push/amend on the
-  # pull request if ok-to-test is done once
-  #
-  # you may want to disable this if ok-to-test should be done on each iteration
-  remember-ok-to-test: "true"
-
-  # Configure a custom console here, the driver support custom parameters from
-  # Repo CR along a few other template variable, see documentation for more
-  # details
-  #
-  # custom-console-name: Console Name
-  # custom-console-url: https://url
-  # custom-console-url-pr-details: https://url/ns/{{ namespace }}/{{ pr }}
-  # custom-console-url-pr-tasklog: https://url/ns/{{ namespace }}/{{ pr }}/logs/{{ task }}
+  config.yaml: |
+    # The application name, you can customize this label. If using the Github App you will need to customize the label on the github app setting as well.
+    application-name: "Pipelines as Code CI"
+
+    # Whether to automatically create a secret with the token to be use by git-clone
+    secret-auto-create: "true"
+
+    # By default we only generate token scoped to the repository from where the
+    # payload come from.
+    # We do this because if the github apps is installed on an github organisation
+    #
+    # and there is a mix of public and private repositories in there
+    # where some users on that org does not have access.
+    #
+    # If you trust every users on your organisations to access any repos there or
+    # not planning to install your github application globally on a Github Organisation
+    # then you can safely set this option to false.
+    secret-github-app-token-scoped: "true"
+
+    # If you don't want to completely disable the scoping of the token, but still
+    # wants some other repos (on the same installation id) available from the
+    # token, then you can add an extra owner/repo here.
+    #
+    # You can have multiple owner/repositories separated by commas:
+    # i.e: "owner/private-repo1, org/repo2"
+    secret-github-app-scope-extra-repos: ""
+
+    # Tekton HUB API urls
+    hub-url: "https://api.hub.tekton.dev/v1"
+
+    # Tekton HUB catalog name
+    hub-catalog-name: "tekton"
+
+    # Additional Hub Catalogs is supported, for example:
+    #
+    # catalog-1-id: anotherhub
+    # catalog-1-name: tekton
+    # catalog-1-url: https://api.other.com/v1
+    #
+    # this configuration will have a new catalog named anotherhub on  https://api.other.com/v1 endpoint and catalog name tekton
+    # to be used by a user in their templates like this:
+    # pipelinesascode.tekton.dev/task: "anotherhub://task"
+    #
+    # Increase the number of the catalog to add more of them
+
+    # Allow fetching remote tasks
+    remote-tasks: "true"
+
+    # Using the URL of the Tekton dashboard, Pipelines-as-Code generates a URL to the
+    # PipelineRun on the Tekton dashboard
+    tekton-dashboard-url: ""
+
+    # Enable or disable the feature to show a log snippet of the failed task when there is
+    # an error in a Pipeline
+    #
+    # It will show the last 3 lines of the first container of the first task
+    # that has error in the pipeline.
+    #
+    # you may want to disable this if you think your pipeline may leak some value
+    error-log-snippet: "true"
+
+    # Enable or disable the inspection of container logs to detect error message
+    # and expose them as annotations on Pull Request. Only Github apps is supported
+    error-detection-from-container-logs: "true"
+
+    # How many lines to grab from the container when inspecting the
+    # logs for error-detection. Increasing this value may increase the watcher
+    # memory usage.  Use -1 for unlimited lines.
+    error-detection-max-number-of-lines: "50"
+
+    # The default regexp used when we use the simple error detection
+    error-detection-simple-regexp: |
+      ^(?P<filename>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+)?([ ]*)?(?P<error>.*)
+
+    # Since public bitbucket doesn't have the concept of Secret, we need to be
+    # able to secure the request by querying https://ip-ranges.atlassian.com/,
+    # this only happen for public bitbucket (ie: when provider.url is not set in
+    # repository spec). If you want to override this, you need to bear in mind
+    # this could be a security issue, a malicious user can send a PR to your repo
+    # with a modification to your PipelineRun that would grab secrets, tunnel or
+    # others and then send a malicious webhook payload to the controller which
+    # look like a authorized owner has send the PR to run it..
+    bitbucket-cloud-check-source-ip: "true"
+
+    # Add extra IPS (ie: 127.0.0.1) or networks (127.0.0.0/16) separated by commas.
+    bitbucket-cloud-additional-source-ip: ""
+
+    # max-keep-run-upper-limit defines the upper limit for max-keep-run annotation
+    # value which a user can set on pipelineRun. the value set on annotation
+    # should be less than or equal to the upper limit otherwise the upper limit
+    # will be used while cleaning up
+    max-keep-run-upper-limit: ""
+
+    # if defined then applies to all pipelineRun who doesn't have max-keep-runs annotation
+    default-max-keep-runs: ""
+
+    # Whether to auto configure newly created repositories, this will create a new
+    # namespace and repository CR, supported only with GitHub App
+    auto-configure-new-github-repo: "false"
+
+    # add a template to generate name for namespace for your auto configured
+    # github repo supported fields are repo_owner, repo_name eg. if defined as
+    # `{{repo_owner}}-{{repo_name}}-ci`, then namespace generated for repository
+    # https://github.com/owner/repo will be `owner-repo-ci`
+    auto-configure-repo-namespace-template: ""
+
+    # Enable or disable the feature to rerun the CI if push event happens on
+    # a pull request
+    #
+    # By default it is true and CI will be re-run in case of push/amend on the
+    # pull request if ok-to-test is done once
+    #
+    # you may want to disable this if ok-to-test should be done on each iteration
+    remember-ok-to-test: "true"
+
+    # Configure a custom console here, the driver support custom parameters from
+    # Repo CR along a few other template variable, see documentation for more
+    # details
+    #
+    # custom-console-name: Console Name
+    # custom-console-url: https://url
+    # custom-console-url-pr-details: https://url/ns/{{ namespace }}/{{ pr }}
+    # custom-console-url-pr-tasklog: https://url/ns/{{ namespace }}/{{ pr }}/logs/{{ task }}
 
 kind: ConfigMap
 metadata:

config/400-controller.yaml

@@ -98,8 +98,13 @@ spec:
             - mountPath: "/etc/pipelines-as-code/tls"
               readOnly: true
               name: tls
+            - mountPath: "/etc/pipelines-as-code/config"
+              name: config
       volumes:
       - name: tls
         secret:
           secretName: pipelines-as-code-tls-secret
           optional: true
+      - name: config
+        configMap:
+          name: pipelines-as-code

config/500-watcher.yaml

@@ -83,4 +83,11 @@ spec:
               scheme: HTTP
             initialDelaySeconds: 5
             periodSeconds: 10
-            timeoutSeconds: 5
+            timeoutSeconds: 5
+          volumeMounts:
+            - mountPath: "/etc/pipelines-as-code/config"
+              name: config
+      volumes:
+      - name: config
+        configMap:
+          name: pipelines-as-code

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.7.0
+	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.8.4
 	github.com/tektoncd/pipeline v0.50.0
 	github.com/xanzy/go-gitlab v0.90.0
@@ -49,15 +50,25 @@ require (
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230321174746-8dcc6526cfb1 // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/davidmz/go-pageant v1.0.2 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-fed/httpsig v1.1.0 // indirect
 	github.com/google/gnostic v0.6.9 // indirect
 	github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mcuadros/go-defaults v1.2.0 // indirect
 	github.com/onsi/ginkgo/v2 v2.9.7 // indirect
 	github.com/onsi/gomega v1.27.8 // indirect
+	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/spf13/afero v1.9.5 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/subosito/gotenv v1.4.2 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20230718181711-3c0fae5ee9fd // indirect
 )