From f4e9d7ec0ea418f1928e1bfe7e05e87b1fdc998c Mon Sep 17 00:00:00 2001 From: Jorge Morales Pou Date: Sat, 1 Jun 2024 12:10:41 +0200 Subject: [PATCH] Updated local development of full gitops and fixed bug in kapp config --- README.md | 6 +++ .../overlays/fetch-app-app_per_bundle.yaml | 5 ++- .../ytt/overlays/fetch-app-one_app.yaml | 5 ++- .../overlays/overlay-configsecretcopier.yaml | 45 +++++++++++++++++++ .../ytt/overlays/overlay-fetch-config.yaml | 16 +++++++ .../src/bundle/config/ytt/values-schema.yaml | 10 +++++ .../src/bundle/config/kapp/kapp-config.yaml | 7 +-- .../ytt/overlays/overlay-create-tp.yaml | 2 + test-local/README.md | 36 +++++++++++++++ test-local/crd-devel.yaml | 37 +++++++++++++++ test-local/kind-config.yaml | 5 +++ test-local/rbac.yaml | 23 ++++++++++ test-local/secret-common.yaml | 24 ++++++++++ test-local/secret-versions.yaml | 32 +++++++++++++ test-local/secret-workshops.yaml | 42 +++++++++++++++++ 15 files changed, 288 insertions(+), 7 deletions(-) create mode 100644 overlays/gitops-app/src/bundle/config/ytt/overlays/overlay-configsecretcopier.yaml create mode 100644 test-local/README.md create mode 100644 test-local/crd-devel.yaml create mode 100644 test-local/kind-config.yaml create mode 100644 test-local/rbac.yaml create mode 100644 test-local/secret-common.yaml create mode 100644 test-local/secret-versions.yaml create mode 100644 test-local/secret-workshops.yaml diff --git a/README.md b/README.md index c4cb0e7..10d49ea 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,12 @@ ytt -v name=global \ -f overlays/portal-app/src/bundle/config/kapp ``` +### Local development/testing of workshop configuration overlays in cluster + +There's [instructions](./test-local/README.md) on how to test the full gitops reconciler on a local +kind cluster for development purposes that don't rely on changes in git, but rather on changes on secrets. + + ## TODO - implement installation in terraform provisioning diff --git a/overlays/gitops-app/src/bundle/config/ytt/overlays/fetch-app-app_per_bundle.yaml b/overlays/gitops-app/src/bundle/config/ytt/overlays/fetch-app-app_per_bundle.yaml index 46888ad..6518973 100644 --- a/overlays/gitops-app/src/bundle/config/ytt/overlays/fetch-app-app_per_bundle.yaml +++ b/overlays/gitops-app/src/bundle/config/ytt/overlays/fetch-app-app_per_bundle.yaml @@ -31,7 +31,10 @@ metadata: namespace: #@ "workshops-{}".format(bundle.name) annotations: kapp.k14s.io/change-group: #@ "workshops-{}-app".format(bundle.name) - kapp.k14s.io/change-rule.insert: #@ "upsert after upserting workshops-{}-crb".format(bundle.name) + kapp.k14s.io/change-rule.insert.1: #@ "upsert after upserting workshops-{}-crb".format(bundle.name) + #@ if/end data.values.config.type == "local": + kapp.k14s.io/change-rule.insert.2: "upsert after upserting local-config-secrets" + kapp.k14s.io/disable-original: "" spec: serviceAccountName: #@ "workshops-{}".format(bundle.name) syncPeriod: "0h10m0s" diff --git a/overlays/gitops-app/src/bundle/config/ytt/overlays/fetch-app-one_app.yaml b/overlays/gitops-app/src/bundle/config/ytt/overlays/fetch-app-one_app.yaml index dfc1d9b..0a6ea88 100644 --- a/overlays/gitops-app/src/bundle/config/ytt/overlays/fetch-app-one_app.yaml +++ b/overlays/gitops-app/src/bundle/config/ytt/overlays/fetch-app-one_app.yaml @@ -16,7 +16,10 @@ metadata: namespace: #@ "workshops-{}".format("global") annotations: kapp.k14s.io/change-group: #@ "workshops-{}-app".format("global") - kapp.k14s.io/change-rule.insert: #@ "upsert after upserting workshops-{}-crb".format("global") + kapp.k14s.io/change-rule.insert.1: #@ "upsert after upserting workshops-{}-crb".format("global") + #@ if/end data.values.config.type == "local": + kapp.k14s.io/change-rule.insert.2: "upsert after upserting local-config-secrets" + kapp.k14s.io/disable-original: "" spec: serviceAccountName: #@ "workshops-{}".format("global") syncPeriod: "0h10m0s" diff --git a/overlays/gitops-app/src/bundle/config/ytt/overlays/overlay-configsecretcopier.yaml b/overlays/gitops-app/src/bundle/config/ytt/overlays/overlay-configsecretcopier.yaml new file mode 100644 index 0000000..ed203e1 --- /dev/null +++ b/overlays/gitops-app/src/bundle/config/ytt/overlays/overlay-configsecretcopier.yaml @@ -0,0 +1,45 @@ +#@ load("@ytt:overlay", "overlay") +#@ load("@ytt:data", "data") + +#@ if data.values.config.type == "local": + +#@ def config_secretcopier(name, ns_from, ns_to): +#@ if ns_from != ns_to: +kind: SecretCopier +apiVersion: secrets.educates.dev/v1beta1 +metadata: + annotations: + kapp.k14s.io/change-group: local-config-secrets + name: #@ "{}-{}".format(name, ns_to) +spec: + rules: + - reclaimPolicy: Delete + sourceSecret: + name: #@ name + namespace: #@ ns_from + targetNamespaces: + nameSelector: + matchNames: + - #@ ns_to +#@ end +#@ end #! config_secretcopier + +#@ if data.values.mode == "app_per_bundle": + +#@ for bundle in data.values.workshop_bundles: +--- #@ config_secretcopier(data.values.config.local.secretRef.common.name, data.values.config.local.secretRef.common.namespace, "workshops-{}".format(bundle.name)) +--- #@ config_secretcopier(data.values.config.local.secretRef.workshops.name, data.values.config.local.secretRef.workshops.namespace, "workshops-{}".format(bundle.name)) +#@ end + +#@ else: + +#@ for bundle in data.values.workshop_bundles: + +--- #@ config_secretcopier(data.values.config.local.secretRef.common.name, data.values.config.local.secretRef.common.namespace, "workshops-{}".format("global")) +--- #@ config_secretcopier(data.values.config.local.secretRef.workshops.name, data.values.config.local.secretRef.workshops.namespace, "workshops-{}".format("global")) + +#@ end #! for + +#@ end #! if/else + +#@ end #! data.values.config.type == "local" diff --git a/overlays/gitops-app/src/bundle/config/ytt/overlays/overlay-fetch-config.yaml b/overlays/gitops-app/src/bundle/config/ytt/overlays/overlay-fetch-config.yaml index 2bf082b..7a3869e 100644 --- a/overlays/gitops-app/src/bundle/config/ytt/overlays/overlay-fetch-config.yaml +++ b/overlays/gitops-app/src/bundle/config/ytt/overlays/overlay-fetch-config.yaml @@ -19,6 +19,8 @@ #@ config_creds = "creds-config" #@ end +#@ if data.values.config.type == "git": + #@overlay/match by=overlay.subset({"kind":"App"}),expects="1+" --- spec: @@ -32,3 +34,17 @@ spec: secretRef: name: #@ config_creds path: environment + +#@ else: +#@overlay/match by=overlay.subset({"kind":"App"}),expects="1+" +--- +spec: + fetch: + - inline: + pathsFrom: + - secretRef: + name: #@ data.values.config.local.secretRef.common.name + - secretRef: + name: #@ data.values.config.local.secretRef.workshops.name + path: environment +#@ end diff --git a/overlays/gitops-app/src/bundle/config/ytt/values-schema.yaml b/overlays/gitops-app/src/bundle/config/ytt/values-schema.yaml index b6baec2..15e771a 100644 --- a/overlays/gitops-app/src/bundle/config/ytt/values-schema.yaml +++ b/overlays/gitops-app/src/bundle/config/ytt/values-schema.yaml @@ -32,12 +32,22 @@ overlays: #@schema/nullable namespace: "" config: + #@schema/validation one_of=["git", "local"] + type: "git" #@schema/validation min_len=1 url: https://github.com/educates/educates-workshop-gitops-config #@schema/validation min_len=1 ref: origin/main #@schema/nullable subPath: "" + local: + secretRef: + common: + name: "common" + namespace: "package-installs" + workshops: + name: "workshops" + namespace: "package-installs" #@schema/nullable #@schema/validation one_not_null=["username", "secretRef"] credentials: diff --git a/overlays/portal-app/src/bundle/config/kapp/kapp-config.yaml b/overlays/portal-app/src/bundle/config/kapp/kapp-config.yaml index fd33520..f97adbe 100644 --- a/overlays/portal-app/src/bundle/config/kapp/kapp-config.yaml +++ b/overlays/portal-app/src/bundle/config/kapp/kapp-config.yaml @@ -2,12 +2,9 @@ apiVersion: kapp.k14s.io/v1alpha1 kind: Config rebaseRules: - paths: + - [status] - [metadata, annotations, "kopf.zalando.org/last-handled-configuration"] type: copy sources: [existing] resourceMatchers: - - apiVersionKindMatcher: { apiVersion: training.educates.dev/v1beta1, kind: TrainingPortal } -diffAgainstExistingFieldExclusionRules: - - path: [status] - resourceMatchers: - - apiVersionKindMatcher: { apiVersion: training.educates.dev/v1beta1, kind: TrainingPortal } + - apiVersionKindMatcher: { apiVersion: training.educates.dev/v1beta1, kind: TrainingPortal } \ No newline at end of file diff --git a/overlays/portal-app/src/bundle/config/ytt/overlays/overlay-create-tp.yaml b/overlays/portal-app/src/bundle/config/ytt/overlays/overlay-create-tp.yaml index 155270c..aaf253f 100644 --- a/overlays/portal-app/src/bundle/config/ytt/overlays/overlay-create-tp.yaml +++ b/overlays/portal-app/src/bundle/config/ytt/overlays/overlay-create-tp.yaml @@ -25,6 +25,8 @@ apiVersion: training.educates.dev/v1beta1 kind: TrainingPortal metadata: + annotations: + kapp.k14s.io/disable-original: "" name: #@ portal.name spec: portal: diff --git a/test-local/README.md b/test-local/README.md new file mode 100644 index 0000000..13e898f --- /dev/null +++ b/test-local/README.md @@ -0,0 +1,36 @@ +# Test locally + +How to test on a local educates kind cluster (with local registry) and kapp-controller: + +``` +educates admin cluster create --config kind-config.yaml +``` + +1. Build the image and push it into your local registry: + + ``` + imgpkg --debug push -i localhost:5001/gitops-configurer:devel -f ../overlays + ``` + +2. Create your version of the configuration files in the [versions](./secret-versions.yaml), [common](./secret-common.yaml) and + [workshops](./secret-workshops.yaml) secrets and deploy them into the cluster: + + ``` + kubectl apply -f secret-versions.yaml + kubectl apply -f secret-common.yaml + kubectl apply -f secret-workshops.yaml + ``` + +3. Create the required [RBAC](./rbac.yaml) and [Gitops App definition](./crd-devel.yaml) and deploy it into your cluster. + + ``` + kubectl apply -f rbac.yaml + kubectl apply -f crd-devel.yaml + ``` + +4. If you want to test any change in configuration, modify the appropriate secret and apply it into the cluster and wait for a reconciliation. + If you don't want to wait, kick the reconciliation manually of the main gitops app: + + ``` + kctrl app kick -a workshops-gitops -n package-installs + ``` diff --git a/test-local/crd-devel.yaml b/test-local/crd-devel.yaml new file mode 100644 index 0000000..55ce6dc --- /dev/null +++ b/test-local/crd-devel.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: kappctrl.k14s.io/v1alpha1 +kind: App +metadata: + name: workshops-gitops + namespace: package-installs + annotations: + educates_gitops_environment: "sample-environment" + kapp.k14s.io/disable-original: "" +spec: + serviceAccountName: workshops-gitops-package-installs + syncPeriod: "0h2m0s" + fetch: + - inline: + pathsFrom: + - secretRef: + name: versions + path: environments + - image: + url: registry.default.svc.cluster.local/gitops-configurer:devel + subPath: "gitops-app/src/bundle/config" + path: config + template: + - ytt: + ignoreUnknownComments: true + paths: + - config/ytt + - config/kapp + valuesFrom: + - path: environments/versions.yaml + - downwardAPI: + items: + - name: environment + fieldPath: metadata.annotations['educates_gitops_environment'] + deploy: + - kapp: + rawOptions: ["--app-changes-max-to-keep=5", "--wait-timeout=5m", "--diff-changes=true"] diff --git a/test-local/kind-config.yaml b/test-local/kind-config.yaml new file mode 100644 index 0000000..ed4ea14 --- /dev/null +++ b/test-local/kind-config.yaml @@ -0,0 +1,5 @@ +clusterInfrastructure: + provider: kind +clusterPackages: + kapp-controller: + enabled: true diff --git a/test-local/rbac.yaml b/test-local/rbac.yaml new file mode 100644 index 0000000..ff88f9a --- /dev/null +++ b/test-local/rbac.yaml @@ -0,0 +1,23 @@ +kind: Namespace +apiVersion: v1 +metadata: + name: package-installs +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: workshops-gitops-package-installs + namespace: package-installs +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: workshops-gitops-package-installs +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: workshops-gitops-package-installs + namespace: package-installs diff --git a/test-local/secret-common.yaml b/test-local/secret-common.yaml new file mode 100644 index 0000000..eb80053 --- /dev/null +++ b/test-local/secret-common.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Secret +metadata: + name: common + namespace: package-installs + annotations: + kapp.k14s.io/disable-original: "" +stringData: + common.yaml: | + live_updates: True + theme_name: light-theme + ancestors: + - https://workshops-test.live + sessions: + anonymous: 1 + registered: 4 + max: 10 + index: https://workshops-test.live + registration: + enabled: false + type: anonymous + cookies_domain: workshops-test.live + analytics: + webhook_url: "https://metrics.educates.dev/?client=name&token=password" diff --git a/test-local/secret-versions.yaml b/test-local/secret-versions.yaml new file mode 100644 index 0000000..74d073d --- /dev/null +++ b/test-local/secret-versions.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Secret +metadata: + name: versions + namespace: package-installs + annotations: + kapp.k14s.io/disable-original: "" +stringData: + versions.yaml: | + mode: "one_app" + overlays: + url: registry.default.svc.cluster.local/gitops-configurer + version: devel + config: + type: "local" + # local: + # secretRef: + # common: + # name: common + # namespace: package-installs + # workshops: + # name: workshops + # namespace: package-installs + subPath: "config" + workshop_bundles: + - name: "workshop-animals" + url: "ghcr.io/educates/educates-workshop-gitops-configurer-workshop-animals-manifests" + #semver: ">=1.0.0" + version: "latest" + - name: "workshop-colours" + url: "ghcr.io/educates/educates-workshop-gitops-configurer-workshop-colours-manifests" + version: "latest" diff --git a/test-local/secret-workshops.yaml b/test-local/secret-workshops.yaml new file mode 100644 index 0000000..7248135 --- /dev/null +++ b/test-local/secret-workshops.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: Secret +metadata: + name: workshops + namespace: package-installs + annotations: + kapp.k14s.io/disable-original: "" +stringData: + workshops.yaml: | + portals: + - name: "workshop-bundle-animals" + title: "Animals workshops" + sessions: + max: 2 + defaults: + expires: 10m + orphaned: 5m + reserved: 0 + workshops: + - name: workshop-dog + expires: 60m + orphaned: 5m + reserved: 1 + - name: workshop-cat + - name: workshop-bird + - name: workshop-fish + - name: workshop-giraffe + - name: "workshop-bundle-colours" + sessions: + max: 2 + defaults: + expires: 10m + orphaned: 5m + reserved: 24 + credentials: + admin: + username: user + password: password + workshops: + - name: workshop-red + - name: workshop-blue + - name: workshop-green