Rework and test (#63)

* add issuer * add charts * initial local and test setup * doc * fix key * enable did-helper * add test workflow * tests and docs * more doc * more documentation * more doc * verify the results * deploy the chart * fix verification * stable version * update verifier chart * Update doc/LOCAL.MD Co-authored-by: Tim Smyth <[email protected]> * Update doc/LOCAL.MD Co-authored-by: Tim Smyth <[email protected]> * Update energyReport.json --------- Co-authored-by: Tim Smyth <[email protected]>
FIWARE-Ops · Jun 5, 2024 · 0f8b8bb · 0f8b8bb
1 parent ef9f2d5
commit 0f8b8bb
Show file tree

Hide file tree

Showing 69 changed files with 4,690 additions and 102 deletions.
diff --git a/.github/workflows/release-helm.yaml b/.github/workflows/release-helm.yaml
@@ -7,52 +7,6 @@ on:
 
 jobs:
 
-    generate-version:
-      runs-on: ubuntu-latest
-
-      outputs:
-        version: ${{ steps.out.outputs.version }}
-
-      steps:
-        - uses: actions/checkout@v2
-
-        - uses: actions/setup-java@v1
-          with:
-            java-version: '17'
-            java-package: jdk
-
-        - id: pr
-          uses: actions-ecosystem/[email protected]
-          with:
-            github_token: ${{ secrets.GITHUB_TOKEN }}
-
-        - name: Match semver label via bash
-          id: match-label-bash
-          run: |
-            LABELS=$(cat <<-END
-            ${{ steps.pr.outputs.labels }}
-            END
-            )
-            IFS='\n' read -ra LABEL <<< "$LABELS"
-            for i in "${LABEL[@]}"; do
-                case $i in
-                    # Will just use the first occurence
-                    'major'|'minor'|'patch')
-                        echo "RELEASE_LABEL=$i" >> $GITHUB_OUTPUT
-                        break
-                esac
-            done
-          
-        - uses: zwaldowski/semver-release-action@v2
-          with:
-            dry_run: true
-            bump: ${{ steps.match-label-bash.outputs.RELEASE_LABEL }}
-            github_token: ${{ secrets.GITHUB_TOKEN }}
-
-        - name: Set version output
-          id: out
-          run: echo "::set-output name=version::$(echo ${VERSION})"
-
     deploy:
 
       needs: [ "generate-version" ]
@@ -68,33 +22,18 @@ jobs:
           run: |
             git config user.name "$GITHUB_ACTOR"
             git config user.email "[email protected]"
-          
-        # See https://github.com/helm/chart-releaser-action/issues/6
+
         - name: Install Helm
           run: |
             curl -fsSLo get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
             chmod 700 get_helm.sh
             ./get_helm.sh
 
-        # prepare yaml parser
-        - uses: actions/setup-go@v4
-        - name: Install yq
-          run: |
-            go install github.com/mikefarah/yq/v4@latest
-            yq --version
-
-        - name: Generate Chart.yaml
-          run: | 
-            ./generate.sh ${{ needs.generate-version.outputs.version }}
-
-
-        - name: Install releaser
-          run: |
-            wget https://github.com/helm/chart-releaser/releases/download/v1.6.0/chart-releaser_1.6.0_linux_amd64.tar.gz          
-            tar -xvzf chart-releaser_1.6.0_linux_amd64.tar.gz
-            ./cr package charts/data-space-connector
-            ./cr upload --owner ${GITHUB_REPOSITORY_OWNER} --git-repo data-space-connector --packages-with-index --token ${{ secrets.GITHUB_TOKEN }} --push --skip-existing
-            ./cr index --owner ${GITHUB_REPOSITORY_OWNER} --git-repo data-space-connector  --packages-with-index --index-path . --token ${{ secrets.GITHUB_TOKEN }} --push
+        - name: Run chart-releaser
+          uses: helm/[email protected]
+          env:
+            CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+            CR_SKIP_EXISTING: true
 
     git-release:
       needs: ["generate-version","deploy"]

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -0,0 +1,20 @@
+name: Test
+on:
+  push
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-java@v1
+        with:
+          java-version: '17'
+          java-package: jdk
+
+      - name: Execute tests
+        id: test
+        run: |
+          mvn clean integration-test -Ptest
diff --git a/README.md b/README.md
@@ -12,6 +12,10 @@ FIWARE [data-space-connector repository](https://github.com/FIWARE/data-space-co
 
 ## Deployment
 
+### Local Deployment
+
+The FIWARE Data Space Connector provides a local deployment of a Minimal Viable Dataspace. 
+Find a detailed documentation here: [Local Deployment](./doc/LOCAL.MD)
 
 ### Deployment with ArgoCD
 
@@ -75,3 +79,13 @@ The chart is [generated](generate.sh) on each merge to master from the current a
 
 Different examples for the deployment of the FIWARE Data Space connector can be found 
 under the [./examples](./examples) directory.
+
+## Testing
+
+In order to test the [helm-charts](./charts/) provided for the FIWARE Data Space Connector, an integration-test framework based on [Cucumber](https://cucumber.io/) and [Junit5](https://junit.org/junit5/) is provided: [it](./it).
+
+The tests can be executed via: 
+```shell
+    mvn clean integration-test -Ptest
+```
+They will spin up the [Local Data Space](./doc/LOCAL.MD) and run the [test-scenarios](./it/src/test/resources/it/mvds_basic.feature) against it.
diff --git a/charts/data-space-connector/Chart.yaml b/charts/data-space-connector/Chart.yaml
@@ -0,0 +1,52 @@
+apiVersion: v2
+name: data-space-connector
+description: Umbrella Chart for the FIWARE Data Space Connector, combining all essential parts to be used by a participant.
+type: application
+version: 3.0.0
+dependencies:
+  - name: postgresql
+    condition: postgresql.enabled
+    repository: oci://registry-1.docker.io/bitnamicharts
+    version: 13.1.5
+  # authentication
+  - name: vcverifier
+    condition: vcverifier.enabled
+    version: 2.7.0
+    repository: https://fiware.github.io/helm-charts
+  - name: credentials-config-service
+    condition: credentials-config-service.enabled
+    version: 0.1.5
+    repository: https://fiware.github.io/helm-charts
+  - name: trusted-issuers-list
+    condition: trusted-issuers-list.enabled
+    version: 0.6.2
+    repository: https://fiware.github.io/helm-charts
+  - name: mysql
+    condition: mysql.enabled
+    version: 9.4.4
+    repository: https://charts.bitnami.com/bitnami
+  # authorization
+  - name: odrl-pap
+    condition: odrl-pap.enabled
+    version: 0.0.22
+    repository: https://fiware.github.io/helm-charts
+  - name: apisix
+    condition: apisix.enabled
+    version: 3.1.0
+    repository: oci://registry-1.docker.io/bitnamicharts
+  # data-service
+  - name: scorpio-broker-aaio
+    alias: scorpio
+    condition: scorpio.enabled
+    repository: https://fiware.github.io/helm-charts
+    version: 0.4.7
+  - name: postgresql
+    alias: postgis
+    condition: postgis.enabled
+    repository: oci://registry-1.docker.io/bitnamicharts
+    version: 13.1.5
+  # issuance
+  - name: keycloak
+    condition: keycloak.enabled
+    version: 21.1.1
+    repository: https://charts.bitnami.com/bitnami
diff --git a/charts/data-space-connector/templates/_helpers.tpl b/charts/data-space-connector/templates/_helpers.tpl
@@ -0,0 +1,55 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "dsc.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "dsc.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "dsc.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "dsc.serviceAccountName" -}}
+{{- if .Values.did.serviceAccount.create -}}
+    {{ default (include "dsc.fullname" .) .Values.did.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.did.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "dsc.labels" -}}
+app.kubernetes.io/name: {{ include "dsc.name" . }}
+helm.sh/chart: {{ include "dsc.chart" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
diff --git a/charts/data-space-connector/templates/apisix-cm.yaml b/charts/data-space-connector/templates/apisix-cm.yaml
@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: apisix-routes
+  namespace: {{ $.Release.Namespace | quote }}
+  labels:
+    {{ include "dsc.labels" . | nindent 4 }}
+data:
+  apisix.yaml: |-
+    routes:
+    {{- if .Values.apisix.catchAllRoute.enabled }}
+      - uri: /*
+        upstream:
+            nodes:
+                {{ .Values.apisix.catchAllRoute.upstream.url}}: 1
+            type: roundrobin
+        plugins:
+          openid-connect:
+            client_id: {{ .Values.apisix.catchAllRoute.oidc.clientId }}
+            client_secret: the-secret
+            bearer_only: true
+            use_jwks: true
+            discovery: {{ .Values.apisix.catchAllRoute.oidc.discoveryEndpoint }}
+          opa:
+            host: "http://localhost:{{ .Values.opa.port }}"
+            policy: policy/main
+    {{- end }}
+    {{- if .Values.apisix.routes }}
+      {{ .Values.apisix.routes | nindent 6 }}
+    {{- end }}
+    #END
diff --git a/charts/data-space-connector/templates/authentication-secrets.yaml b/charts/data-space-connector/templates/authentication-secrets.yaml
@@ -0,0 +1,14 @@
+{{- if .Values.authentication.generatePasswords.enabled }}
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: {{ .Values.authentication.generatePasswords.secretName }}
+  namespace: {{ $.Release.Namespace | quote }}
+  labels:
+    {{ include "dsc.labels" . | nindent 4 }}
+data:
+  mysql-root-password: {{ randAlphaNum 30 | b64enc | quote }}
+  mysql-replication-password: {{ randAlphaNum 30 | b64enc | quote }}
+  mysql-password: {{ randAlphaNum 30 | b64enc | quote }}
+{{- end }}
diff --git a/charts/data-space-connector/templates/data-plane-secrets.yaml b/charts/data-space-connector/templates/data-plane-secrets.yaml
@@ -0,0 +1,13 @@
+{{- if .Values.dataplane.generatePasswords.enabled }}
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: {{ .Values.dataplane.generatePasswords.secretName }}
+  namespace: {{ $.Release.Namespace | quote }}
+  labels:
+    {{ include "dsc.labels" . | nindent 4 }}
+data:
+  postgres-user-password: {{ randAlphaNum 30 | b64enc | quote }}
+  postgres-admin-password: {{ randAlphaNum 30 | b64enc | quote }}
+{{- end }}
diff --git a/charts/data-space-connector/templates/database-secrets.yaml b/charts/data-space-connector/templates/database-secrets.yaml
@@ -0,0 +1,13 @@
+{{- if .Values.postgresql.generatePasswords.enabled }}
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: {{ .Values.postgresql.generatePasswords.secretName }}
+  namespace: {{ $.Release.Namespace | quote }}
+  labels:
+    {{ include "dsc.labels" . | nindent 4 }}
+data:
+  postgres-user-password: {{ randAlphaNum 30 | b64enc | quote }}
+  postgres-admin-password: {{ randAlphaNum 30 | b64enc | quote }}
+{{- end }}
diff --git a/charts/data-space-connector/templates/dataplane-registration.yaml b/charts/data-space-connector/templates/dataplane-registration.yaml
@@ -0,0 +1,39 @@
+{{- if and (eq .Values.scorpio.enabled true) (.Values.scorpio.ccs) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.scorpio.ccs.configMap }}
+  namespace: {{ $.Release.Namespace | quote }}
+  labels:
+    {{- include "dsc.labels" . | nindent 4 }}
+data:
+  init.sh: |-
+    # credentials config service registration
+    curl -X 'POST' \
+      '{{ .Values.scorpio.ccs.endpoint }}/service' \
+      -H 'accept: */*' \
+      -H 'Content-Type: application/json' \
+      -d '{
+      "id": {{ .Values.scorpio.ccs.id | quote }},
+      "defaultOidcScope": {{ .Values.scorpio.ccs.defaultOidcScope.name | quote }},
+      {{- if and (.Values.scorpio.ccs.defaultOidcScope.credentialType) (.Values.scorpio.ccs.defaultOidcScope.trustedParticipantsLists) (.Values.scorpio.ccs.defaultOidcScope.trustedIssuersLists) -}}
+      "oidcScopes": {
+        {{ .Values.scorpio.ccs.defaultOidcScope.name | quote }}: [
+          {
+            "type": {{ .Values.scorpio.ccs.defaultOidcScope.credentialType | quote }},
+            "trustedParticipantsLists": [
+              {{ .Values.scorpio.ccs.defaultOidcScope.trustedParticipantsLists | quote }}
+            ],
+            "trustedIssuersLists": [
+              {{ .Values.scorpio.ccs.defaultOidcScope.trustedIssuersLists | quote }}
+            ]
+          }
+        ]
+      }
+      {{- end }}
+      {{- if .Values.scorpio.ccs.oidcScopes -}}
+      "oidcScopes": {{- toJson .Values.scorpio.ccs.oidcScopes }}
+      {{- end }}
+    }'
+
+{{- end }}