Merge pull request #121 from neicnordic/refactoring

Refactoring

.github/workflows/sda-notls.yml → .github/workflows/sda-pipeline-notls.yml

@@ -1,15 +1,9 @@
-name: Deployment without TLS
+name: standalone sda deployment without TLS
 
 on: [push,pull_request]
 
-env:
-  svc_list: 'auth backup finalize inbox ingest mapper verify'
-
 jobs:
   build:
-    strategy:
-      matrix:
-        test: [sda-notls]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
@@ -26,24 +20,20 @@ jobs:
     - name: Create secrets
       run: bash ./dev_tools/scripts/create-secrets.sh
     - name: Deploy SDA database
-      run: bash ./dev_tools/scripts/deploy-no-tls.sh database
-    - name: Wait for database to become ready
-      run: bash ./dev_tools/scripts/wait-for-pods.sh database
+      run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh database
     - name: Deploy SDA message broker
-      run: bash ./dev_tools/scripts/deploy-no-tls.sh broker
-    - name: Wait for broker to become ready
-      run: bash ./dev_tools/scripts/wait-for-pods.sh broker
+      run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh broker
+    - name: Deploy SDA orchestrator
+      run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh orchestrate
     - name: Deploy mock oidc server
       run: bash ./dev_tools/scripts/deploy-oidc.sh
     - name: Deploy minio
-      run: bash ./dev_tools/scripts/deploy-no-tls.sh minio
-    - name: Wait for minio to become ready
-      run: bash ./dev_tools/scripts/wait-for-pods.sh minio app
+      run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh minio
     - name: Create s3 buckets
       run: bash ./dev_tools/scripts/create-s3-buckets-no-tls.sh
     - name: Deploy the SDA stack
-      run: bash ./dev_tools/scripts/deploy-no-tls.sh pipeline
+      run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh pipeline
     - name: Wait for sda to become ready
-      run: bash ./dev_tools/scripts/wait-for-pods.sh "${{ env.svc_list }}"
+      run: bash ./dev_tools/scripts/wait-for-pods.sh standalone_s3_svc_list
     - name: Run helm test
       run: bash ./dev_tools/scripts/run-helm-test.sh

.github/workflows/sda-pipeline.yml

@@ -1,73 +1,94 @@
 name: sda-pipeline deployment
 
-on: [push,pull_request]
+on: [push, pull_request]
 
 jobs:
   build:
     strategy:
+      fail-fast: false
       matrix:
-        test: [sftp-inbox, s3-inbox]
+        inbox: [posix, s3]
+        deployment: [federated, standalone]
+        cert: [issuer, manual]
     runs-on: ubuntu-latest
     steps:
-    - name: ACTIONS_ALLOW_UNSECURE_COMMANDS
-      id: ACTIONS_ALLOW_UNSECURE_COMMANDS
-      run: echo 'ACTIONS_ALLOW_UNSECURE_COMMANDS=true' >> $GITHUB_ENV
-    # Currently the sftp inbox test covers the standalone test, whereas the s3 test checks the federated topology
-    - name: fixup env
-      run: |
-        if [ "${{ matrix.test }}" = "sftp-inbox" ]; then
-          echo "SVCS=backup doa finalize inbox ingest intercept mapper verify download" >> "$GITHUB_ENV";
-        else
-          echo "SVCS=auth backup doa finalize inbox ingest mapper verify download" >> "$GITHUB_ENV";
-        fi
-    - uses: actions/checkout@v3
-    - name: Install kube dependencies
-      run: bash ./dev_tools/scripts/install-kube-deps.sh
-    - name: Initialise k3d
-      run: bash ./dev_tools/scripts/init-k3d.sh
-    - name: Wait for k3d to become ready
-      run: bash ./dev_tools/scripts/wait-for-pods.sh metrics-server k8s-app kube-system
-    - name: Install sda dependencies
-      run: bash ./dev_tools/scripts/install-sda-deps.sh
-    - name: Create certificates
-      run: bash ./dev_tools/scripts/make-certs.sh
-    - name: Create secrets
-      run: bash ./dev_tools/scripts/create-secrets.sh
-    - name: Set up services configuration
-      run: bash ./dev_tools/scripts/svc-setup.sh
-    - name: Deploy SDA database
-      run: bash ./dev_tools/scripts/deploy-db.sh
-    - name: Wait for database to become ready
-      run: bash ./dev_tools/scripts/wait-for-pods.sh database
-    - name: Deploy mock oidc server
-      if: matrix.test == 's3-inbox'
-      run: bash ./dev_tools/scripts/deploy-oidc.sh
-    - name: Deploy minio
-      if: matrix.test == 's3-inbox'
-      run: bash ./dev_tools/scripts/deploy-minio.sh
-    - name: Wait for minio to become ready
-      if: matrix.test == 's3-inbox'
-      run: bash ./dev_tools/scripts/wait-for-pods.sh minio app
-    - name: Create s3 buckets
-      if: matrix.test == 's3-inbox'
-      run: bash ./dev_tools/scripts/create-s3-buckets.sh
-    - name: Start CEGA services
-      run: bash ./dev_tools/scripts/deploy-cega.sh
-    - name: Wait for CEGA to become ready
-      run: bash ./dev_tools/scripts/wait-for-pods.sh cega-mq app
-    - name: Deploy SDA message broker
-      run: bash ./dev_tools/scripts/deploy-mq.sh
-    - name: Wait for broker to become ready
-      run: bash ./dev_tools/scripts/wait-for-pods.sh broker
-    - name: Deploy the SDA stack for posix
-      if: matrix.test == 'sftp-inbox'
-      run: |
-        kubectl apply -f dev_tools/config/posix-volumes.yaml;
-        bash ./dev_tools/scripts/sda/deploy-posix-standalone.sh "not-orchestrated";
-    - name: Deploy the SDA stack for s3
-      if: matrix.test == 's3-inbox'
-      run: bash ./dev_tools/scripts/sda/deploy-s3-federated.sh
-    - name: Wait for sda to become ready
-      run: bash ./dev_tools/scripts/wait-for-pods.sh "${{ env.SVCS }}"
-    - name: Run helm test
-      run: bash ./dev_tools/scripts/run-helm-test.sh
+      - uses: actions/checkout@v2
+
+      - name: Install kube dependencies
+        run: bash ./dev_tools/scripts/install-kube-deps.sh
+
+      - name: Initialise k3d
+        run: bash ./dev_tools/scripts/init-k3d.sh
+
+      - name: Wait for k3d to become ready
+        run: bash ./dev_tools/scripts/wait-for-pods.sh metrics-server k8s-app kube-system
+
+      - name: Install sda dependencies
+        run: bash ./dev_tools/scripts/install-sda-deps.sh
+
+      - name: Create certificates
+        if: matrix.cert == 'manual'
+        run: bash ./dev_tools/scripts/make-certs.sh
+
+      - name: Create certificate issuer
+        if: matrix.cert == 'issuer'
+        run: bash ./dev_tools/scripts/deploy-cert-manager.sh
+
+      - name: Create secrets
+        run: bash ./dev_tools/scripts/create-secrets.sh
+
+      - name: Set up services configuration
+        if: matrix.cert == 'manual'
+        run: bash ./dev_tools/scripts/svc-setup.sh
+
+      - name: Deploy SDA database
+        run: bash ./dev_tools/scripts/deploy-db.sh "${{ matrix.cert }}"
+
+      - name: Wait for database to become ready
+        run: bash ./dev_tools/scripts/wait-for-pods.sh database
+
+      - name: Deploy mock oidc server
+        if: matrix.inbox == 's3'
+        run: bash ./dev_tools/scripts/deploy-oidc.sh 
+
+      - name: Deploy minio
+        if: matrix.inbox == 's3'
+        run: bash ./dev_tools/scripts/deploy-minio.sh "${{ matrix.cert }}"
+
+      - name: Wait for minio to become ready
+        if: matrix.inbox == 's3'
+        run: bash ./dev_tools/scripts/wait-for-pods.sh minio app
+
+      - name: Create s3 buckets
+        if: matrix.inbox == 's3'
+        run: bash ./dev_tools/scripts/create-s3-buckets.sh
+
+      - name: Start CEGA services
+        if: matrix.deployment == 'federated'
+        run: bash ./dev_tools/scripts/deploy-cega.sh "${{ matrix.cert }}"
+
+      - name: Wait for CEGA to become ready
+        if: matrix.deployment == 'federated'
+        run: bash ./dev_tools/scripts/wait-for-pods.sh cega-mq app
+
+      - name: Deploy SDA message broker
+        run: bash ./dev_tools/scripts/deploy-mq.sh "${{ matrix.deployment }}" "${{ matrix.cert }}"
+
+      - name: Wait for broker to become ready
+        run: bash ./dev_tools/scripts/wait-for-pods.sh broker
+
+      - name: Deploy the SDA stack for posix
+        if: matrix.inbox == 'posix'
+        run: |
+          kubectl apply -f dev_tools/config/posix-volumes.yaml;
+          bash ./dev_tools/scripts/sda/deploy-posix.sh "${{ matrix.deployment }}" "${{ matrix.cert }}"
+
+      - name: Deploy the SDA stack for s3
+        if: matrix.inbox == 's3'
+        run: bash ./dev_tools/scripts/sda/deploy-s3.sh "${{ matrix.deployment }}" "${{ matrix.cert }}"
+
+      - name: Wait for sda to become ready
+        run: bash ./dev_tools/scripts/wait-for-pods.sh "${{ format('{0}_{1}_svc_list', matrix.deployment, matrix.inbox) }}"
+
+      - name: Run helm test
+        run: bash ./dev_tools/scripts/run-helm-test.sh

.github/workflows/shellcheck.yml

@@ -10,13 +10,3 @@ jobs:
     - uses: actions/checkout@v3
     - name: Run ShellCheck
       uses: ludeeus/action-shellcheck@master
-  python-scripts:
-    name: Python scripts linting
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v3
-    - name: Install Black
-      run: pip install black
-    - name: Run black --check .
-      run: black --check . -l 160

charts/sda-db/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: sda-db
-version: 0.4.0
+version: "0.4.1"
 description: Database component for Sensitive Data Archive (SDA) installation
 home: https://neic-sda.readthedocs.io
 icon: https://neic.no/assets/images/logo.png

charts/sda-db/README.md

@@ -4,17 +4,19 @@ Source repository: [https://github.com/neicnordic/sda-db](https://github.com/nei
 
 ## Installing the Chart
 
-Edit the values.yaml file and specify the relevant parts of the `default` section.
+Edit the values.yaml file and specify the relevant parts of the `global` section.
 
 Parameter | Description | Default
 --------- | ----------- | -------
 `global.pg_in_password` | Password for `lega_in` user, used for `data in` services. |`""`
 `global.pg_out_password` | Password for `lega_out` user, used for `data out` services. |`""`
 `global.tls.enabled` | Enable TLS for all connections. |`true`
+`global.tls.issuer` | Issuer for TLS certificate creation. |`""`
+`global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. |`""`
 `global.tls.secretName` | Name of the secret holding the certificates. |``
-`global.tls.certName` | Server certificate. |`postgresql.crt`
-`global.tls.keyName` | Server private key. |`postgresql.key`
-`global.tls.CAFile` | CA root certificate. |`root.crt`
+`global.tls.certName` | Server certificate. |`tls.crt`
+`global.tls.keyName` | Server private key. |`tls.key`
+`global.tls.CAFile` | CA root certificate. |`ca.crt`
 `global.tls.verifyPeer` | Require client certificates. |`verify-ca`
 `externalPkiService.tlsPath` | If an external PKI service is used, this is the path where the certifiates are placed | `""`
 `image.repository` | sda-db container image repository | `ghcr.io/neicnordic/sda-db`
@@ -44,11 +46,13 @@ Parameter | Description | Default
 
 ### TLS
 
-Create a secret that contains the certificates
+Automatic certificates can be generated by setting `global.tls.issuer` or `glbal.tls.clusterIssuer` if `cert-manger` is installed and configured.
+
+If certificates are created manually the secret that holds them needs to be created manually.
 
 ```cmd
-kubectl create secret generic ca-secret \
---from-file=root.crt=ca.crt \
---from-file=postgresql.crt=server.crt \
---from-file=postgresql.key=server.key
+kubectl create secret generic db-certs \
+--from-file=ca.crt\
+--from-file=tls.crt\
+--from-file=tls.key
 ```

charts/sda-db/templates/_helpers.tpl

@@ -41,23 +41,61 @@ Create chart name and version as used by the chart label.
 {{- define "pgCert" -}}
     {{- if .Values.externalPkiService.tlsPath }}
         {{- printf "%s" (regexReplaceAll "^/*|/+" (printf "%s/%s" .Values.externalPkiService.tlsPath .Values.global.tls.certName) "/")}}
+    {{- else if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }}
+        {{- printf "%s/tls/tls.crt" .Values.persistence.mountPath -}}
     {{- else }}
-        {{- printf "%s/tls/%s" .Values.persistence.mountPath .Values.global.tls.certName }}
+        {{- printf "%s/tls/%s" .Values.persistence.mountPath (required "name of tls certificate is required" .Values.global.tls.certName) }}
     {{- end -}}
 {{- end -}}
 
 {{- define "pgKey" -}}
     {{- if .Values.externalPkiService.tlsPath }}
         {{- printf "%s" (regexReplaceAll "^/*|/+" (printf "%s/%s" .Values.externalPkiService.tlsPath .Values.global.tls.keyName) "/")}}
+    {{- else if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }}
+        {{- printf "%s/tls/tls.key" .Values.persistence.mountPath -}}
     {{- else }}
-        {{- printf "%s/tls/%s" .Values.persistence.mountPath .Values.global.tls.keyName }}
+        {{- printf "%s/tls/%s" .Values.persistence.mountPath (required "name of tls key is required" .Values.global.tls.keyName) }}
     {{- end -}}
 {{- end -}}
 
 {{- define "caCert" -}}
     {{- if .Values.externalPkiService.tlsPath }}
         {{- printf "%s" (regexReplaceAll "^/*|/+" (printf "%s/%s" .Values.externalPkiService.tlsPath .Values.global.tls.CAFile) "/")}}
+    {{- else if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }}
+        {{- printf "%s/tls/ca.crt" .Values.persistence.mountPath -}}
     {{- else }}
-        {{- printf "%s/tls/%s" .Values.persistence.mountPath .Values.global.tls.CAFile }}
+        {{- printf "%s/tls/%s" .Values.persistence.mountPath (required "name of ca file is required" .Values.global.tls.CAFile) }}
+    {{- end -}}
+{{- end -}}
+
+{{- define "TLSissuer" -}}
+    {{- if and .Values.global.tls.clusterIssuer .Values.global.tls.issuer }}
+        {{- fail "Only one of global.tls.issuer or global.tls.clusterIssuer should be set" }}
+    {{- end -}}
+
+    {{- if and .Values.global.tls.issuer }}
+        {{- printf "%s" .Values.global.tls.issuer }}
+    {{- else if and .Values.global.tls.clusterIssuer }}
+        {{- printf "%s" .Values.global.tls.clusterIssuer }}
+    {{- end -}}
+{{- end -}}
+
+{{- define "TLSsecret" -}}
+    {{- if and .Values.global.tls.enabled (not .Values.externalPkiService.tlsPath) }}
+        {{- if and (not .Values.global.tls.issuer) (not .Values.global.tls.clusterIssuer) }}
+            {{ printf "%s" (required "TLS secret name is required when TLS in enabled without issuer or PKI service" .Values.global.tls.secretName) }}
+        {{- else }}
+            {{- printf "%s-certs" (include "sda.fullname" .) }}
+        {{- end -}}
+    {{- end -}}
+{{- end -}}
+
+{{- define "testTLSsecret" -}}
+    {{- if and .Values.global.tls.enabled (not .Values.externalPkiService.tlsPath) }}
+        {{- if and (not .Values.global.tls.issuer) (not .Values.global.tls.clusterIssuer) }}
+            {{ printf "%s" (required "TLS secret name is required when TLS in enabled without issuer or PKI service" .Values.testimage.tls.secretName) }}
+        {{- else }}
+            {{- printf "%s-test-certs" (include "sda.fullname" .) }}
+        {{- end -}}
     {{- end -}}
 {{- end -}}