diff --git a/.github/workflows/sda-orch.yml b/.github/workflows/sda-orch.yml deleted file mode 100644 index 35afe5bc..00000000 --- a/.github/workflows/sda-orch.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: sda standalone deployment - -on: [push,pull_request] - -env: - svc_list: 'auth backup doa finalize inbox ingest mapper verify download' - -jobs: - build: - strategy: - matrix: - test: [sda-standalone] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Install kube dependencies - run: bash ./dev_tools/scripts/install-kube-deps.sh - - name: Initialise k3d - run: bash ./dev_tools/scripts/init-k3d.sh - - name: Wait for k3d to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh metrics-server k8s-app kube-system - - name: Install sda dependencies - run: bash ./dev_tools/scripts/install-sda-deps.sh - - name: Create certificates - run: bash ./dev_tools/scripts/make-certs.sh - - name: Create secrets - run: bash ./dev_tools/scripts/create-secrets.sh - - name: Set up services configuration - run: bash ./dev_tools/scripts/svc-setup.sh - - name: Deploy SDA database - run: bash ./dev_tools/scripts/deploy-db.sh - - name: Wait for database to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh database - - name: Deploy SDA message broker - run: bash ./dev_tools/scripts/deploy-mq.sh - - name: Wait for broker to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh broker - - name: Deploy mock oidc server - run: bash ./dev_tools/scripts/deploy-oidc.sh - - name: Deploy the SDA orch - run: bash ./dev_tools/scripts/sda/deploy-orch.sh - - name: Wait for orchestrate to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh orchestrate - - name: Deploy minio - run: bash ./dev_tools/scripts/deploy-minio.sh - - name: Wait for minio to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh minio app - - name: Create s3 buckets - run: bash ./dev_tools/scripts/create-s3-buckets.sh - - name: Deploy the SDA stack - run: bash ./dev_tools/scripts/sda/deploy-s3-standalone.sh "orchestrated" - - name: Wait for sda to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh "${{ env.svc_list }}" - - name: Run helm test - run: bash ./dev_tools/scripts/run-helm-test.sh "broker postgres" diff --git a/.github/workflows/sda-notls.yml b/.github/workflows/sda-pipeline-notls.yml similarity index 58% rename from .github/workflows/sda-notls.yml rename to .github/workflows/sda-pipeline-notls.yml index 2e860259..76504057 100644 --- a/.github/workflows/sda-notls.yml +++ b/.github/workflows/sda-pipeline-notls.yml @@ -1,15 +1,9 @@ -name: Deployment without TLS +name: standalone sda deployment without TLS on: [push,pull_request] -env: - svc_list: 'auth backup finalize inbox ingest mapper verify' - jobs: build: - strategy: - matrix: - test: [sda-notls] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -26,24 +20,20 @@ jobs: - name: Create secrets run: bash ./dev_tools/scripts/create-secrets.sh - name: Deploy SDA database - run: bash ./dev_tools/scripts/deploy-no-tls.sh database - - name: Wait for database to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh database + run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh database - name: Deploy SDA message broker - run: bash ./dev_tools/scripts/deploy-no-tls.sh broker - - name: Wait for broker to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh broker + run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh broker + - name: Deploy SDA orchestrator + run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh orchestrate - name: Deploy mock oidc server run: bash ./dev_tools/scripts/deploy-oidc.sh - name: Deploy minio - run: bash ./dev_tools/scripts/deploy-no-tls.sh minio - - name: Wait for minio to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh minio app + run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh minio - name: Create s3 buckets run: bash ./dev_tools/scripts/create-s3-buckets-no-tls.sh - name: Deploy the SDA stack - run: bash ./dev_tools/scripts/deploy-no-tls.sh pipeline + run: bash ./dev_tools/scripts/sda/deploy-no-tls.sh pipeline - name: Wait for sda to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh "${{ env.svc_list }}" + run: bash ./dev_tools/scripts/wait-for-pods.sh standalone_s3_svc_list - name: Run helm test run: bash ./dev_tools/scripts/run-helm-test.sh diff --git a/.github/workflows/sda-pipeline.yml b/.github/workflows/sda-pipeline.yml index 1e494b7f..ae3b3abf 100644 --- a/.github/workflows/sda-pipeline.yml +++ b/.github/workflows/sda-pipeline.yml @@ -1,73 +1,94 @@ name: sda-pipeline deployment -on: [push,pull_request] +on: [push, pull_request] jobs: build: strategy: + fail-fast: false matrix: - test: [sftp-inbox, s3-inbox] + inbox: [posix, s3] + deployment: [federated, standalone] + cert: [issuer, manual] runs-on: ubuntu-latest steps: - - name: ACTIONS_ALLOW_UNSECURE_COMMANDS - id: ACTIONS_ALLOW_UNSECURE_COMMANDS - run: echo 'ACTIONS_ALLOW_UNSECURE_COMMANDS=true' >> $GITHUB_ENV - # Currently the sftp inbox test covers the standalone test, whereas the s3 test checks the federated topology - - name: fixup env - run: | - if [ "${{ matrix.test }}" = "sftp-inbox" ]; then - echo "SVCS=backup doa finalize inbox ingest intercept mapper verify download" >> "$GITHUB_ENV"; - else - echo "SVCS=auth backup doa finalize inbox ingest mapper verify download" >> "$GITHUB_ENV"; - fi - - uses: actions/checkout@v3 - - name: Install kube dependencies - run: bash ./dev_tools/scripts/install-kube-deps.sh - - name: Initialise k3d - run: bash ./dev_tools/scripts/init-k3d.sh - - name: Wait for k3d to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh metrics-server k8s-app kube-system - - name: Install sda dependencies - run: bash ./dev_tools/scripts/install-sda-deps.sh - - name: Create certificates - run: bash ./dev_tools/scripts/make-certs.sh - - name: Create secrets - run: bash ./dev_tools/scripts/create-secrets.sh - - name: Set up services configuration - run: bash ./dev_tools/scripts/svc-setup.sh - - name: Deploy SDA database - run: bash ./dev_tools/scripts/deploy-db.sh - - name: Wait for database to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh database - - name: Deploy mock oidc server - if: matrix.test == 's3-inbox' - run: bash ./dev_tools/scripts/deploy-oidc.sh - - name: Deploy minio - if: matrix.test == 's3-inbox' - run: bash ./dev_tools/scripts/deploy-minio.sh - - name: Wait for minio to become ready - if: matrix.test == 's3-inbox' - run: bash ./dev_tools/scripts/wait-for-pods.sh minio app - - name: Create s3 buckets - if: matrix.test == 's3-inbox' - run: bash ./dev_tools/scripts/create-s3-buckets.sh - - name: Start CEGA services - run: bash ./dev_tools/scripts/deploy-cega.sh - - name: Wait for CEGA to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh cega-mq app - - name: Deploy SDA message broker - run: bash ./dev_tools/scripts/deploy-mq.sh - - name: Wait for broker to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh broker - - name: Deploy the SDA stack for posix - if: matrix.test == 'sftp-inbox' - run: | - kubectl apply -f dev_tools/config/posix-volumes.yaml; - bash ./dev_tools/scripts/sda/deploy-posix-standalone.sh "not-orchestrated"; - - name: Deploy the SDA stack for s3 - if: matrix.test == 's3-inbox' - run: bash ./dev_tools/scripts/sda/deploy-s3-federated.sh - - name: Wait for sda to become ready - run: bash ./dev_tools/scripts/wait-for-pods.sh "${{ env.SVCS }}" - - name: Run helm test - run: bash ./dev_tools/scripts/run-helm-test.sh + - uses: actions/checkout@v2 + + - name: Install kube dependencies + run: bash ./dev_tools/scripts/install-kube-deps.sh + + - name: Initialise k3d + run: bash ./dev_tools/scripts/init-k3d.sh + + - name: Wait for k3d to become ready + run: bash ./dev_tools/scripts/wait-for-pods.sh metrics-server k8s-app kube-system + + - name: Install sda dependencies + run: bash ./dev_tools/scripts/install-sda-deps.sh + + - name: Create certificates + if: matrix.cert == 'manual' + run: bash ./dev_tools/scripts/make-certs.sh + + - name: Create certificate issuer + if: matrix.cert == 'issuer' + run: bash ./dev_tools/scripts/deploy-cert-manager.sh + + - name: Create secrets + run: bash ./dev_tools/scripts/create-secrets.sh + + - name: Set up services configuration + if: matrix.cert == 'manual' + run: bash ./dev_tools/scripts/svc-setup.sh + + - name: Deploy SDA database + run: bash ./dev_tools/scripts/deploy-db.sh "${{ matrix.cert }}" + + - name: Wait for database to become ready + run: bash ./dev_tools/scripts/wait-for-pods.sh database + + - name: Deploy mock oidc server + if: matrix.inbox == 's3' + run: bash ./dev_tools/scripts/deploy-oidc.sh + + - name: Deploy minio + if: matrix.inbox == 's3' + run: bash ./dev_tools/scripts/deploy-minio.sh "${{ matrix.cert }}" + + - name: Wait for minio to become ready + if: matrix.inbox == 's3' + run: bash ./dev_tools/scripts/wait-for-pods.sh minio app + + - name: Create s3 buckets + if: matrix.inbox == 's3' + run: bash ./dev_tools/scripts/create-s3-buckets.sh + + - name: Start CEGA services + if: matrix.deployment == 'federated' + run: bash ./dev_tools/scripts/deploy-cega.sh "${{ matrix.cert }}" + + - name: Wait for CEGA to become ready + if: matrix.deployment == 'federated' + run: bash ./dev_tools/scripts/wait-for-pods.sh cega-mq app + + - name: Deploy SDA message broker + run: bash ./dev_tools/scripts/deploy-mq.sh "${{ matrix.deployment }}" "${{ matrix.cert }}" + + - name: Wait for broker to become ready + run: bash ./dev_tools/scripts/wait-for-pods.sh broker + + - name: Deploy the SDA stack for posix + if: matrix.inbox == 'posix' + run: | + kubectl apply -f dev_tools/config/posix-volumes.yaml; + bash ./dev_tools/scripts/sda/deploy-posix.sh "${{ matrix.deployment }}" "${{ matrix.cert }}" + + - name: Deploy the SDA stack for s3 + if: matrix.inbox == 's3' + run: bash ./dev_tools/scripts/sda/deploy-s3.sh "${{ matrix.deployment }}" "${{ matrix.cert }}" + + - name: Wait for sda to become ready + run: bash ./dev_tools/scripts/wait-for-pods.sh "${{ format('{0}_{1}_svc_list', matrix.deployment, matrix.inbox) }}" + + - name: Run helm test + run: bash ./dev_tools/scripts/run-helm-test.sh diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index a8a99818..5996e659 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -10,13 +10,3 @@ jobs: - uses: actions/checkout@v3 - name: Run ShellCheck uses: ludeeus/action-shellcheck@master - python-scripts: - name: Python scripts linting - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 - - name: Install Black - run: pip install black - - name: Run black --check . - run: black --check . -l 160 diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 8338429b..74a24ce1 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sda-db -version: 0.4.0 +version: "0.4.1" description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io icon: https://neic.no/assets/images/logo.png diff --git a/charts/sda-db/README.md b/charts/sda-db/README.md index 683077fd..6ea07774 100644 --- a/charts/sda-db/README.md +++ b/charts/sda-db/README.md @@ -4,17 +4,19 @@ Source repository: [https://github.com/neicnordic/sda-db](https://github.com/nei ## Installing the Chart -Edit the values.yaml file and specify the relevant parts of the `default` section. +Edit the values.yaml file and specify the relevant parts of the `global` section. Parameter | Description | Default --------- | ----------- | ------- `global.pg_in_password` | Password for `lega_in` user, used for `data in` services. |`""` `global.pg_out_password` | Password for `lega_out` user, used for `data out` services. |`""` `global.tls.enabled` | Enable TLS for all connections. |`true` +`global.tls.issuer` | Issuer for TLS certificate creation. |`""` +`global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. |`""` `global.tls.secretName` | Name of the secret holding the certificates. |`` -`global.tls.certName` | Server certificate. |`postgresql.crt` -`global.tls.keyName` | Server private key. |`postgresql.key` -`global.tls.CAFile` | CA root certificate. |`root.crt` +`global.tls.certName` | Server certificate. |`tls.crt` +`global.tls.keyName` | Server private key. |`tls.key` +`global.tls.CAFile` | CA root certificate. |`ca.crt` `global.tls.verifyPeer` | Require client certificates. |`verify-ca` `externalPkiService.tlsPath` | If an external PKI service is used, this is the path where the certifiates are placed | `""` `image.repository` | sda-db container image repository | `ghcr.io/neicnordic/sda-db` @@ -44,11 +46,13 @@ Parameter | Description | Default ### TLS -Create a secret that contains the certificates +Automatic certificates can be generated by setting `global.tls.issuer` or `glbal.tls.clusterIssuer` if `cert-manger` is installed and configured. + +If certificates are created manually the secret that holds them needs to be created manually. ```cmd -kubectl create secret generic ca-secret \ ---from-file=root.crt=ca.crt \ ---from-file=postgresql.crt=server.crt \ ---from-file=postgresql.key=server.key +kubectl create secret generic db-certs \ +--from-file=ca.crt\ +--from-file=tls.crt\ +--from-file=tls.key ``` diff --git a/charts/sda-db/templates/_helpers.tpl b/charts/sda-db/templates/_helpers.tpl index 588a94ee..18a3cefa 100644 --- a/charts/sda-db/templates/_helpers.tpl +++ b/charts/sda-db/templates/_helpers.tpl @@ -41,23 +41,61 @@ Create chart name and version as used by the chart label. {{- define "pgCert" -}} {{- if .Values.externalPkiService.tlsPath }} {{- printf "%s" (regexReplaceAll "^/*|/+" (printf "%s/%s" .Values.externalPkiService.tlsPath .Values.global.tls.certName) "/")}} + {{- else if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + {{- printf "%s/tls/tls.crt" .Values.persistence.mountPath -}} {{- else }} - {{- printf "%s/tls/%s" .Values.persistence.mountPath .Values.global.tls.certName }} + {{- printf "%s/tls/%s" .Values.persistence.mountPath (required "name of tls certificate is required" .Values.global.tls.certName) }} {{- end -}} {{- end -}} {{- define "pgKey" -}} {{- if .Values.externalPkiService.tlsPath }} {{- printf "%s" (regexReplaceAll "^/*|/+" (printf "%s/%s" .Values.externalPkiService.tlsPath .Values.global.tls.keyName) "/")}} + {{- else if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + {{- printf "%s/tls/tls.key" .Values.persistence.mountPath -}} {{- else }} - {{- printf "%s/tls/%s" .Values.persistence.mountPath .Values.global.tls.keyName }} + {{- printf "%s/tls/%s" .Values.persistence.mountPath (required "name of tls key is required" .Values.global.tls.keyName) }} {{- end -}} {{- end -}} {{- define "caCert" -}} {{- if .Values.externalPkiService.tlsPath }} {{- printf "%s" (regexReplaceAll "^/*|/+" (printf "%s/%s" .Values.externalPkiService.tlsPath .Values.global.tls.CAFile) "/")}} + {{- else if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + {{- printf "%s/tls/ca.crt" .Values.persistence.mountPath -}} {{- else }} - {{- printf "%s/tls/%s" .Values.persistence.mountPath .Values.global.tls.CAFile }} + {{- printf "%s/tls/%s" .Values.persistence.mountPath (required "name of ca file is required" .Values.global.tls.CAFile) }} + {{- end -}} +{{- end -}} + +{{- define "TLSissuer" -}} + {{- if and .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + {{- fail "Only one of global.tls.issuer or global.tls.clusterIssuer should be set" }} + {{- end -}} + + {{- if and .Values.global.tls.issuer }} + {{- printf "%s" .Values.global.tls.issuer }} + {{- else if and .Values.global.tls.clusterIssuer }} + {{- printf "%s" .Values.global.tls.clusterIssuer }} + {{- end -}} +{{- end -}} + +{{- define "TLSsecret" -}} + {{- if and .Values.global.tls.enabled (not .Values.externalPkiService.tlsPath) }} + {{- if and (not .Values.global.tls.issuer) (not .Values.global.tls.clusterIssuer) }} + {{ printf "%s" (required "TLS secret name is required when TLS in enabled without issuer or PKI service" .Values.global.tls.secretName) }} + {{- else }} + {{- printf "%s-certs" (include "sda.fullname" .) }} + {{- end -}} + {{- end -}} +{{- end -}} + +{{- define "testTLSsecret" -}} + {{- if and .Values.global.tls.enabled (not .Values.externalPkiService.tlsPath) }} + {{- if and (not .Values.global.tls.issuer) (not .Values.global.tls.clusterIssuer) }} + {{ printf "%s" (required "TLS secret name is required when TLS in enabled without issuer or PKI service" .Values.testimage.tls.secretName) }} + {{- else }} + {{- printf "%s-test-certs" (include "sda.fullname" .) }} + {{- end -}} {{- end -}} {{- end -}} diff --git a/charts/sda-db/templates/certificate.yaml b/charts/sda-db/templates/certificate.yaml new file mode 100644 index 00000000..4bc11891 --- /dev/null +++ b/charts/sda-db/templates/certificate.yaml @@ -0,0 +1,36 @@ +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }} +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: {{ template "sda.fullname" . }} + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - server auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }} + - {{ template "sda.fullname" . }}.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} diff --git a/charts/sda-db/templates/release-test-deploy.yaml b/charts/sda-db/templates/release-test-deploy.yaml index 8d1dac16..46823314 100644 --- a/charts/sda-db/templates/release-test-deploy.yaml +++ b/charts/sda-db/templates/release-test-deploy.yaml @@ -1,22 +1,9 @@ --- -{{- define "dbfullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - apiVersion: v1 kind: Pod metadata: - name: "{{ .Release.Name }}-test" - resourceVersion: {{ template "dbfullname" . }} + name: "{{ template "sda.fullname" . }}-test" + resourceVersion: {{ template "sda.fullname" . }} annotations: "helm.sh/hook": test "helm.sh/hook-weight": "10" @@ -26,7 +13,7 @@ spec: runAsUser: 70 fsGroup: 70 containers: - - name: {{ .Release.Name }}-test + - name: {{ template "sda.fullname" . }}-test image: {{ printf "%s:%s" .Values.image.repository .Values.image.tag | quote }} imagePullPolicy: {{ .Values.image.pullPolicy | quote }} {{- if and .Values.global.tls.enabled (not .Values.externalPkiService.tlsPath) }} @@ -46,11 +33,11 @@ spec: value: ".postgresql" {{- end }} - name: DB_HOST - value: "{{ template "dbfullname" . }}" + value: "{{ template "sda.fullname" . }}" - name: PGPASSWORD valueFrom: secretKeyRef: - name: {{ template "dbfullname" . }} + name: {{ template "sda.fullname" . }} key: pgInPasswd command: [ "/bin/bash" ] args: @@ -81,7 +68,49 @@ spec: volumes: - name: certs secret: - secretName: {{ .Values.global.tls.secretName }} + secretName: {{ template "testTLSsecret" . }} {{- end }} restartPolicy: Never +--- +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer -}} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-test +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-test-certs + + # The block below requires cert-manger v1.7, needs testing + # secretTemplate: + # annotations: + # "helm.sh/hook": test + # "helm.sh/hook-weight": "10" + # "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded + + duration: 2160h # 90d + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: lega_in + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - client auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }}-test + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end }} diff --git a/charts/sda-db/templates/statefulset.yaml b/charts/sda-db/templates/statefulset.yaml index fe26c910..187a36e8 100644 --- a/charts/sda-db/templates/statefulset.yaml +++ b/charts/sda-db/templates/statefulset.yaml @@ -140,7 +140,7 @@ spec: {{- if and .Values.global.tls.enabled (not .Values.externalPkiService.tlsPath) }} - name: certs secret: - secretName: {{ required "TLS secret name is required when TLS in enabled" .Values.global.tls.secretName }} + secretName: {{ template "TLSsecret" . }} {{- end }} - name: data {{- if and .Values.persistence.enabled .Values.persistence.existingClaim }} diff --git a/charts/sda-db/values.yaml b/charts/sda-db/values.yaml index 32f0f9a9..946ba385 100644 --- a/charts/sda-db/values.yaml +++ b/charts/sda-db/values.yaml @@ -1,12 +1,14 @@ global: - pg_in_password: - pg_out_password: + pg_in_password: "" + pg_out_password: "" tls: enabled: true - secretName: - certName: postgresql.crt - keyName: postgresql.key - CAFile: root.crt + issuer: "" + clusterIssuer: "" + secretName: "" + certName: tls.crt + keyName: tls.key + CAFile: ca.crt # If client verification is to be used set verifyPeer to verify-full or verify-ca # To disable certificate validation set to verify-none verifyPeer: verify-ca @@ -90,3 +92,10 @@ service: port: 5432 updateStrategyType: RollingUpdate + + +# secret containing the TLS certificates for the release tester +# if no certificate issuer is used +testimage: + tls: + secretName: "" diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 630eb4e9..76f8019f 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sda-mq -version: "0.4.1" +version: "0.4.2" description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io icon: https://neic.no/assets/images/logo.png diff --git a/charts/sda-mq/README.md b/charts/sda-mq/README.md index f0fb4939..f539e9d8 100644 --- a/charts/sda-mq/README.md +++ b/charts/sda-mq/README.md @@ -1,6 +1,6 @@ # SDA Message broker -Source repository: https://github.com/neicnordic/sda-mq +Source repository: [https://github.com/neicnordic/sda-mq](https://github.com/neicnordic/sda-mq) ## Installing the Chart @@ -11,20 +11,22 @@ Parameter | Description | Default `image.repository` | sda-mq container image repository | `ghcr.io/neicnordic/sda-mq` `image.tag` | sda-mq container image version | `v1.3.0` `image.pullPolicy` | sda-mq container image pull policy | `Always` -`config.adminUser` | Username of admin user |`""` -`config.adminPasswordHash` | Passwordhash for admin user. |`""` -`config.tls.enabled` | Use TLS for all connections. |`true` -`config.tls.secretName` | Name of the secret that holds the certificates. |`""` -`config.tls.serverKey` | Name of the certificate private key file. |`""` -`config.tls.serverCert` | Name of the certificate file. |`""` -`config.tls.caCert` | Name of the CA file. |`""` -`config.tls.verifyPeer` | Require client certificates. |`true` -`config.vhost` | default vhost is '/' unless specifically named |`""` -`config.shovel.host` | Hostname of federated server |`""` -`config.shovel.pass` | Password to federated server |`""` -`config.shovel.port` | Port that federated server listens on |`5671` -`config.shovel.user` | Username to federated server |`""` -`config.shovel.vhost` | Vhost on federated sever to connect to |`""` +`global.adminUser` | Username of admin user |`""` +`global.adminPasswordHash` | Passwordhash for admin user. |`""` +`global.tls.enabled` | Use TLS for all connections. |`true` +`global.tls.issuer` | Issuer for TLS certificate creation. |`""` +`global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. |`""` +`global.tls.secretName` | Name of the secret that holds the certificates. |`""` +`global.tls.serverKey` | Name of the certificate private key file. |`""` +`global.tls.serverCert` | Name of the certificate file. |`""` +`global.tls.caCert` | Name of the CA file. |`""` +`global.tls.verifyPeer` | Require client certificates. |`true` +`global.vhost` | default vhost is '/' unless specifically named |`""` +`global.shovel.host` | Hostname of federated server |`""` +`global.shovel.pass` | Password to federated server |`""` +`global.shovel.port` | Port that federated server listens on |`5671` +`global.shovel.user` | Username to federated server |`""` +`global.shovel.vhost` | Vhost on federated sever to connect to |`""` `externalPkiService.tlsPath` | If an external PKI service is used, this is the path where the certifiates are placed | `""` `rbacEnabled` | Use role based access control. |`true` `revisionHistory` | Number of revisions to keep for the option to rollback a deployment | `3` @@ -50,22 +52,24 @@ Parameter | Description | Default ### TLS -Create a secret that contains the certificates +Automatic certificates can be generated by setting `global.tls.issuer` or `glbal.tls.clusterIssuer` if `cert-manger` is installed and configured. + +If certificates are created manually the secret that holds them needs to be created manually. ```cmd -kubectl create secret generic mq-certs \ ---from-file=ca.crt \ ---from-file=server.crt \ ---from-file=server.key +kubectl create secret generic ca-secret \ +--from-file=ca.crt\ +--from-file=tls.crt\ +--from-file=tls.key ``` If helm tests are to be used the testers client certificates must be accessible as a secret. ```cmd kubectl create secret generic tester-certs \ ---from-file=ca.crt \ ---from-file=tester.crt \ ---from-file=tester.key +--from-file=ca.crt\ +--from-file=tls.crt\ +--from-file=tls.key ``` ## Password hash diff --git a/charts/sda-mq/templates/_helpers.tpl b/charts/sda-mq/templates/_helpers.tpl index d015fc4c..dbc94bd0 100644 --- a/charts/sda-mq/templates/_helpers.tpl +++ b/charts/sda-mq/templates/_helpers.tpl @@ -31,7 +31,7 @@ Create chart name and version as used by the chart label. {{- end -}} {{- define "verifyPeer" -}} - {{ if .Values.config.tls.verifyPeer }} + {{ if and .Values.global.tls.enabled .Values.global.tls.verifyPeer }} {{- print "verify_peer" -}} {{ else }} {{- print "verify_none" -}} @@ -40,24 +40,62 @@ Create chart name and version as used by the chart label. {{- define "mqCert" -}} {{- if .Values.externalPkiService.tlsPath }} - {{- printf "%s" (regexReplaceAll "^/*|/+" (printf "%s/%s" .Values.externalPkiService.tlsPath (required "a TLS certificate is required" .Values.config.tls.serverCert)) "/")}} + {{- printf "%s" (regexReplaceAll "^/*|/+" (printf "%s/%s" .Values.externalPkiService.tlsPath .Values.global.tls.certName) "/")}} + {{- else if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + {{- print "/etc/rabbitmq/tls/tls.crt" -}} {{- else }} - {{- printf "/etc/rabbitmq/tls/%s" (required "a TLS certificate is required" .Values.config.tls.serverCert) }} + {{- printf "/etc/rabbitmq/tls/%s" (required "name of tls certificate is required" .Values.global.tls.certName) }} {{- end -}} {{- end -}} {{- define "mqKey" -}} {{- if .Values.externalPkiService.tlsPath }} - {{- printf "%s" (regexReplaceAll "^/*|/+" (printf "%s/%s" .Values.externalPkiService.tlsPath (required "a TLS key is required" .Values.config.tls.serverKey)) "/")}} + {{- printf "%s" (regexReplaceAll "^/*|/+" (printf "%s/%s" .Values.externalPkiService.tlsPath .Values.global.tls.keyName) "/")}} + {{- else if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + {{- printf "/etc/rabbitmq/tls/tls.key" -}} {{- else }} - {{- printf "/etc/rabbitmq/tls/%s" (required "a TLS key is required" .Values.config.tls.serverKey) }} + {{- printf "/etc/rabbitmq/tls/%s" (required "name of tls key is required" .Values.global.tls.keyName) }} {{- end -}} {{- end -}} {{- define "caCert" -}} {{- if .Values.externalPkiService.tlsPath }} - {{- printf "%s" (regexReplaceAll "^/*|/+" (printf "%s/%s" .Values.externalPkiService.tlsPath (required "a CA certificate is required" .Values.config.tls.caCert)) "/")}} + {{- printf "%s" (regexReplaceAll "^/*|/+" (printf "%s/%s" .Values.externalPkiService.tlsPath .Values.global.tls.caCert) "/")}} + {{- else if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + {{- printf "/etc/rabbitmq/tls/ca.crt" -}} {{- else }} - {{- printf "/etc/rabbitmq/tls/%s" (required "a CA certificate is required" .Values.config.tls.caCert) }} + {{- printf "/etc/rabbitmq/tls/%s" (required "name of ca file is required" .Values.global.tls.caCert) }} + {{- end -}} +{{- end -}} + +{{- define "TLSissuer" -}} + {{- if and .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + {{- fail "Only one of global.tls.issuer or global.tls.clusterIssuer should be set" }} + {{- end -}} + + {{- if and .Values.global.tls.issuer }} + {{- printf "%s" .Values.global.tls.issuer }} + {{- else if and .Values.global.tls.clusterIssuer }} + {{- printf "%s" .Values.global.tls.clusterIssuer }} + {{- end -}} +{{- end -}} + +{{- define "TLSsecret" -}} + {{- if and .Values.global.tls.enabled (not .Values.externalPkiService.tlsPath) }} + {{- if and (not .Values.global.tls.issuer) (not .Values.global.tls.clusterIssuer) }} + {{ printf "%s" (required "TLS secret name is required when TLS in enabled without issuer or PKI service" .Values.global.tls.secretName) }} + {{- else }} + {{- printf "%s-certs" (include "sda.fullname" .) }} + {{- end -}} + {{- end -}} +{{- end -}} + +{{- define "testTLSsecret" -}} + {{- if and .Values.global.tls.enabled (not .Values.externalPkiService.tlsPath) }} + {{- if and (not .Values.global.tls.issuer) (not .Values.global.tls.clusterIssuer) }} + {{ printf "%s" (required "TLS secret name is required when TLS in enabled without issuer or PKI service" .Values.testimage.tls.secretName) }} + {{- else }} + {{- printf "%s-test-certs" (include "sda.fullname" .) }} + {{- end -}} {{- end -}} {{- end -}} diff --git a/charts/sda-mq/templates/certificate.yaml b/charts/sda-mq/templates/certificate.yaml new file mode 100644 index 00000000..4bc11891 --- /dev/null +++ b/charts/sda-mq/templates/certificate.yaml @@ -0,0 +1,36 @@ +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }} +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: {{ template "sda.fullname" . }} + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - server auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }} + - {{ template "sda.fullname" . }}.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} diff --git a/charts/sda-mq/templates/release-test-deploy.yaml b/charts/sda-mq/templates/release-test-deploy.yaml index 969281e8..7901527a 100644 --- a/charts/sda-mq/templates/release-test-deploy.yaml +++ b/charts/sda-mq/templates/release-test-deploy.yaml @@ -24,14 +24,14 @@ metadata: spec: containers: - name: {{ .Release.Name }}-test - {{- if .Values.config.tls.enabled }} + {{- if .Values.global.tls.enabled }} image: "{{ .Values.testimage.repository }}:{{ .Values.testimage.tag }}" {{- else }} image: "subfuzion/netcat:latest" {{- end }} imagePullPolicy: {{ .Values.testimage.pullPolicy | quote }} volumeMounts: - {{- if and .Values.config.tls.enabled (not .Values.externalPkiService.tlsPath) }} + {{- if and .Values.global.tls.enabled (not .Values.externalPkiService.tlsPath) }} - name: certs mountPath: /certs {{- end }} @@ -46,7 +46,7 @@ spec: args: - "-x" - "-c" - {{- if .Values.config.tls.enabled }} + {{- if .Values.global.tls.enabled }} - 'P=${PKI_VOLUME_PATH:-/certs}; count=1; until openssl s_client -connect "${MQ_HOST}:5671" -verify 50 -key "$P/{{ .Values.testimage.tls.tlsKey }}" -cert "$P/{{ .Values.testimage.tls.tlsCert }}" -verify_return_error -CAfile "$P/{{ .Values.testimage.tls.caCert }}" = 1.19.0" description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 5b5c918f..0f31e742 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -1,9 +1,10 @@ # SDA services Source repositories: -- https://github.com/neicnordic/sda-pipeline -- https://github.com/neicnordic/sda-doa -- https://github.com/neicnordic/sda-download + +- [https://github.com/neicnordic/sda-pipeline](https://github.com/neicnordic/sda-pipeline) +- [https://github.com/neicnordic/sda-doa](https://github.com/neicnordic/sda-doa) +- [https://github.com/neicnordic/sda-download](https://github.com/neicnordic/sda-download) ## Installing the Chart @@ -17,9 +18,9 @@ The following table lists the configurable parameters of the `sda-svc` chart and Parameter | Description | Default --------- | ----------- | ------- `global.secretsPath` | Path where the sensitive files can be found | `/.secrets` -`global.c4ghPath` | This path will be a subpath to the secretsPath | `""` -`global.tlsPath` | This path will be a subpath to the secretsPath | `""` -`global.jwtPath` | This path will be a subpath to the secretsPath | `""` +`global.c4ghPath` | This path will be a subpath to the secretsPath | `c4gh` +`global.tlsPath` | This path will be a subpath to the secretsPath | `tls` +`global.jwtPath` | This path will be a subpath to the secretsPath | `jwt` `global.confFile` | Name of config file, used when secrets are handled by hasicorp vault | `config.yaml` `global.confFilePath` | This path will be a subpath to the secretsPath | `""` `global.deploymentType` | Deployment can be split into `external` and `internal` components, available options are `all`, `external` and `internal`. | `all` @@ -31,14 +32,16 @@ Parameter | Description | Default `global.ingress.secretNames.auth` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` `global.ingress.secretNames.doa` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` `global.ingress.secretNames.s3Inbox` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` +`global.ingress.clusterIssuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured clusterIssuer can be specified to automate certificate configuration for the ingress endpoint. | `""` `global.ingress.issuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured issuer can be specified to automate certificate configuration for the ingress endpoint. | `""` +`global.ingress.annotations` | extra annotations for the ingress objects | `""` +`global.ingress.ingressClassName` | Class of ingress controller to use | `nginx` `global.logLevel` | Log level for all services. | `info` `global.networkPolicy.create` | Use network isolation. | `false` `global.networkPolicy.brokerNamespace` | Namespace where the broker is deployed. | `""` `global.networkPolicy.databaseNamespace` | Namespace where the database is deployed. | `""` `global.networkPolicy.externalNamespace` | Namespace where the external components are deployed. | `""` `global.networkPolicy.internalNamespace` | Namespace where the internal components are deployed. | `""` -`global.persistence.enabled` | Enable persistent datastorage | `true` `global.revisionHistory` | Number of revisions to keep for the option to rollback a deployment | `3` `global.podAnnotations` | Annotations applied to pods of all services. |`{}` `global.pkiService` | If an external PKI infrastructure is used set this to true. |`false` @@ -92,6 +95,7 @@ Parameter | Description | Default `global.db.passOutgest` | Password used for `data out` services. |`""` `global.db.port` | Port that the database is listening on. |`5432` `global.db.sslMode` | SSL mode for the database connection, options are `verify-ca` or `verify-full`. |`verify-full` +`global.doa.enabled` | Deploy the DOA service | `false` `global.doa.envFile` | File to source when credentials are managed by Hasicorp vault | `env` `global.doa.serviceport` | Port that the DOA service is accessible on | `443` `global.doa.outbox.enabled` | Enable Outbox functionality of Data Out API | `false` @@ -105,7 +109,16 @@ Parameter | Description | Default `global.doa.outbox.s3CaFile` | Outbox S3 CA certificate to use | `null` `global.doa.outbox.s3AccessKey` | Outbox S3 Access Key | `null` `global.doa.outbox.s3SecretKey` | Outbox S3 Secret key | `null` -`global.elixir.oidcdHost` | URL to get the public key used to verify Elixir JWT. | `"https://login.elixir-czech.org/oidc/"` +`global.download.enabled` | Deploy the download service | `true` +`global.download.sessionExpiration` | Session key expiration time in seconds | `28800` +`global.download.trusted.configPath` | Path to the ISS config file | `$secrets/iss` +`global.download.trusted.configFile` | Name of ISS config file | `iss.json` +`global.download.trusted.iss` | Array of trusted OIDC endpoints | `` +`global.download.trusted.iss[iss]` | URI to the OIDC service | `https://login.elixir-czech.org/oidc/` +`global.download.trusted.iss[jku]` | The URI to the OIDCs jwk endpoint | `https://login.elixir-czech.org/oidc/jwk` + +`global.elixir.oidcdHost` | URL to the OIDc service. | `"https://login.elixir-czech.org/oidc/"` +`global.elixir.jwkPath` | Public key path on the OIDC host. | `jwk` `global.inbox.servicePort` | The port that the inbox is accessible via. | `2222` `global.inbox.storageType` | Storage type for the inbox, available options are `s3` and `posix`. |`posix` `global.inbox.path` | Path to the mounted `posix` volume. |`/inbox` @@ -121,6 +134,9 @@ Parameter | Description | Default `global.inbox.s3SecretKey` | Secret key to S3 inbox. |`null` `global.inbox.s3CaFile` | CA certificate to use if the S3 inbox is internal. |`null` `global.inbox.s3ReadyPath` | Endpoint to verify that the inbox is respondig. |`""` +`global.tls.enabled` | Use TLS for all connections. |`true` +`global.tls.issuer` | Issuer for TLS certificate creation. |`""` +`global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. |`""` ### Credentials diff --git a/charts/sda-svc/templates/_helpers.yaml b/charts/sda-svc/templates/_helpers.yaml index 5b989118..f753f21d 100644 --- a/charts/sda-svc/templates/_helpers.yaml +++ b/charts/sda-svc/templates/_helpers.yaml @@ -298,3 +298,15 @@ Create chart name and version as used by the chart label. {{- printf "%s" .Values.global.inbox.s3Url }} {{- end }} {{- end -}} + +{{- define "TLSissuer" -}} + {{- if and .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + {{- fail "Only one of global.tls.issuer or global.tls.clusterIssuer should be set" }} + {{- end -}} + + {{- if .Values.global.tls.issuer }} + {{- printf "%s" .Values.global.tls.issuer }} + {{- else if and .Values.global.tls.clusterIssuer }} + {{- printf "%s" .Values.global.tls.clusterIssuer }} + {{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/auth-certificate.yaml b/charts/sda-svc/templates/auth-certificate.yaml new file mode 100644 index 00000000..f9693d63 --- /dev/null +++ b/charts/sda-svc/templates/auth-certificate.yaml @@ -0,0 +1,38 @@ +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +{{- if eq "s3" .Values.global.inbox.storageType }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-auth-certs +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-auth-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: {{ template "sda.fullname" . }}-auth + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - server auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }}-auth + - {{ template "sda.fullname" . }}-auth.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/auth-deploy.yaml b/charts/sda-svc/templates/auth-deploy.yaml index 03c3d35e..cf2da992 100644 --- a/charts/sda-svc/templates/auth-deploy.yaml +++ b/charts/sda-svc/templates/auth-deploy.yaml @@ -87,13 +87,21 @@ spec: value: "{{ .Values.global.elixir.oidcdHost }}" - name: ELIXIR_SCOPE value: "ga4gh_passport_v1" + - name: ELIXIR_JWTPRIVATEKEY + value: "{{ template "jwtPath" . }}/{{ .Values.global.auth.jwtKey }}" + - name: ELIXIR_JWTSIGNATUREALG + value: {{ .Values.global.auth.jwtAlg }} {{- if or ( eq "federated" .Values.global.schemaType) ( eq "" .Values.global.schemaType) }} - name: ELIXIR_JWTPRIVATEKEY value: "{{ template "jwtPath" . }}/{{ .Values.global.auth.jwtKey }}" - name: ELIXIR_JWTSIGNATUREALG value: {{ .Values.global.auth.jwtAlg | quote }} - name: CEGA_JWTISSUER + {{- if .Values.global.tls.enabled }} value: "https://{{ .Values.global.ingress.hostName.auth }}" + {{- else }} + value: "http://{{ .Values.global.ingress.hostName.auth }}" + {{- end }} - name: CEGA_AUTHURL value: {{ .Values.global.cega.host | quote }} - name: CEGA_JWTPRIVATEKEY @@ -138,40 +146,35 @@ spec: volumeMounts: {{- if and .Values.global.pkiPermissions .Values.global.tls.enabled }} - name: tls - mountPath: {{ include "tlsPath" . }} + mountPath: {{ template "tlsPath" . }} {{- end }} - {{- if and .Values.global.auth.jwtKey .Values.global.jwtPath }} + {{- if not .Values.global.vaultSecrets }} - name: jwt - mountPath: {{ include "jwtPath" . }} + mountPath: {{ template "jwtPath" . }} {{- end }} volumes: - {{- if and .Values.global.auth.jwtKey .Values.global.jwtPath }} + {{- if not .Values.global.vaultSecrets }} - name: jwt projected: defaultMode: 0440 sources: - secret: - name: {{ .Values.global.auth.jwtSecret }} + name: {{ required "A secret for the JWT signing key is needed" .Values.global.auth.jwtSecret }} items: - - key: {{ .Values.global.auth.jwtKey }} + - key: {{ required "The name of the JWT signing key is needed" .Values.global.auth.jwtKey }} path: {{ .Values.global.auth.jwtKey }} {{- end }} - {{- if .Values.global.tls.enabled }} + {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls projected: sources: + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} - secret: - name: {{ required "TLS secret name is requrired for auth" .Values.auth.tls.secretName }} - {{- if .Values.global.tls.caCert.secretName }} - - secret: - name: {{ .Values.global.tls.caCert.secretName }} - items: - - key: {{ required "A name for the CA cert is required" .Values.global.tls.caCert.secretKey }} - path: ca.crt - {{- else }} + name: {{ template "sda.fullname" . }}-auth-certs + {{- else }} - secret: - name: {{ template "sda.fullname" . }}-caCert - {{- end }} + name: {{ required "An certificate issuer or a TLS secret name is required for auth" .Values.auth.tls.secretName }} + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/auth-ingress.yaml b/charts/sda-svc/templates/auth-ingress.yaml index f7418526..66a9c222 100644 --- a/charts/sda-svc/templates/auth-ingress.yaml +++ b/charts/sda-svc/templates/auth-ingress.yaml @@ -13,16 +13,22 @@ metadata: {{ toYaml .Values.global.ingress.labels | indent 4 }} {{- end }} annotations: - kubernetes.io/ingress.class: "nginx" + {{- if eq "nginx" .Values.global.ingress.ingressClassName }} nginx.ingress.kubernetes.io/rewrite-target: "/" nginx.ingress.kubernetes.io/backend-protocol: "{{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }}" - {{- if .Values.global.ingress.issuer }} - cert-manager.io/cluster-issuer: {{ .Values.global.ingress.issuer | quote }} {{- end }} -{{- if .Values.global.ingress.annotations }} -{{ toYaml .Values.global.ingress.annotations | indent 4 }} + {{- if .Values.global.ingress.clusterIssuer }} + cert-manager.io/cluster-issuer: {{ .Values.global.ingress.clusterIssuer | quote }} + {{- else if .Values.global.ingress.issuer }} + cert-manager.io/issuer: {{ .Values.global.ingress.issuer | quote }} + {{- end }} +{{- if .Values.global.ingress.annotations }} +{{ toYaml .Values.global.ingress.annotations | indent 4 }} {{- end }} spec: +{{- if .Values.global.ingress.ingressClassName }} + ingressClassName: {{ .Values.global.ingress.ingressClassName }} +{{- end }} rules: - host: {{ required "An ingress hostname is required!" .Values.global.ingress.hostName.auth }} http: diff --git a/charts/sda-svc/templates/backup-certificate.yaml b/charts/sda-svc/templates/backup-certificate.yaml new file mode 100644 index 00000000..44753ace --- /dev/null +++ b/charts/sda-svc/templates/backup-certificate.yaml @@ -0,0 +1,38 @@ +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +{{- if .Values.backup.deploy}} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-backup-certs +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-backup-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: {{ template "sda.fullname" . }}-backup + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - client auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }}-backup + - {{ template "sda.fullname" . }}-backup.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/backup-deploy.yaml b/charts/sda-svc/templates/backup-deploy.yaml index 3ae433c2..2860a26a 100644 --- a/charts/sda-svc/templates/backup-deploy.yaml +++ b/charts/sda-svc/templates/backup-deploy.yaml @@ -148,11 +148,9 @@ spec: {{- end }} {{- end }} {{- if .Values.global.tls.enabled }} - {{- if or (eq "verify-ca" .Values.global.db.sslMode) (eq "verify-full" .Values.global.db.sslMode) }} - name: DB_CACERT value: {{ include "tlsPath" . }}/ca.crt - {{- end }} - {{- if eq "verify-full" .Values.global.db.sslMode }} + {{- if ne "verify-none" .Values.global.db.sslMode }} - name: DB_CLIENTCERT value: {{ include "tlsPath" . }}/tls.crt - name: DB_CLIENTKEY @@ -242,19 +240,12 @@ spec: - name: {{ ternary "tls" "tls-certs" (empty .Values.global.pkiPermissions) }} projected: sources: - {{- if or .Values.global.broker.verifyPeer (eq "verify-full" .Values.global.db.sslMode) }} - - secret: - name: {{ required "TLS secret name is requrired for backup" .Values.backup.tls.secretName }} - {{- end }} - {{- if .Values.global.tls.caCert.secretName }} + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} - secret: - name: {{ .Values.global.tls.caCert.secretName }} - items: - - key: {{ required "A name for the CA cert is required" .Values.global.tls.caCert.secretKey }} - path: ca.crt + name: {{ template "sda.fullname" . }}-backup-certs {{- else }} - - secret: - name: {{ template "sda.fullname" . }}-caCert + - secret: + name: {{ required "An certificate issuer or a TLS secret name is required for backup" .Values.backup.tls.secretName }} {{- end }} {{- if .Values.global.pkiPermissions }} - name: tls diff --git a/charts/sda-svc/templates/doa-certificate.yaml b/charts/sda-svc/templates/doa-certificate.yaml new file mode 100644 index 00000000..bbabde2e --- /dev/null +++ b/charts/sda-svc/templates/doa-certificate.yaml @@ -0,0 +1,39 @@ +{{- if .Values.global.doa.enabled }} +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-doa-certs +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-doa-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: {{ template "sda.fullname" . }}-doa + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - client auth + - server auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }}-doa + - {{ template "sda.fullname" . }}-doa.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/doa-deploy.yaml b/charts/sda-svc/templates/doa-deploy.yaml index 4880ca03..9e8aad80 100644 --- a/charts/sda-svc/templates/doa-deploy.yaml +++ b/charts/sda-svc/templates/doa-deploy.yaml @@ -1,5 +1,6 @@ {{- if or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }} {{- if .Values.global.tls.enabled }} +{{- if .Values.global.doa.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -269,18 +270,13 @@ spec: - name: tls-certs projected: sources: + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} - secret: - name: {{ required "TLS secret name is requrired for DOA" .Values.doa.tls.secretName }} - {{- if .Values.global.tls.caCert.secretName }} + name: {{ template "sda.fullname" . }}-doa-certs + {{- else }} - secret: - name: {{ .Values.global.tls.caCert.secretName }} - items: - - key: {{ required "A name for the CA cert is required" .Values.global.tls.caCert.secretKey }} - path: ca.crt - {{- else }} - - secret: - name: {{ template "sda.fullname" . }}-cacert - {{- end }} + name: {{ required "An certificate issuer or a TLS secret name is required for doa" .Values.doa.tls.secretName }} + {{- end }} {{- end }} {{- if not .Values.global.vaultSecrets }} {{- if .Values.global.elixir.pubKey }} @@ -312,3 +308,4 @@ spec: {{- end }} {{- end }} {{- end }} +{{- end }} diff --git a/charts/sda-svc/templates/doa-ingress.yaml b/charts/sda-svc/templates/doa-ingress.yaml index 0ecff2ee..64d4782c 100644 --- a/charts/sda-svc/templates/doa-ingress.yaml +++ b/charts/sda-svc/templates/doa-ingress.yaml @@ -1,5 +1,6 @@ {{- if (or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType)) (not .Values.global.deploymentType)) }} {{- if and .Values.global.ingress.deploy .Values.global.tls.enabled }} +{{- if .Values.global.doa.enabled }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -13,16 +14,22 @@ metadata: {{ toYaml .Values.global.ingress.labels | indent 4 }} {{- end }} annotations: - kubernetes.io/ingress.class: "nginx" + {{- if eq "nginx" .Values.global.ingress.ingressClassName }} nginx.ingress.kubernetes.io/rewrite-target: "/" nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" - {{- if .Values.global.ingress.issuer }} - cert-manager.io/cluster-issuer: {{ .Values.global.ingress.issuer | quote }} {{- end }} -{{- if .Values.global.ingress.annotations }} -{{ toYaml .Values.global.ingress.annotations | indent 4 }} + {{- if .Values.global.ingress.clusterIssuer }} + cert-manager.io/cluster-issuer: {{ .Values.global.ingress.clusterIssuer | quote }} + {{- else if .Values.global.ingress.issuer }} + cert-manager.io/issuer: {{ .Values.global.ingress.issuer | quote }} + {{- end }} +{{- if .Values.global.ingress.annotations }} +{{ toYaml .Values.global.ingress.annotations | indent 4 }} {{- end }} spec: +{{- if .Values.global.ingress.ingressClassName }} + ingressClassName: {{ .Values.global.ingress.ingressClassName }} +{{- end }} rules: - host: {{ required "An ingress hostname is required!" .Values.global.ingress.hostName.doa }} http: @@ -40,3 +47,4 @@ spec: secretName: {{ if .Values.global.ingress.secretNames.doa }}{{ .Values.global.ingress.secretNames.doa }}{{- else }}"{{ template "sda.fullname" . }}-ingress-doa"{{- end }} {{- end }} {{- end }} +{{- end }} diff --git a/charts/sda-svc/templates/doa-secrets.yaml b/charts/sda-svc/templates/doa-secrets.yaml index ab0bbe39..c7fb633a 100644 --- a/charts/sda-svc/templates/doa-secrets.yaml +++ b/charts/sda-svc/templates/doa-secrets.yaml @@ -1,5 +1,7 @@ {{- if or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }} -{{- if and (not .Values.global.vaultSecrets) .Values.global.tls.enabled }} +{{- if and (not .Values.global.vaultSecrets) .Values.global.tls.enabled }} +{{- if .Values.global.doa.enabled }} + apiVersion: v1 kind: Secret metadata: @@ -17,3 +19,4 @@ data: {{- end }} {{- end }} {{- end }} +{{- end }} diff --git a/charts/sda-svc/templates/doa-service.yaml b/charts/sda-svc/templates/doa-service.yaml index 3546bcd1..23a66597 100644 --- a/charts/sda-svc/templates/doa-service.yaml +++ b/charts/sda-svc/templates/doa-service.yaml @@ -1,5 +1,6 @@ {{- if or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }} -{{- if .Values.global.tls.enabled }} +{{- if and .Values.global.tls.enabled .Values.global.doa.enabled }} +{{- if .Values.global.doa.enabled }} apiVersion: v1 kind: Service metadata: @@ -16,3 +17,4 @@ spec: app: {{ template "sda.fullname" . }}-doa {{- end }} {{- end }} +{{- end }} diff --git a/charts/sda-svc/templates/download-certificate.yaml b/charts/sda-svc/templates/download-certificate.yaml new file mode 100644 index 00000000..2455fcc5 --- /dev/null +++ b/charts/sda-svc/templates/download-certificate.yaml @@ -0,0 +1,39 @@ +{{- if .Values.global.download.enabled }} +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-download-certs +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-download-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: {{ template "sda.fullname" . }}-download + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - client auth + - server auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }}-download + - {{ template "sda.fullname" . }}-download.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/download-deploy.yaml b/charts/sda-svc/templates/download-deploy.yaml index a20c4a82..e6b225af 100644 --- a/charts/sda-svc/templates/download-deploy.yaml +++ b/charts/sda-svc/templates/download-deploy.yaml @@ -1,4 +1,5 @@ {{- if or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }} +{{- if .Values.global.download.enabled }} apiVersion: apps/v1 kind: Deployment metadata: @@ -67,10 +68,14 @@ spec: command: ["sda-download"] env: - name: ARCHIVE_TYPE - {{- if eq "s3" .Values.global.archive.storageType }} + {{- if eq "s3" .Values.global.archive.storageType }} value: "s3" - name: ARCHIVE_URL value: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} + {{- if .Values.global.archive.s3Port }} + - name: ARCHIVE_PORT + value: {{ .Values.global.archive.s3Port | quote }} + {{- end }} - name: ARCHIVE_BUCKET value: {{ required "S3 archive bucket missing" .Values.global.archive.s3Bucket }} - name: ARCHIVE_REGION @@ -94,15 +99,20 @@ spec: {{- end }} - name: C4GH_FILEPATH value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }}" - {{- if or (eq "verify-ca" .Values.global.db.sslMode) (eq "verify-full" .Values.global.db.sslMode) }} + {{- if .Values.global.tls.enabled }} - name: DB_CACERT value: {{ include "tlsPath" . }}/ca.crt - {{- end }} - {{- if eq "verify-full" .Values.global.db.sslMode }} + {{- if ne "verify-none" .Values.global.db.sslMode }} - name: DB_CLIENTCERT value: {{ include "tlsPath" . }}/tls.crt - name: DB_CLIENTKEY value: {{ include "tlsPath" . }}/tls.key + {{- end }} + - name: DB_SSLMODE + value: {{ .Values.global.db.sslMode | quote }} + {{- else }} + - name: DB_SSLMODE + value: "disable" {{- end }} - name: DB_DATABASE value: {{ default "lega" .Values.global.db.name | quote }} @@ -110,8 +120,6 @@ spec: value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - name: DB_PORT value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ .Values.global.db.sslMode | quote }} {{- if .Values.global.logLevel }} - name: LOG_LEVEL value: {{ .Values.global.logLevel | quote }} @@ -123,6 +131,9 @@ spec: value: "{{ template "tlsPath" . }}/tls.crt" - name: APP_SERVERKEY value: "{{ template "tlsPath" . }}/tls.key" + {{- else }} + - name: SESSION_SECURE + value: "false" {{- end }} - name: SESSION_DOMAIN value: {{ .Values.global.ingress.hostName.download | quote }} @@ -206,19 +217,12 @@ spec: - name: {{ ternary "tls" "tls-certs" (empty .Values.global.pkiPermissions) }} projected: sources: - {{- if or .Values.global.broker.verifyPeer (eq "verify-full" .Values.global.db.sslMode) }} + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} - secret: - name: {{ required "TLS secret name is requrired for download" .Values.download.tls.secretName }} - {{- end }} - {{- if .Values.global.tls.caCert.secretName }} - - secret: - name: {{ .Values.global.tls.caCert.secretName }} - items: - - key: {{ required "A name for the CA cert is required" .Values.global.tls.caCert.secretKey }} - path: ca.crt + name: {{ template "sda.fullname" . }}-download-certs {{- else }} - - secret: - name: {{ template "sda.fullname" . }}-cacert + - secret: + name: {{ required "An certificate issuer or a TLS secret name is required for download" .Values.download.tls.secretName }} {{- end }} {{- if .Values.global.pkiPermissions }} - name: tls @@ -256,3 +260,4 @@ spec: {{- end }} restartPolicy: Always {{- end }} +{{- end }} diff --git a/charts/sda-svc/templates/download-ingress.yaml b/charts/sda-svc/templates/download-ingress.yaml index 5f75ecf6..61e01d06 100644 --- a/charts/sda-svc/templates/download-ingress.yaml +++ b/charts/sda-svc/templates/download-ingress.yaml @@ -1,5 +1,5 @@ {{- if (or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType)) (not .Values.global.deploymentType)) }} -{{- if .Values.global.ingress.deploy }} +{{- if and .Values.global.ingress.deploy .Values.global.download.enabled }} apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -13,16 +13,22 @@ metadata: {{ toYaml .Values.global.ingress.labels | indent 4 }} {{- end }} annotations: - kubernetes.io/ingress.class: "nginx" + {{- if eq "nginx" .Values.global.ingress.ingressClassName }} nginx.ingress.kubernetes.io/rewrite-target: "/" nginx.ingress.kubernetes.io/backend-protocol: "{{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }}" - {{- if .Values.global.ingress.issuer }} - cert-manager.io/cluster-issuer: {{ .Values.global.ingress.issuer | quote }} {{- end }} -{{- if .Values.global.ingress.annotations }} -{{ toYaml .Values.global.ingress.annotations | indent 4 }} + {{- if .Values.global.ingress.clusterIssuer }} + cert-manager.io/cluster-issuer: {{ .Values.global.ingress.clusterIssuer | quote }} + {{- else if .Values.global.ingress.issuer }} + cert-manager.io/issuer: {{ .Values.global.ingress.issuer | quote }} + {{- end }} +{{- if .Values.global.ingress.annotations }} +{{ toYaml .Values.global.ingress.annotations | indent 4 }} {{- end }} spec: +{{- if .Values.global.ingress.ingressClassName }} + ingressClassName: {{ .Values.global.ingress.ingressClassName }} +{{- end }} rules: - host: {{ required "An ingress hostname is required!" .Values.global.ingress.hostName.download }} http: @@ -33,10 +39,12 @@ spec: service: name: {{ template "sda.fullname" . }}-download port: - number: {{ ternary 443 8080 .Values.global.tls.enabled }} + number: {{ ternary 443 80 .Values.global.tls.enabled }} +{{- if .Values.global.tls.enabled }} tls: - hosts: - {{ required "An ingress hostname is required!" .Values.global.ingress.hostName.download }} secretName: {{ if .Values.global.ingress.secretNames.download }}{{ .Values.global.ingress.secretNames.download }}{{- else }}"{{ template "sda.fullname" . }}-ingress-download"{{- end }} {{- end }} {{- end }} +{{- end }} diff --git a/charts/sda-svc/templates/download-secrets.yaml b/charts/sda-svc/templates/download-secrets.yaml index acbd402f..141e60e5 100644 --- a/charts/sda-svc/templates/download-secrets.yaml +++ b/charts/sda-svc/templates/download-secrets.yaml @@ -1,5 +1,5 @@ {{- if or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }} -{{- if not .Values.global.vaultSecrets }} +{{- if and .Values.global.download.enabled (not .Values.global.vaultSecrets) }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/sda-svc/templates/download-service.yaml b/charts/sda-svc/templates/download-service.yaml index 3d98f230..1f2a1d94 100644 --- a/charts/sda-svc/templates/download-service.yaml +++ b/charts/sda-svc/templates/download-service.yaml @@ -1,4 +1,5 @@ {{- if or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }} +{{- if .Values.global.download.enabled }} apiVersion: v1 kind: Service metadata: @@ -8,9 +9,10 @@ metadata: spec: ports: - name: download - port: {{ ternary 443 8080 .Values.global.tls.enabled }} + port: {{ ternary 443 80 .Values.global.tls.enabled }} targetPort: download protocol: TCP selector: app: {{ template "sda.fullname" . }}-download {{- end }} +{{- end }} diff --git a/charts/sda-svc/templates/finalize-certificate.yaml b/charts/sda-svc/templates/finalize-certificate.yaml new file mode 100644 index 00000000..2eeaefcb --- /dev/null +++ b/charts/sda-svc/templates/finalize-certificate.yaml @@ -0,0 +1,36 @@ +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-finalize-certs +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-finalize-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: {{ template "sda.fullname" . }}-finalize + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - client auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }}-finalize + - {{ template "sda.fullname" . }}-finalize.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} diff --git a/charts/sda-svc/templates/finalize-deploy.yaml b/charts/sda-svc/templates/finalize-deploy.yaml index f8306442..d6d796e3 100644 --- a/charts/sda-svc/templates/finalize-deploy.yaml +++ b/charts/sda-svc/templates/finalize-deploy.yaml @@ -99,11 +99,9 @@ spec: {{- end }} {{- end }} {{- if .Values.global.tls.enabled }} - {{- if or (eq "verify-ca" .Values.global.db.sslMode) (eq "verify-full" .Values.global.db.sslMode) }} - name: DB_CACERT value: {{ include "tlsPath" . }}/ca.crt - {{- end }} - {{- if eq "verify-full" .Values.global.db.sslMode }} + {{- if ne "verify-none" .Values.global.db.sslMode }} - name: DB_CLIENTCERT value: {{ include "tlsPath" . }}/tls.crt - name: DB_CLIENTKEY @@ -159,19 +157,12 @@ spec: - name: {{ ternary "tls" "tls-certs" (empty .Values.global.pkiPermissions) }} projected: sources: - {{- if or .Values.global.broker.verifyPeer (eq "verify-full" .Values.global.db.sslMode) }} - - secret: - name: {{ required "TLS secret name is requrired for finalize" .Values.finalize.tls.secretName }} - {{- end }} - {{- if .Values.global.tls.caCert.secretName }} + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} - secret: - name: {{ .Values.global.tls.caCert.secretName }} - items: - - key: {{ required "A name for the CA cert is required" .Values.global.tls.caCert.secretKey }} - path: ca.crt + name: {{ template "sda.fullname" . }}-finalize-certs {{- else }} - secret: - name: {{ template "sda.fullname" . }}-cacert + name: {{ required "An certificate issuer or a TLS secret name is required for finalize" .Values.finalize.tls.secretName }} {{- end }} {{- if .Values.global.pkiPermissions }} - name: tls diff --git a/charts/sda-svc/templates/inbox-certificate.yaml b/charts/sda-svc/templates/inbox-certificate.yaml new file mode 100644 index 00000000..935f3deb --- /dev/null +++ b/charts/sda-svc/templates/inbox-certificate.yaml @@ -0,0 +1,37 @@ +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-inbox-certs +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-inbox-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: {{ template "sda.fullname" . }}-inbox + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - server auth + - client auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }}-inbox + - {{ template "sda.fullname" . }}-inbox.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} diff --git a/charts/sda-svc/templates/ingest-certificate.yaml b/charts/sda-svc/templates/ingest-certificate.yaml new file mode 100644 index 00000000..2a956167 --- /dev/null +++ b/charts/sda-svc/templates/ingest-certificate.yaml @@ -0,0 +1,36 @@ +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-ingest-certs +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-ingest-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: {{ template "sda.fullname" . }}-ingest + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - client auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }}-ingest + - {{ template "sda.fullname" . }}-ingest.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} diff --git a/charts/sda-svc/templates/ingest-deploy.yaml b/charts/sda-svc/templates/ingest-deploy.yaml index 509a1d29..d94f2218 100644 --- a/charts/sda-svc/templates/ingest-deploy.yaml +++ b/charts/sda-svc/templates/ingest-deploy.yaml @@ -126,11 +126,9 @@ spec: - name: C4GH_FILEPATH value: {{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }} {{- if .Values.global.tls.enabled }} - {{- if or (eq "verify-ca" .Values.global.db.sslMode) (eq "verify-full" .Values.global.db.sslMode) }} - name: DB_CACERT value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- if eq "verify-full" .Values.global.db.sslMode }} + {{- if ne "verify-none" .Values.global.db.sslMode }} - name: DB_CLIENTCERT value: {{ template "tlsPath" . }}/tls.crt - name: DB_CLIENTKEY @@ -258,19 +256,12 @@ spec: - name: {{ ternary "tls" "tls-certs" (empty .Values.global.pkiPermissions) }} projected: sources: - {{- if or .Values.global.broker.verifyPeer (eq "verify-full" .Values.global.db.sslMode) }} - - secret: - name: {{ required "TLS secret name is requrired for ingest" .Values.ingest.tls.secretName }} - {{- end }} - {{- if .Values.global.tls.caCert.secretName }} + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} - secret: - name: {{ .Values.global.tls.caCert.secretName }} - items: - - key: {{ required "A name for the CA cert is required" .Values.global.tls.caCert.secretKey }} - path: ca.crt + name: {{ template "sda.fullname" . }}-ingest-certs {{- else }} - - secret: - name: {{ template "sda.fullname" . }}-cacert + - secret: + name: {{ required "An certificate issuer or a TLS secret name is required for ingest" .Values.ingest.tls.secretName }} {{- end }} {{- if .Values.global.pkiPermissions }} - name: tls diff --git a/charts/sda-svc/templates/intercept-certificate.yaml b/charts/sda-svc/templates/intercept-certificate.yaml new file mode 100644 index 00000000..364344f3 --- /dev/null +++ b/charts/sda-svc/templates/intercept-certificate.yaml @@ -0,0 +1,36 @@ +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-intercept-certs +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-intercept-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: {{ template "sda.fullname" . }}-intercept + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - client auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }}-intercept + - {{ template "sda.fullname" . }}-intercept.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} diff --git a/charts/sda-svc/templates/intercept-deploy.yaml b/charts/sda-svc/templates/intercept-deploy.yaml index e06121e1..3c449464 100644 --- a/charts/sda-svc/templates/intercept-deploy.yaml +++ b/charts/sda-svc/templates/intercept-deploy.yaml @@ -110,19 +110,12 @@ spec: - name: {{ "tls" }} projected: sources: - {{- if .Values.intercept.tls.secretName }} + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} - secret: - name: {{ .Values.intercept.tls.secretName }} - {{- end }} - {{- if .Values.global.tls.caCert.secretName }} - - secret: - name: {{ .Values.global.tls.caCert.secretName }} - items: - - key: {{ required "A name for the CA cert is required" .Values.global.tls.caCert.secretKey }} - path: ca.crt + name: {{ template "sda.fullname" . }}-intercept-certs {{- else }} - - secret: - name: {{ template "sda.fullname" . }}-cacert + - secret: + name: {{ required "An certificate issuer or a TLS secret name is required for ingest" .Values.ingest.tls.secretName }} {{- end }} {{- end }} restartPolicy: Always diff --git a/charts/sda-svc/templates/mapper-certificate.yaml b/charts/sda-svc/templates/mapper-certificate.yaml new file mode 100644 index 00000000..e83ac5c5 --- /dev/null +++ b/charts/sda-svc/templates/mapper-certificate.yaml @@ -0,0 +1,36 @@ +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-mapper-certs +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-mapper-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: {{ template "sda.fullname" . }}-mapper + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - client auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }}-mapper + - {{ template "sda.fullname" . }}-mapper.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} diff --git a/charts/sda-svc/templates/mapper-deploy.yaml b/charts/sda-svc/templates/mapper-deploy.yaml index 08584021..260bd6bd 100644 --- a/charts/sda-svc/templates/mapper-deploy.yaml +++ b/charts/sda-svc/templates/mapper-deploy.yaml @@ -97,11 +97,9 @@ spec: {{- end }} {{- end }} {{- if .Values.global.tls.enabled }} - {{- if or (eq "verify-ca" .Values.global.db.sslMode) (eq "verify-full" .Values.global.db.sslMode) }} - name: DB_CACERT value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- if eq "verify-full" .Values.global.db.sslMode }} + {{- if ne "verify-none" .Values.global.db.sslMode }} - name: DB_CLIENTCERT value: {{ template "tlsPath" . }}/tls.crt - name: DB_CLIENTKEY @@ -157,20 +155,13 @@ spec: - name: {{ ternary "tls" "tls-certs" (empty .Values.global.pkiPermissions) }} projected: sources: - {{- if or .Values.global.broker.verifyPeer (eq "verify-full" .Values.global.db.sslMode) }} - - secret: - name: {{ required "TLS secret name is requrired for mapper" .Values.mapper.tls.secretName }} - {{- end }} - {{- if .Values.global.tls.caCert.secretName }} + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} - secret: - name: {{ .Values.global.tls.caCert.secretName }} - items: - - key: {{ required "A name for the CA cert is required" .Values.global.tls.caCert.secretKey }} - path: ca.crt - {{- else }} + name: {{ template "sda.fullname" . }}-mapper-certs + {{- else if .Values.mapper.tls.secretName }} - secret: - name: {{ template "sda.fullname" . }}-cacert - {{- end }} + name: {{ required "An certificate issuer or a TLS secret name is required for mapper" .Values.mapper.tls.secretName }} + {{- end }} {{- if .Values.global.pkiPermissions }} - name: tls emptyDir: diff --git a/charts/sda-svc/templates/release-test-deploy.yml b/charts/sda-svc/templates/release-test-deploy.yml index 5ae38621..cc86a47d 100644 --- a/charts/sda-svc/templates/release-test-deploy.yml +++ b/charts/sda-svc/templates/release-test-deploy.yml @@ -1,5 +1,5 @@ ---- -{{- if and .Values.releasetest.run .Values.global.c4gh.publicFile -}} +{{- if .Values.releasetest.run }} +{{- if eq "s3" .Values.global.inbox.storageType }} apiVersion: v1 kind: Secret metadata: @@ -9,13 +9,9 @@ metadata: "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed type: Opaque data: - mqPassword: {{ include "mqPassReleaseTest" . | b64enc }} - mqUser: {{ include "mqUserReleaseTest" . | b64enc }} - dbPassword: {{ include "dbPassReleaseTest" . | b64enc }} - dbUser: {{ include "dbUserReleaseTest" . | b64enc }} - + accessToken: {{ required "Access token is required for testing the S3 inbox" .Values.releasetest.secrets.accessToken | b64enc }} --- - +{{- end }} apiVersion: v1 kind: ConfigMap metadata: @@ -24,13 +20,8 @@ metadata: "helm.sh/hook": test "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed data: - {{- ( .Files.Glob ( "test/{verify-inboxes.py,trigger-ingest.py,accession.py}" )).AsConfig | nindent 2 }} {{- ( .Files.Glob ( "test/release-test.sh" )).AsConfig | nindent 2 }} -{{- if .Values.global.tls }} - {{- ( .Files.Glob ( "test/{verify-doa.py,verify-download.py}" )).AsConfig | nindent 2 }} -{{- end }} --- - apiVersion: v1 kind: Pod metadata: @@ -56,18 +47,10 @@ spec: volumeMounts: {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: certs - mountPath: {{ template "tlsPath" . }} + mountPath: /tls {{- end }} - name: release-test-app mountPath: "/release-test-app" - {{- if eq "posix" .Values.global.inbox.storageType }} - - name: "inbox-release-test" - mountPath: "/posix_inbox" - {{- end }} - {{- if eq "posix" .Values.global.archive.storageType }} - - name: "archive-release-test" - mountPath: "/posix_archive" - {{- end }} env: - name: TLS value: {{ .Values.global.tls.enabled | quote}} @@ -76,118 +59,33 @@ spec: value: {{include "secretsPath" }} {{- else }} - name: PKI_PATH - value: {{ template "tlsPath" . }} - {{- end }} - - name: DEPLOYMENT_TOPOLOGY - {{- if .Values.intercept.deploy }} - value: "federated" - {{- else }} - value: "standalone" + value: /tls {{- end }} - name: DEPLOYMENT_TYPE value: "{{ .Values.global.deploymentType }}" - name: INBOX_SERVICE_NAME value: "{{ template "sda.fullname" . }}-inbox" + {{- if eq "s3" .Values.global.inbox.storageType }} + - name: AUTH_SERVICE_NAME + value: "{{ template "sda.fullname" . }}-auth" + - name: INBOX_ACCESS_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-release-test-passwords + key: accessToken + {{- end }} + {{- if and .Values.global.tls.enabled .Values.global.doa.enabled }} - name: DOA_SERVICE_NAME - value: "{{ template "sda.fullname" . }}-doa" + value: "{{ template "sda.fullname" . }}-doa" + {{- end }} + {{- if .Values.global.download.enabled }} - name: DOWNLOAD_SERVICE_NAME value: "{{ template "sda.fullname" . }}-download" + {{- end }} - name: INBOX_STORAGE_TYPE value: {{ required "Missing storage type for inbox" .Values.global.inbox.storageType | quote }} - {{- if eq "s3" .Values.global.inbox.storageType }} - - name: INBOX_BUCKET - value: {{ required "S3 inbox bucket missing" .Values.global.inbox.s3Bucket }} - {{- if .Values.global.inbox.s3CaFile }} - - name: INBOX_CACERT - value: {{ template "tlsPath" . }}/{{ .Values.global.inbox.s3CaFile }} - {{- end }} - - name: INBOX_REGION - value: {{ default "us-east-1" .Values.global.inbox.s3Region }} - - name: INBOX_URL - value: {{ template "S3InboxURL" . }} - - name: INBOX_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxAccessKey - - name: INBOX_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxSecretKey - {{- end }} - name: ARCHIVE_STORAGE_TYPE value: {{ required "Missing storage type for archive" .Values.global.archive.storageType | quote }} - {{- if eq "s3" .Values.global.archive.storageType }} - - name: ARCHIVE_BUCKET - value: {{ required "S3 archive bucket missing" .Values.global.archive.s3Bucket }} - {{- if .Values.global.archive.s3CaFile }} - - name: ARCHIVE_CACERT - value: {{ template "tlsPath" . }}/{{ .Values.global.archive.s3CaFile }} - {{- end }} - - name: ARCHIVE_REGION - value: {{ default "us-east-1" .Values.global.archive.s3Region }} - - name: ARCHIVE_URL - value: {{ template "S3ArchiveURL" . }} - - name: ARCHIVE_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveAccessKey - - name: ARCHIVE_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveSecretKey - {{- end }} - {{- if or (eq "verify-ca" .Values.global.db.sslMode) (eq "verify-full" .Values.global.db.sslMode) }} - - name: DB_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- end }} - {{- if eq "verify-full" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ template "dbSSLmode" .}} - - name: MQ_HOST - value: {{ required "Broker host must be set" .Values.global.broker.host | quote }} - - name: MQ_EXCHANGE - value: {{ default "lega" .Values.global.broker.exchange | quote }} - {{- if not .Values.global.vaultSecrets }} - - name: MQ_PASSWORD - valueFrom: - secretKeyRef: - name: "{{ .Release.Name }}-release-test-passwords" - key: mqPassword - - name: MQ_USER - valueFrom: - secretKeyRef: - name: "{{ .Release.Name }}-release-test-passwords" - key: mqUser - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: "{{ .Release.Name }}-release-test-passwords" - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: "{{ .Release.Name }}-release-test-passwords" - key: dbUser - {{- end }} - {{- if .Values.global.broker.vhost }} - - name: MQ_VHOST - value: {{ .Values.global.broker.vhost | quote }} - {{- end }} command: [ "/bin/bash" ] args: - "/release-test-app/release-test.sh" @@ -196,40 +94,13 @@ spec: - name: certs projected: sources: - - secret: - name: {{ .Values.releasetest.tls.secretName }} - {{- if .Values.global.tls.caCert.secretName }} - - secret: - name: {{ .Values.global.tls.caCert.secretName }} - items: - - key: {{ required "A name for the CA cert is required" .Values.global.tls.caCert.secretKey }} - path: ca.crt + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + - secret: + name: {{ template "sda.fullname" . }}-tester-certs {{- else }} - - secret: - name: {{ template "sda.fullname" . }}-caCert + - secret: + name: {{ required "An certificate issuer or a TLS secret name is required for releasetest" .Values.releasetest.tls.secretName }} {{- end }} - {{- end }} - {{- if eq .Values.global.inbox.storageType "posix" }} - - name: "inbox-release-test" - {{- if .Values.global.inbox.existingClaim }} - persistentVolumeClaim: - claimName: {{ .Values.global.inbox.existingClaim }} - {{- else }} - nfs: - server: {{ required "An inbox NFS server is required" .Values.global.inbox.nfsServer | quote }} - path: {{ if .Values.global.inbox.nfsPath }}{{ .Values.global.inbox.nfsPath | quote}}{{ else }}{{ "/" }}{{ end }} - {{- end }} - {{- end }} - {{- if eq .Values.global.archive.storageType "posix" }} - - name: "archive-release-test" - {{- if .Values.global.archive.existingClaim }} - persistentVolumeClaim: - claimName: {{ .Values.global.archive.existingClaim }} - {{- else }} - nfs: - server: {{ required "An archive NFS server is required" .Values.global.archive.nfsServer | quote }} - path: {{ if .Values.global.archive.nfsPath }}{{ .Values.global.archive.nfsPath | quote}}{{ else }}{{ "/" }}{{ end }} - {{- end }} {{- end }} - name: release-test-app projected: @@ -237,24 +108,42 @@ spec: - configMap: name: "{{ .Release.Name }}-release-test-app" items: - - key: trigger-ingest.py - path: trigger-ingest.py - - key: accession.py - path: accession.py - - key: verify-inboxes.py - path: verify-inboxes.py - - key: verify-doa.py - path: verify-doa.py - - key: verify-download.py - path: verify-download.py - key: release-test.sh path: release-test.sh - - secret: - name: {{ .Values.global.c4gh.secretName}} - items: - - key: {{ .Values.global.c4gh.publicFile }} - path: c4gh.pub restartPolicy: Never - -{{- /* end for only run if given publicFile */ -}} +--- +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-tester-certs +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-tester-certs + duration: 2160h + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: {{ template "sda.fullname" . }}-tester + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - client auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }}-tester + - {{ template "sda.fullname" . }}-tester.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} {{- end -}} diff --git a/charts/sda-svc/templates/s3-inbox-deploy.yaml b/charts/sda-svc/templates/s3-inbox-deploy.yaml index 1db33d6f..98de7003 100644 --- a/charts/sda-svc/templates/s3-inbox-deploy.yaml +++ b/charts/sda-svc/templates/s3-inbox-deploy.yaml @@ -121,13 +121,11 @@ spec: - name: SERVER_KEY value: {{ include "tlsPath" . }}/tls.key {{- end }} - {{- if or .Values.global.auth.jwtPub .Values.global.elixir.pubKey }} + {{- if.Values.global.auth.jwtPub }} - name: SERVER_JWTPUBKEYPATH value: {{ include "jwtPath" . }} - {{- end }} - {{- if and .Values.global.elixir.oidcdHost .Values.global.elixir.jwkPath }} - - name: SERVER_JWTPUBKEYURL - value: "{{ .Values.global.elixir.oidcdHost }}{{ .Values.global.elixir.jwkPath }}" + {{- else }} + {{ fail "The global.auth.jwtPub is required for s3Inbox" }} {{- end }} - name: LOG_LEVEL value: {{ .Values.global.logLevel }} @@ -159,8 +157,7 @@ spec: {{- end }} {{- if .Values.global.auth.jwtPub }} - name: jwt - mountPath: {{ include "jwtPath" . }}/{{ .Values.global.ingress.hostName.auth }}.pub - subPath: {{ .Values.global.auth.jwtPub }} + mountPath: {{ include "jwtPath" . }} {{- end }} volumes: {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} @@ -168,23 +165,21 @@ spec: projected: sources: - secret: - name: {{ required "TLS secret name is requrired for S3Inbox" .Values.s3Inbox.tls.secretName }} - {{- if .Values.global.tls.caCert.secretName }} - - secret: - name: {{ .Values.global.tls.caCert.secretName }} - items: - - key: {{ required "A name for the CA cert is required" .Values.global.tls.caCert.secretKey }} - path: ca.crt - {{- else }} - - secret: - name: {{ template "sda.fullname" . }}-cacert - {{- end }} + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + name: {{ template "sda.fullname" . }}-inbox-certs + {{- else }} + name: {{ required "An certificate issuer or a TLS secret name is required for S3Inbox" .Values.s3Inbox.tls.secretName }} + {{- end }} {{- end }} {{- if .Values.global.auth.jwtPub }} - name: jwt - secret: - defaultMode: 0440 - secretName: {{ .Values.global.auth.jwtSecret }} + projected: + sources: + - secret: + name: {{ .Values.global.auth.jwtSecret }} + items: + - key: {{ required "The name of the JWT signing key is needed" .Values.global.auth.jwtPub }} + path: "{{ .Values.global.ingress.hostName.auth }}.pub" {{- end }} restartPolicy: Always {{- end }} diff --git a/charts/sda-svc/templates/s3-inbox-ingress.yaml b/charts/sda-svc/templates/s3-inbox-ingress.yaml index 56e397d4..ab041437 100644 --- a/charts/sda-svc/templates/s3-inbox-ingress.yaml +++ b/charts/sda-svc/templates/s3-inbox-ingress.yaml @@ -13,18 +13,24 @@ metadata: {{ toYaml .Values.global.ingress.labels | indent 4 }} {{- end }} annotations: - kubernetes.io/ingress.class: "nginx" + {{- if eq "nginx" .Values.global.ingress.ingressClassName }} nginx.ingress.kubernetes.io/rewrite-target: "/" nginx.ingress.kubernetes.io/backend-protocol: "{{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }}" nginx.ingress.kubernetes.io/proxy-body-size: 2000m nginx.ingress.kubernetes.io/proxy-request-buffering: "off" - {{- if .Values.global.ingress.issuer }} - cert-manager.io/cluster-issuer: {{ .Values.global.ingress.issuer | quote }} {{- end }} -{{- if .Values.global.ingress.annotations }} -{{ toYaml .Values.global.ingress.annotations | indent 4 }} + {{- if .Values.global.ingress.clusterIssuer }} + cert-manager.io/cluster-issuer: {{ .Values.global.ingress.clusterIssuer | quote }} + {{- else if .Values.global.ingress.issuer }} + cert-manager.io/issuer: {{ .Values.global.ingress.issuer | quote }} + {{- end }} +{{- if .Values.global.ingress.annotations }} +{{ toYaml .Values.global.ingress.annotations | indent 4 }} {{- end }} spec: +{{- if .Values.global.ingress.ingressClassName }} + ingressClassName: {{ .Values.global.ingress.ingressClassName }} +{{- end }} rules: - host: {{ required "An ingress hostname is required!" .Values.global.ingress.hostName.s3Inbox }} http: diff --git a/charts/sda-svc/templates/sftp-inbox-deploy.yaml b/charts/sda-svc/templates/sftp-inbox-deploy.yaml index fcb7ae97..02149001 100644 --- a/charts/sda-svc/templates/sftp-inbox-deploy.yaml +++ b/charts/sda-svc/templates/sftp-inbox-deploy.yaml @@ -111,7 +111,7 @@ spec: - name: BROKER_VHOST value: {{ .Values.global.broker.vhost | quote }} - name: CEGA_ENDPOINT - value: {{ printf "%s%s" (.Values.global.cega.host) "/lega/v1/legas/users/%s?idType=username" | quote }} + value: {{ printf "%s%s" (.Values.global.cega.host) "/%s?idType=username" | quote }} - name: CEGA_ENDPOINT_CREDS valueFrom: secretKeyRef: @@ -167,17 +167,12 @@ spec: - name: tls-certs projected: sources: + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} - secret: - name: {{ required "TLS secret name is requrired for sftpInbox" .Values.sftpInbox.tls.secretName }} - {{- if .Values.global.tls.caCert.secretName }} + name: {{ template "sda.fullname" . }}-inbox-certs + {{- else}} - secret: - name: {{ .Values.global.tls.caCert.secretName }} - items: - - key: {{ required "A name for the CA cert is required" .Values.global.tls.caCert.secretKey }} - path: ca.crt - {{- else }} - - secret: - name: {{ template "sda.fullname" . }}-cacert + name: {{ required "An certificate issuer or a TLS secret name is required for sftpInbobx" .Values.sftpInbox.tls.secretName }} {{- end }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/shared-secrets.yaml b/charts/sda-svc/templates/shared-secrets.yaml index a66f523a..c83d876e 100644 --- a/charts/sda-svc/templates/shared-secrets.yaml +++ b/charts/sda-svc/templates/shared-secrets.yaml @@ -22,16 +22,4 @@ data: s3InboxSecretKey: {{ .Values.global.inbox.s3SecretKey | quote | trimall "\"" | b64enc }} {{- end }} {{- end }} ---- -{{- if not .Values.global.pkiService }} -{{- if and .Values.global.tls.enabled (not .Values.global.tls.caCert.secretName) }} -apiVersion: v1 -kind: Secret -metadata: - name: {{ template "sda.fullname" . }}-cacert -type: Opaque -data: - ca.crt: | -{{ required "CA certificates are requrired" .Values.global.tls.caCert.cert | b64enc | indent 4 }} -{{- end }} -{{- end }} + diff --git a/charts/sda-svc/templates/verify-certificate.yaml b/charts/sda-svc/templates/verify-certificate.yaml new file mode 100644 index 00000000..d3e5be52 --- /dev/null +++ b/charts/sda-svc/templates/verify-certificate.yaml @@ -0,0 +1,36 @@ +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-verify-certs +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-verify-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: {{ template "sda.fullname" . }}-verify + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - client auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }}-verify + - {{ template "sda.fullname" . }}-verify.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} diff --git a/charts/sda-svc/templates/verify-deploy.yaml b/charts/sda-svc/templates/verify-deploy.yaml index c84f158a..f179cfbc 100644 --- a/charts/sda-svc/templates/verify-deploy.yaml +++ b/charts/sda-svc/templates/verify-deploy.yaml @@ -130,7 +130,7 @@ spec: {{- if .Values.global.tls.enabled }} - name: DB_CACERT value: {{ template "tlsPath" . }}/ca.crt - {{- if eq "verify-full" .Values.global.db.sslMode }} + {{- if ne "verify-none" .Values.global.db.sslMode }} - name: DB_CLIENTCERT value: {{ template "tlsPath" . }}/tls.crt - name: DB_CLIENTKEY @@ -211,19 +211,12 @@ spec: - name: {{ ternary "tls" "tls-certs" (empty .Values.global.pkiPermissions) }} projected: sources: - {{- if or .Values.global.broker.verifyPeer (eq "verify-full" .Values.global.db.sslMode) }} + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} - secret: - name: {{ required "TLS secret name is requrired for verify" .Values.verify.tls.secretName }} - {{- end }} - {{- if .Values.global.tls.caCert.secretName }} - - secret: - name: {{ .Values.global.tls.caCert.secretName }} - items: - - key: {{ required "A name for the CA cert is required" .Values.global.tls.caCert.secretKey }} - path: ca.crt - {{- else }} + name: {{ template "sda.fullname" . }}-verify-certs + {{- else if .Values.verify.tls.secretName }} - secret: - name: {{ template "sda.fullname" . }}-cacert + name: {{ required "An certificate issuer or a TLS secret name is required for verify" .Values.verify.tls.secretName }} {{- end }} {{- if .Values.global.pkiPermissions }} - name: tls diff --git a/charts/sda-svc/test/accession.py b/charts/sda-svc/test/accession.py deleted file mode 100644 index 26f3ad71..00000000 --- a/charts/sda-svc/test/accession.py +++ /dev/null @@ -1,112 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -"""Tests the installed release""" - -import argparse -import os -import sys -import ssl -import urllib.parse -from pathlib import Path - -import pika - -# Command-line arguments -parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument("--connection", help="of the form 'amqp://:@:/'") -parser.add_argument("--latest_message", action="store_true") -parser.add_argument("user") -parser.add_argument("inboxpath") -parser.add_argument("accessionid") -parser.add_argument("decsha256") -parser.add_argument("decmd5") -parser.add_argument("routing_key") -args = parser.parse_args() - -exchange = os.getenv("MQ_EXCHANGE", "sda") -mq_vhost = os.getenv("MQ_VHOST", "/").lstrip("/") -tls = os.getenv("TLS", "true") -pki_path = os.getenv("PKI_PATH", "/certs") - -if tls == "true": - env_connection = "amqps://%s:%s@%s/%s" % ( - os.getenv("MQ_USER", "user"), - os.getenv("MQ_PASSWORD", "password"), - os.getenv("MQ_HOST", "mq"), - urllib.parse.quote(mq_vhost, safe=""), - ) -else: - env_connection = "amqp://%s:%s@%s/%s" % ( - os.getenv("MQ_USER", "user"), - os.getenv("MQ_PASSWORD", "password"), - os.getenv("MQ_HOST", "mq"), - urllib.parse.quote(mq_vhost, safe=""), - ) - - -# MQ Connection -mq_connection = args.connection if args.connection else env_connection -parameters = pika.URLParameters(mq_connection) - -if mq_connection.startswith("amqps"): - - context = ssl.SSLContext(protocol=ssl.PROTOCOL_TLS) # Enforcing (highest) TLS version (so... 1.2?) - - context.check_hostname = False - - cacertfile = Path("%s/ca.crt" % pki_path) - certfile = Path("%s/tls.crt" % pki_path) - keyfile = Path("%s/tls.key" % pki_path) - - context.verify_mode = ssl.CERT_NONE - # Require server verification - if cacertfile.exists(): - context.verify_mode = ssl.CERT_REQUIRED - context.load_verify_locations(cafile=str(cacertfile)) - - # If client verification is required - if certfile.exists() and keyfile.exists(): - context.load_cert_chain(str(certfile), keyfile=str(keyfile)) - else: - sys.exit(1) - - # Finally, the pika ssl options - parameters.ssl_options = pika.SSLOptions(context=context, server_hostname=None) - -connection = pika.BlockingConnection(parameters) -channel = connection.channel() - -message = """ -{ - "type": "accession", - "user": "%s", - "filepath": "%s", - "accession_id": "%s", - "decrypted_checksums": [ - { - "type": "sha256", - "value": "%s" - }, - { - "type": "md5", - "value": "%s" - } - ] -} -""" % ( - args.user, - args.inboxpath, - args.accessionid, - args.decsha256, - args.decmd5, -) - -channel.basic_publish( - exchange=exchange, - routing_key=args.routing_key, - body=message, - properties=pika.BasicProperties(correlation_id="1", content_type="application/json", delivery_mode=2), -) - -connection.close() diff --git a/charts/sda-svc/test/release-test.sh b/charts/sda-svc/test/release-test.sh index f51f24da..c1f3b2af 100644 --- a/charts/sda-svc/test/release-test.sh +++ b/charts/sda-svc/test/release-test.sh @@ -1,285 +1,114 @@ #!/bin/bash -if [ "${DEPLOYMENT_TOPOLOGY}" = "federated" ]; then - ingest_routing_key="files" - accession_routing_key="files" -else - ingest_routing_key="ingest" - accession_routing_key="accessionIDs" +if [ "$INBOX_STORAGE_TYPE" == "s3" ]; then + if [ "$TLS" == true ]; then + cat >> "/tmp/s3cfg" <<-EOF + host_base = $INBOX_SERVICE_NAME + host_bucket = $INBOX_SERVICE_NAME + access_key = dummy + access_token = $INBOX_ACCESS_TOKEN + use_https = True + ca_certs_file = /tls/ca.crt + EOF + else + cat >> "/tmp/s3cfg" <<-EOF + host_base = $INBOX_SERVICE_NAME + host_bucket = $INBOX_SERVICE_NAME + access_key = dummy + access_token = $INBOX_ACCESS_TOKEN + use_https = False + EOF + fi fi -export MQ_EXCHANGE='' +cat /tmp/s3cfg if [ "${DEPLOYMENT_TYPE}" = all ] || [ "${DEPLOYMENT_TYPE}" = external ]; then - - if ! python3 /release-test-app/verify-inboxes.py ; then - echo "Failed inbox verification, bailing out" - exit 1 - fi - - echo "Inbox seems okay" - - if [ "${TLS}" = true ]; then - - if ! python3 /release-test-app/verify-doa.py ; then - echo "Failed doa verification, bailing out" - exit 1 - fi - - echo "DOA seems okay" - - if ! python3 /release-test-app/verify-download.py ; then - echo "Failed download verification, bailing out" - exit 1 - fi - - echo "Download seems okay" - - fi + if [ "$INBOX_STORAGE_TYPE" == "posix" ] && [ "$TLS" == true ]; then + # Connect and see that we get a ssh greeting + if ! echo "test" | nc "$INBOX_SERVICE_NAME" 2222 | grep "SSH-2.0"; then + echo "Failed inbox verification, bailing out" + exit 1 + fi + elif [ "$INBOX_STORAGE_TYPE" == "s3" ]; then + if [ "$TLS" == true ]; then + echo "Will try connecting to https://$INBOX_SERVICE_NAME/" + if ! s3cmd -c "/tmp/s3cfg" ls s3://dummy ; then + echo "expected 403 got: $responsecode" + echo "Failed inbox verification, bailing out" + exit 1 + fi + + echo "Will try connecting to http://$AUTH_SERVICE_NAME/" + responsecode=$(curl --cacert /tls/ca.crt -s -o /dev/null -w "%{http_code}" "https://$AUTH_SERVICE_NAME") + if ! [ "$responsecode" -eq 200 ]; then + echo "expected 200 got: $responsecode" + echo "Failed auth verification, bailing out" + exit 1 + fi + else + echo "Will try connecting to http://$INBOX_SERVICE_NAME/" + if ! s3cmd -c "/tmp/s3cfg" ls s3://dummy ; then + echo "Failed inbox verification, bailing out" + exit 1 + fi + + echo "Will try connecting to http://$AUTH_SERVICE_NAME/" + responsecode=$(curl -s -o /dev/null -w "%{http_code}" "http://$AUTH_SERVICE_NAME") + if ! [ "$responsecode" -eq 200 ]; then + echo "expected 200 got: $responsecode" + echo "Failed auth verification, bailing out" + exit 1 + fi + fi + else + echo "Unknown inbox storageType: $INBOX_STORAGE_TYPE, bailing out" + exit 1 + fi + + echo "Inbox seems okay" + + if [ -n "$DOA_SERVICE_NAME" ] && [ "${TLS}" = true ]; then + responsecode=$(curl --cacert /tls/ca.crt -s -o /dev/null -w "%{http_code}" "https://$DOA_SERVICE_NAME/metadata/datasets") + if ! [ "$responsecode" -eq 401 ]; then + echo "expected 401 got: $responsecode" + echo "Failed DOA verification, bailing out" + exit 1 + fi + echo "DOA seems okay" + elif [ -n "$DOWNLOAD_SERVICE_NAME" ] ; then + if [ "${TLS}" = true ]; then + responsecode=$(curl --cacert /tls/ca.crt -s -o /dev/null -w "%{http_code}" "https://$DOWNLOAD_SERVICE_NAME/health") + if ! [ "$responsecode" -eq 200 ]; then + echo "expected 200 got: $responsecode" + echo "Failed Download verification, bailing out" + exit 1 + fi + else + responsecode=$(curl --cacert /tls/ca.crt -s -o /dev/null -w "%{http_code}" "http://$DOWNLOAD_SERVICE_NAME/health") + if ! [ "$responsecode" -eq 200 ]; then + echo "expected 200 got: $responsecode" + echo "Failed Download verification, bailing out" + exit 1 + fi + fi + echo "Download seems okay" + else + echo "No data out solution deployed, bailing out" + exit 1 + fi fi if [ "${DEPLOYMENT_TYPE}" = external ]; then - echo "External-only deployment, nothing more to check." - exit 0 -fi - -mkdir -p /tmp/c4gh - -user=release-test-${RANDOM} - -tmpfile=$(mktemp) -uploaded=${tmpfile##*/}.encrypted - -echo "Creating file $tmpfile" - -dd if=/dev/urandom of="$tmpfile" bs=1M count=16 - -echo "Creating c4gh key" -crypt4gh-keygen --sk /tmp/c4gh/key --pk /tmp/c4gh/key.pub --nocrypt -f - -echo "Encrypting file" -crypt4gh encrypt --recipient_pk /release-test-app/c4gh.pub --sk /tmp/c4gh/key \ - <"$tmpfile" \ - >"${tmpfile}.encrypted" - -echo "Noting checksums" -sha=$(sha256sum "${tmpfile}.encrypted" | cut -d' ' -f 1) -decsha=$(sha256sum "${tmpfile}" | cut -d' ' -f 1) -decmd=$(md5sum "${tmpfile}" | cut -d' ' -f 1) - -echo -echo "Encrypted sha256: $sha" -echo "Decrypted sha256: $decsha" -echo "Decrypted md5: $decmd" -echo - -if [ "${INBOX_STORAGE_TYPE}" = posix ]; then - echo "Copying file $tmpfile to posix inbox" - cp "${tmpfile}.encrypted" /posix_inbox/ - ls -la "/posix_inbox/${uploaded}" -else - - cat - >/tmp/s3cmd.cfg <>/tmp/s3cmd.cfg - fi - - echo "Copying to s3://${INBOX_BUCKET}/${uploaded}" - s3cmd --region="${INBOX_REGION}" -c /tmp/s3cmd.cfg put "${tmpfile}.encrypted" "s3://${INBOX_BUCKET}" - - echo "Waiting for s3 consistency" - - sleep 60 - s3cmd --region="${INBOX_REGION}" -c /tmp/s3cmd.cfg ls "s3://${INBOX_BUCKET}/${uploaded}" - -fi - -count=0 - -echo "Trying to trigger ingestion by message through the routing key $ingest_routing_key" -until python3 /release-test-app/trigger-ingest.py "$user" "$uploaded" "$ingest_routing_key"; do - echo "MQ failed, will wait and retry" - sleep 10 - count=$((count + 1)) - if [ "$count" -gt 10 ]; then - echo "Could not send message after 100 s, bailing out" - exit 1 - fi -done - -# Fix db certs permissions -PGSSL=/tmp/pgcerts -mkdir -p "$PGSSL" - -if [ -n "${DB_SSLMODE}" ]; then - PGSSLMODE=${DB_SSLMODE} - export PGSSLMODE -fi - -if [ "${TLS}" = true ]; then - PGSSLKEY=$PGSSL/postgresql.key - PGSSLCERT=$PGSSL/postgresql.crt - PGSSLROOTCERT=$PGSSL/root.crt - export PGSSLKEY PGSSLCERT PGSSLROOTCERT - - s=${PKI_PATH:-/certs} - - cp "$s/tls.key" "$PGSSL/postgresql.key" - cp "$s/tls.crt" "$PGSSL/postgresql.crt" - cp "$s/ca.crt" $PGSSL/root.crt - - chmod -R og-rw $PGSSL -fi - -PGPASSWORD=${DB_PASSWORD} -export PGPASSWORD - -echo "Will check in DB if file is archived" -echo -echo "Command used: psql -A -t -h \"${DB_HOST}\" -U lega_in lega -c \ - \"select archive_path from local_ega.files where - inbox_file_checksum='$sha' and - inbox_path='${uploaded}' and - elixir_id='$user' and - status in ('ARCHIVED', 'COMPLETED', 'READY');\"" -echo -# psql -h "${DB_HOST}" -U lega_in lega - -count=1 -until archivepath=$(psql -A -t -h "${DB_HOST}" -U lega_in lega -c \ - "select archive_path from local_ega.files where - inbox_file_checksum='$sha' and - inbox_path='${uploaded}' and - elixir_id='$user' and - status in ('ARCHIVED', 'COMPLETED', 'READY');" | grep '.') || [ "$count" -ge 12 ]; do - sleep 10 - count=$((count + 1)) -done - -if [ -z "$archivepath" ]; then - echo "File did not show up in database after 2 minutes, giving up." - exit 1 -fi - -echo "File was archived as $archivepath" - -access=$(printf "EGAF%011d" "${RANDOM}${RANDOM}") - -echo "Will send an accession id message through the routing key $accession_routing_key" -until python3 /release-test-app/accession.py "$user" "$uploaded" "$access" "$decsha" "$decmd" "$accession_routing_key"; do - sleep 10 - count=$((count + 1)) - if [ "$count" -gt 10 ]; then - echo "Could not send message after 100 seconds, giving up." - exit 1 - fi -done - -echo "Will check database for ready file" -echo -echo "Command used: psql -A -t -h \"${DB_HOST}\" -U lega_in lega -c \ - \"select archive_path from local_ega.files where - inbox_file_checksum='$sha' and - inbox_path='$uploaded' and - stable_id='$access' and - elixir_id='$user' and - status='READY';\"" -echo - -count=0 -until readypath=$(psql -A -t -h "${DB_HOST}" -U lega_in lega -c \ - "select archive_path from local_ega.files where - inbox_file_checksum='$sha' and - inbox_path='$uploaded' and - stable_id='$access' and - elixir_id='$user' and - status='READY';" | grep '.') || [ "$count" -ge 12 ]; do - sleep 10 - echo "Not completed yet, will wait and retry" - count=$((count + 1)) -done - -echo -if [ -z "$readypath" ]; then - echo "File did not show up as ready in database after 2 minutes, giving up." - exit 1 -fi - -echo "File was ready - test succeeded" - -echo -echo "Doing teardown" -echo - -echo "Removing from database" -echo -psql -A -t -h "${DB_HOST}" -U lega_in lega -c \ - "delete from local_ega.files where - inbox_file_checksum='$sha' and - inbox_path='$uploaded' and - archive_path='$archivepath' and - stable_id='$access' and - elixir_id='$user' and - status='READY';" - -echo -if [ "${INBOX_STORAGE_TYPE}" = posix ]; then - echo "Removing ${uploaded} from posix inbox if it's there" - rm -f "/posix_inbox/${uploaded}" -else - echo "Removing s3://${INBOX_BUCKET}/${uploaded}" - s3cmd --region="${INBOX_REGION}" -c /tmp/s3cmd.cfg del "s3://${INBOX_BUCKET}/${uploaded}" + echo "External-only deployment, nothing more to check." + exit 0 fi -if [ "${ARCHIVE_STORAGE_TYPE}" = posix ]; then - echo "Removing $archivepath from posix archive if it's there" - rm -f "/posix_archive/$archivepath" -else - - cat - >/tmp/s3cmd.cfg <>/tmp/s3cmd.cfg - fi - echo "Removing s3://${ARCHIVE_BUCKET}/$archivepath" - s3cmd --region="${ARCHIVE_REGION}" -c /tmp/s3cmd.cfg del "s3://${ARCHIVE_BUCKET}/$archivepath" +if [ "${DEPLOYMENT_TYPE}" = internal ]; then + echo "Internal-only deployment, nothing more to check." + exit 0 fi exit 0 diff --git a/charts/sda-svc/test/trigger-ingest.py b/charts/sda-svc/test/trigger-ingest.py deleted file mode 100644 index c867cb55..00000000 --- a/charts/sda-svc/test/trigger-ingest.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -"""Tests the installed release""" - -import argparse -import os -import sys -import ssl -import urllib.parse -from pathlib import Path - -import pika - -# Command-line arguments -parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument("--connection", help="of the form 'amqp://:@:/'") -parser.add_argument("--latest_message", action="store_true") -parser.add_argument("user") -parser.add_argument("filepath") -parser.add_argument("routing_key") -args = parser.parse_args() - -exchange = os.getenv("MQ_EXCHANGE", "sda") -mq_vhost = os.getenv("MQ_VHOST", "/").lstrip("/") -tls = os.getenv("TLS", "true") -pki_path = os.getenv("PKI_PATH", "/certs") - -if tls == "true": - env_connection = "amqps://%s:%s@%s/%s" % ( - os.getenv("MQ_USER", "user"), - os.getenv("MQ_PASSWORD", "password"), - os.getenv("MQ_HOST", "mq"), - urllib.parse.quote(mq_vhost, safe=""), - ) -else: - env_connection = "amqp://%s:%s@%s/%s" % ( - os.getenv("MQ_USER", "user"), - os.getenv("MQ_PASSWORD", "password"), - os.getenv("MQ_HOST", "mq"), - urllib.parse.quote(mq_vhost, safe=""), - ) - -# MQ Connection -mq_connection = args.connection if args.connection else env_connection -parameters = pika.URLParameters(mq_connection) - -if mq_connection.startswith("amqps"): - - # Enforcing (highest) TLS version (so... 1.2?) - context = ssl.SSLContext(protocol=ssl.PROTOCOL_TLS) - - context.check_hostname = False - - cacertfile = Path("%s/ca.crt" % pki_path) - certfile = Path("%s/tls.crt" % pki_path) - keyfile = Path("%s/tls.key" % pki_path) - - print("CAcert: %s" % cacertfile) - print("cert: %s" % certfile) - print("key: %s" % keyfile) - - context.verify_mode = ssl.CERT_NONE - # Require server verification - if cacertfile.exists(): - context.verify_mode = ssl.CERT_REQUIRED - context.load_verify_locations(cafile=str(cacertfile)) - - # If client verification is required - if certfile.exists() and keyfile.exists(): - context.load_cert_chain(str(certfile), keyfile=str(keyfile)) - else: - sys.exit(1) - - # Finally, the pika ssl options - parameters.ssl_options = pika.SSLOptions(context=context, server_hostname=None) - -connection = pika.BlockingConnection(parameters) -channel = connection.channel() - -message = """ -{ - "type": "ingest", - "user": "%s", - "filepath": "%s" -} -""" % ( - args.user, - args.filepath, -) - -channel.basic_publish( - exchange=exchange, - routing_key=args.routing_key, - body=message, - properties=pika.BasicProperties(correlation_id="1", content_type="application/json", delivery_mode=2), -) - -connection.close() diff --git a/charts/sda-svc/test/verify-doa.py b/charts/sda-svc/test/verify-doa.py deleted file mode 100644 index ad2521fc..00000000 --- a/charts/sda-svc/test/verify-doa.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 - -import os -import ssl -import sys -import urllib.request -import urllib.error - - -backendhost = os.environ.get("DOA_SERVICE_NAME", "localhost") - -print(f"Will try connecting to {backendhost}:443") - -try: - r = urllib.request.urlopen(f"https://{backendhost}:443/files", context=ssl._create_unverified_context()) - sys.exit(0) -except urllib.error.HTTPError as e: - if e.code == 404: - # 404 is okay here - sys.exit(0) - print("Unexpected error talking to doa: \n\n%s" % e) - -sys.exit(1) diff --git a/charts/sda-svc/test/verify-download.py b/charts/sda-svc/test/verify-download.py deleted file mode 100644 index 5d1bf2d3..00000000 --- a/charts/sda-svc/test/verify-download.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 - -import os -import ssl -import sys -import urllib.request -import urllib.error - - -backendhost = os.environ.get("DOWNLOAD_SERVICE_NAME", "localhost") - -print(f"Will try connecting to {backendhost}:443") - -try: - r = urllib.request.urlopen(f"https://{backendhost}:443/files", context=ssl._create_unverified_context()) - sys.exit(0) -except urllib.error.HTTPError as e: - if e.code == 404: - # 404 is okay here - sys.exit(0) - print("Unexpected error talking to download: \n\n%s" % e) - -sys.exit(1) diff --git a/charts/sda-svc/test/verify-inboxes.py b/charts/sda-svc/test/verify-inboxes.py deleted file mode 100644 index 000f20ae..00000000 --- a/charts/sda-svc/test/verify-inboxes.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 - -import os -import socket -import ssl -import sys -import urllib.request - -backendtype = os.environ.get("INBOX_STORAGE_TYPE", "s3") -backendhost = os.environ.get("INBOX_SERVICE_NAME", "localhost") -tls = os.environ.get("TLS", "true") - -if backendtype == "posix": - # Connect and see that we get a ssh greeting - s = socket.socket() - - print("Will try connecting to %s:2222" % backendhost) - s.connect( - (backendhost, 2222), - ) - hello = s.recv(8) - - if hello != b"SSH-2.0-": - sys.exit(1) - sys.exit(0) - -if backendtype == "s3": - if tls == "true": - print(f"Will try connecting to https://{backendhost}/") - try: - r = urllib.request.urlopen(f"https://{backendhost}/", context=ssl._create_unverified_context()) - - sys.exit(0) - except urllib.error.HTTPError as e: - if e.code == 403: - # 403 is okay here - sys.exit(0) - print("Unexpected error talking to inbox: \n\n%s" % e) - else: - print(f"Will try connecting to http://{backendhost}/") - try: - r = urllib.request.urlopen(f"http://{backendhost}/") - - sys.exit(0) - except urllib.error.HTTPError as e: - if e.code == 403: - # 403 is okay here - sys.exit(0) - print("Unexpected error talking to inbox: \n\n%s" % e) - -sys.exit(1) diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index a36acb63..5583f7e5 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -7,9 +7,9 @@ global: # TLS certificates or C4GH key locations can be set using global.tlsPath or global.c4ghPath respectively, # this path will be a subpath to the secretsPath. secretsPath: /.secrets - c4ghPath: "" - tlsPath: "" - jwtPath: "" + c4ghPath: "c4gh" + tlsPath: "tls" + jwtPath: "jwt" # Name of an injected config file, located in the secretsPath # If the file is located in a subpath it can be set using the confFilePath @@ -25,10 +25,11 @@ global: # standalone version requires a custom schema e.g. isolated # the files can still be overwritten in container with no change # options are `federated` or `isolated` - schemaType: "" + schemaType: "federated" ingress: deploy: true + ingressClassName: "nginx" hostName: auth: "" doa: "" @@ -39,9 +40,16 @@ global: secretNames: auth: "" doa: "" + download: "" s3Inbox: "" # issuer requires a configured cert-manager issuer: "" + clusterIssuer: "" + # extra annotations for the ingress + annotations: {} + # | + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" logLevel: "info" @@ -56,15 +64,6 @@ global: externalNamespace: "" internalNamespace: "" - - persistence: - enabled: true -## Persistent Volume Storage Class - ## If defined, storageClassName: - ## If set to "-", storageClassName: "", which disables dynamic provisioning - ## If undefined (the default) or set to null, no storageClassName spec is - ## set, choosing the default provisioner. - ## RevisionHistory ## If defined, set the revisionHistoryLimit of the deployment, defaults to 3 ## RevisionHistoryLimit is number of old ReplicaSets to retain to allow rollback. @@ -96,20 +95,8 @@ global: # Not to be used in production but handy for testing. tls: enabled: true - caCert: -# secretName is the name of the Kubernetes secret that holds the root CA certificate. -# A Kubernetes secret must be in the same namespace that sda-pipeline is installed into. - secretName: "" -# secretKey is the key within the Kubernetes secret that holds the root CA certificate. - secretKey: "" -# PEM-encoded CA certificates that will be added to the trusted system CAs -# cert: | -# -----BEGIN CERTIFICATE----- -# MIIC7jCCApSgAwIBAgIRAIq2zQEVexqxvtxP6J0bXAwwCgYIKoZIzj0EAwIwgbkx -# ... -# @type: string - cert: - + issuer: "" + clusterIssuer: "" # If an external secrets management service is used set this to true vaultSecrets: false @@ -147,11 +134,17 @@ global: nfsPath: "" auth: + # @param elixirID, client ID to the Elixir OIDC for the service endpoint elixirID: + # @param elixirSecret, client secret to the Elixir OIDC for the service endpoint elixirSecret: + # @param jwtSecret, name of the secret holding the jwt signing key jwtSecret: + # @param jwtAlg, cipher type of the signing key jwtAlg: ES256 + # @param jwtKey, name of the signing key jwtKey: + # @param jwtPub, name of the public signing key jwtPub: broker: @@ -168,8 +161,11 @@ global: vhost: "/" cega: + ## @param host, URI to CEGA NSS server users endpoint host: "" + ## @param user, usernamen for accessing the CEGA NSS host user: "" + ## @param password, password for accessing the CEGA NSS host password: "" c4gh: @@ -186,9 +182,10 @@ global: passIngest: "" passOutgest: "" port: 5432 - sslMode: "verify-full" + sslMode: "verify-ca" doa: + enabled: false outbox: enabled: false # MQ queue name for files/datasets export requests @@ -208,6 +205,7 @@ global: envFile: env download: + enabled: true # session key expiration time in seconds # default value = -1 for disabled state # a positive integer enables sessions @@ -216,7 +214,7 @@ global: # this should translate into JSON array of objects # with 2 keys iss and jku trusted: - configPath: "" + configPath: "iss" configFile: "iss.json" iss: - iss: "https://login.elixir-czech.org/oidc/" @@ -305,7 +303,7 @@ auth: name: auth replicaCount: 1 repository: ghcr.io/neicnordic/sda-auth - imageTag: v0.6.3 + imageTag: v0.6.11 imagePullPolicy: IfNotPresent resources: requests: @@ -326,7 +324,7 @@ backup: deploy: false replicaCount: 1 repository: ghcr.io/neicnordic/sda-pipeline - imageTag: v0.2.11 + imageTag: v0.2.25 imagePullPolicy: IfNotPresent resources: requests: @@ -389,7 +387,7 @@ finalize: name: finalize replicaCount: 1 repository: ghcr.io/neicnordic/sda-pipeline - imageTag: v0.2.11 + imageTag: v0.2.25 imagePullPolicy: IfNotPresent resources: requests: @@ -409,7 +407,7 @@ ingest: name: ingest replicaCount: 1 repository: ghcr.io/neicnordic/sda-pipeline - imageTag: v0.2.11 + imageTag: v0.2.25 imagePullPolicy: IfNotPresent resources: requests: @@ -430,7 +428,7 @@ intercept: name: ingest replicaCount: 1 repository: ghcr.io/neicnordic/sda-pipeline - imageTag: v0.2.11 + imageTag: v0.2.25 imagePullPolicy: IfNotPresent resources: requests: @@ -449,7 +447,7 @@ intercept: mapper: replicaCount: 1 repository: ghcr.io/neicnordic/sda-pipeline - imageTag: v0.2.11 + imageTag: v0.2.25 imagePullPolicy: IfNotPresent resources: requests: @@ -468,7 +466,7 @@ mapper: s3Inbox: name: s3Inbox repository: ghcr.io/neicnordic/sda-s3proxy - imageTag: v0.1.16 + imageTag: v0.1.44 imagePullPolicy: IfNotPresent replicaCount: 1 resources: @@ -489,7 +487,7 @@ s3Inbox: sftpInbox: name: sftpInbox repository: ghcr.io/neicnordic/sda-inbox-sftp - imageTag: v1.7.8 + imageTag: v1.7.18 imagePullPolicy: IfNotPresent replicaCount: 1 resources: @@ -516,11 +514,13 @@ releasetest: imagePullPolicy: IfNotPresent tls: secretName: "" + secrets: + accessToken: verify: replicaCount: 1 repository: ghcr.io/neicnordic/sda-pipeline - imageTag: v0.2.11 + imageTag: v0.2.25 imagePullPolicy: IfNotPresent resources: requests: diff --git a/dev_tools/cega/cega-issuer.yaml b/dev_tools/cega/cega-issuer.yaml new file mode 100644 index 00000000..85d775ae --- /dev/null +++ b/dev_tools/cega/cega-issuer.yaml @@ -0,0 +1,34 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cega-certs +spec: + # Secret names are always required. + secretName: cega-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: cega + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - server auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - cega-mq + - cega-users + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: ca-issuer + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: Issuer + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io diff --git a/dev_tools/cega/cega.conf b/dev_tools/cega/cega.conf index b54ad232..acaeb0eb 100644 --- a/dev_tools/cega/cega.conf +++ b/dev_tools/cega/cega.conf @@ -1,15 +1,15 @@ listeners.ssl.default = 5671 -ssl_options.cacertfile = /etc/rabbitmq/ssl/root.ca.crt -ssl_options.certfile = /etc/rabbitmq/ssl/cega-mq.crt -ssl_options.keyfile = /etc/rabbitmq/ssl/cega-mq.key +ssl_options.cacertfile = /etc/rabbitmq/ssl/ca.crt +ssl_options.certfile = /etc/rabbitmq/ssl/tls.crt +ssl_options.keyfile = /etc/rabbitmq/ssl/tls.key ssl_options.verify = verify_none ssl_options.fail_if_no_peer_cert = true ssl_options.versions.1 = tlsv1.2 management.load_definitions = /etc/rabbitmq/conf/cega.json management.listener.port = 15671 management.listener.ssl = true -management.listener.ssl_opts.cacertfile = /etc/rabbitmq/ssl/root.ca.crt -management.listener.ssl_opts.certfile = /etc/rabbitmq/ssl/cega-mq.crt -management.listener.ssl_opts.keyfile = /etc/rabbitmq/ssl/cega-mq.key +management.listener.ssl_opts.cacertfile = /etc/rabbitmq/ssl/ca.crt +management.listener.ssl_opts.certfile = /etc/rabbitmq/ssl/tls.crt +management.listener.ssl_opts.keyfile = /etc/rabbitmq/ssl/tls.key default_vhost = lega -disk_free_limit.absolute = 1GB \ No newline at end of file +disk_free_limit.absolute = 1GB diff --git a/dev_tools/cega/cega.json b/dev_tools/cega/cega.json index 39117cb8..57991b55 100644 --- a/dev_tools/cega/cega.json +++ b/dev_tools/cega/cega.json @@ -1,6 +1,6 @@ {"rabbit_version":"3.7", "users":[{"name":"lega", - "password_hash":"pULgDlZCWXfEEkD5hlEr9+PbbKiNAUHCqhAe0vnT/KB22fLz","hashing_algorithm":"rabbit_password_hashing_sha256","tags":"administrator"}], "vhosts":[{"name":"lega"}], + "password_hash":"tBwQTdorHZnIdJI7AUK71L56JVbYhjfhNoVO2y1nWmt2Cgdm","hashing_algorithm":"rabbit_password_hashing_sha256","tags":"administrator"}], "vhosts":[{"name":"lega"}], "permissions":[{"user":"lega", "vhost":"lega", "configure":".*", "write":".*", "read":".*"}], "parameters":[], "global_parameters":[{"name":"cluster_name", "value":"rabbit@localhost"}], @@ -20,4 +20,4 @@ {"source":"localega.v1","vhost":"lega","destination_type":"queue","arguments":{},"destination":"v1.files.verified","routing_key":"files.verified"}, {"source":"localega.v1","vhost":"lega","destination_type":"queue","arguments":{},"destination":"v1.files.completed","routing_key":"files.completed"}] -} \ No newline at end of file +} diff --git a/dev_tools/cega/deploy.yaml b/dev_tools/cega/deploy.yaml new file mode 100644 index 00000000..524dd23c --- /dev/null +++ b/dev_tools/cega/deploy.yaml @@ -0,0 +1,172 @@ +--- +# Source: cega/templates/cega-deploy.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cega-users + labels: + role: fake-users +spec: + replicas: 1 + selector: + matchLabels: + app: cega-users + template: + metadata: + labels: + app: cega-users + role: fake-users + spec: + serviceAccountName: cega + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + containers: + - name: cega-users + image: "egarchive/lega-base:release.v0.2.0" + imagePullPolicy: "Always" + command: ["python", "/cega/users.py", "0.0.0.0", "8443", "/cega/users.json"] + securityContext: + allowPrivilegeEscalation: false + env: + - name: LEGA_INSTANCES + value: legatest + - name: CEGA_USERS_PASSWORD + value: OfEoDPVadIfd4CZUWkisrrVQbJ2yQPIH + - name: CEGA_USERS_USER + value: legatest + ports: + - name: cega-users + containerPort: 443 + protocol: TCP + volumeMounts: + - name: cega-config + mountPath: /cega + - name: cega-certs + mountPath: /tls/ + volumes: + - name: cega-config + secret: + secretName: cega-users-config + defaultMode: 0440 + - name: cega-certs + secret: + secretName: cega-certs + defaultMode: 0440 +--- +# Source: cega/templates/cegamq-deploy.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cega-mq + labels: + role: cega-broker + app: cega-mq +spec: + replicas: 1 + selector: + matchLabels: + app: cega-mq + template: + metadata: + labels: + app: cega-mq + spec: + serviceAccountName: cega + securityContext: + runAsUser: 100 + runAsGroup: 101 + fsGroup: 101 + containers: + - name: cega-mq + image: "rabbitmq:3.7.8-management-alpine" + imagePullPolicy: "IfNotPresent" + securityContext: + allowPrivilegeEscalation: false + env: + - name: RABBITMQ_CONFIG_FILE + value: /etc/rabbitmq/conf/cega + - name: RABBITMQ_ENABLED_PLUGINS_FILE + value: /etc/rabbitmq/conf/cega.plugins + ports: + - containerPort: 15671 + protocol: TCP + name: https + - containerPort: 15672 + protocol: TCP + name: http + - containerPort: 5672 + name: amqp + - containerPort: 5671 + name: amqps + volumeMounts: + - name: conf + mountPath: /etc/rabbitmq/conf + - name: rabbitmq + mountPath: /var/lib/rabbitmq + - name: ssl-certs + mountPath: /etc/rabbitmq/ssl + volumes: + - name: ssl-certs + secret: + secretName: cega-certs + defaultMode: 0440 + - name: conf + secret: + secretName: cega-mq-config + defaultMode: 0440 + - name: rabbitmq + emptyDir: {} +--- +# Source: cega/templates/cega-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: cega-users + labels: + app: cega-users +spec: + ports: + - port: 443 + targetPort: 8443 + protocol: TCP + selector: + app: cega-users +--- +# Source: cega/templates/cegamq-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: cega-mq + labels: + app: cega-mq +spec: + ports: + - port: 5672 + targetPort: 5672 + protocol: TCP + name: amqp + - port: 5671 + targetPort: 5671 + protocol: TCP + name: amqps + - port: 4369 + name: epmd + - port: 25672 + name: rabbitmq-dist + selector: + app: cega-mq +--- +# Source: cega/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: cega-serviceAccount + chart: cega-0.3.3 + heritage: Helm + release: cega + name: cega + namespace: default + diff --git a/dev_tools/cega/users.py b/dev_tools/cega/users.py new file mode 100644 index 00000000..74fc5a3b --- /dev/null +++ b/dev_tools/cega/users.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3.6 +# -*- coding: utf-8 -*- + +''' +Test server to act as CentralEGA endpoint for users + +:author: Frédéric Haziza +:copyright: (c) 2018, EGA System Developers. +''' + +import sys +import os +import logging +import asyncio +import json +from base64 import b64decode +import ssl + +from aiohttp import web + +#logging.basicConfig(format='[%(asctime)s][%(levelname)-8s] (L:%(lineno)s) %(message)s', datefmt='%Y-%m-%d %H:%M:%S') +logging.basicConfig(format='[%(levelname)-8s] (L:%(lineno)s) %(message)s') +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.INFO) + +filepath = None +instances = {} +store = None +usernames = {} +uids = {} + +def fetch_user_info(identifier, query): + id_type = query.get('idType', None) + if not id_type: + raise web.HTTPBadRequest(text='Missing or wrong idType') + LOG.info(f'Requesting User {identifier} [type {id_type}]') + if id_type == 'username': + pos = usernames.get(identifier, None) + return store[pos] if pos is not None else None + if id_type == 'uid': + try: + pos = uids.get(int(identifier), None) + return store[pos] if pos is not None else None + except Exception: + return None + raise web.HTTPBadRequest(text='Missing or wrong idType') + +async def user(request): + # Authenticate + auth_header = request.headers.get('AUTHORIZATION') + if not auth_header: + raise web.HTTPUnauthorized(text=f'Protected access\n') + _, token = auth_header.split(None, 1) # Skipping the Basic keyword + instance, passwd = b64decode(token).decode().split(':', 1) + info = instances.get(instance) + if info is None or info != passwd: + raise web.HTTPUnauthorized(text=f'Protected access\n') + + # Reload users list + load_users() + + # Find user + user_info = fetch_user_info(request.match_info['identifier'], request.rel_url.query) + if user_info is None: + raise web.HTTPBadRequest(text=f'No info for that user\n') + return web.json_response({ 'header': { "apiVersion": "v1", + "code": "200", + "service": "users", + "developerMessage": "", + "userMessage": "OK", + "errorCode": "1", + "docLink": "https://ega-archive.org", + "errorStack": "" }, + 'response': { "numTotalResults": 1, + "resultType": "eu.crg.ega.microservice.dto.lega.v1.users.LocalEgaUser", + "result": [ user_info ]} + }) + +def main(): + + if len(sys.argv) < 3: + print('Usage: {sys.argv[0] }', file=sys.stderr) + sys.exit(2) + + host = sys.argv[1] + port = sys.argv[2] + + global filepath + filepath = sys.argv[3] + + server = web.Application() + load_users() + + # Registering the routes + server.router.add_get('/lega/v1/legas/users/{identifier}', user, name='user') + + # SSL settings + cacertfile = '/tls/ca.crt' + certfile = '/tls/tls.crt' + keyfile = '/tls/tls.key' + + ssl_ctx = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH, cafile=cacertfile) + ssl_ctx.check_hostname = False + ssl_ctx.verify_mode = ssl.CERT_NONE + + ssl_ctx.load_cert_chain(certfile, keyfile=keyfile) + + # aaaand... cue music + web.run_app(server, host=host, port=port, shutdown_timeout=0, ssl_context=ssl_ctx) + + +def load_users(): + # Initialization + global filepath, instances, store, usernames, uids + instances[os.environ[f'CEGA_USERS_USER']] = os.environ[f'CEGA_USERS_PASSWORD'] #'legatest' # Hard-coding legatest:legatest + with open(filepath, 'rt') as f: + store = json.load(f) + for i, d in enumerate(store): + usernames[d['username']] = i # No KeyError, should be there + uids[d['uid']] = i + + +if __name__ == '__main__': + main() diff --git a/dev_tools/config/cega.yaml b/dev_tools/config/cega.yaml deleted file mode 100644 index 5f5ec88c..00000000 --- a/dev_tools/config/cega.yaml +++ /dev/null @@ -1,6 +0,0 @@ -secrets: - cega_users_pass: "OfEoDPVadIfd4CZUWkisrrVQbJ2yQPIH" - cega_mq_pass: "xFOumcdbIftsXxZqOW5sE8BB2BrXoMVC" -config: - cega_users_user: "legatest" - tls_ca_root_file: root.ca.crt \ No newline at end of file diff --git a/dev_tools/config/cert-issuer.yaml b/dev_tools/config/cert-issuer.yaml new file mode 100644 index 00000000..1c2440aa --- /dev/null +++ b/dev_tools/config/cert-issuer.yaml @@ -0,0 +1,30 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: selfsigned-ca +spec: + isCA: true + commonName: selfsigned-ca + secretName: root-secret + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: selfsigned-issuer + kind: Issuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-issuer +spec: + ca: + secretName: root-secret diff --git a/dev_tools/config/minio-issuer.yaml b/dev_tools/config/minio-issuer.yaml new file mode 100644 index 00000000..e33f40b5 --- /dev/null +++ b/dev_tools/config/minio-issuer.yaml @@ -0,0 +1,33 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: minio-certs +spec: + # Secret names are always required. + secretName: minio-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: cega + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - server auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - minio + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: ca-issuer + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: Issuer + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io diff --git a/dev_tools/config/no-tls.yaml b/dev_tools/config/no-tls.yaml index 3a03c175..7fc237b8 100644 --- a/dev_tools/config/no-tls.yaml +++ b/dev_tools/config/no-tls.yaml @@ -15,8 +15,8 @@ global: elixirSecret: DfCieZLuBU jwtSecret: oidc jwtAlg: ES256 - jwtKey: token.key - jwtPub: token.pub + jwtKey: jwt.key + jwtPub: jwt.pub backupArchive: storageType: s3 s3Url: "http://minio" @@ -34,8 +34,8 @@ global: host: "postgres-sda-db" c4gh: secretName: c4gh - keyFile: c4gh.key - publicFile: c4gh.pub + keyFile: c4gh.sec.pem + publicFile: c4gh.pub.pem elixir: oidcdHost: "http://oidc-server:8080" jwkPath: "" diff --git a/dev_tools/config/posix.yaml b/dev_tools/config/posix.yaml index 50dcb88a..995efc5c 100644 --- a/dev_tools/config/posix.yaml +++ b/dev_tools/config/posix.yaml @@ -22,14 +22,14 @@ global: backupRoutingKey: "backup" c4gh: secretName: c4gh - keyFile: c4gh.key - publicFile: c4gh.pub + keyFile: c4gh.sec.pem + publicFile: c4gh.pub.pem cega: - host: "cega-users" + host: "cega-users/lega/v1/legas/users" user: "legatest" db: host: "postgres-sda-db" - sslMode: "verify-full" + sslMode: "verify-ca" inbox: storageType: posix path: "/inbox" @@ -42,9 +42,6 @@ global: logLevel: debug tls: enabled: true - caCert: - secretName: "ca-root" - secretKey: "ca.crt" backup: deploy: true tls: diff --git a/dev_tools/config/s3.yaml b/dev_tools/config/s3.yaml index 633eb1f3..308e2b5b 100644 --- a/dev_tools/config/s3.yaml +++ b/dev_tools/config/s3.yaml @@ -16,8 +16,8 @@ global: elixirSecret: DfCieZLuBU jwtSecret: oidc jwtAlg: ES256 - jwtKey: token.key - jwtPub: token.pub + jwtKey: jwt.key + jwtPub: jwt.pub backupArchive: storageType: s3 s3Url: "https://minio" @@ -39,11 +39,11 @@ global: password: 3iSGc5loLN7hu2EwT2Z2CzCT26iUrk3t db: host: "postgres-sda-db" - sslMode: "verify-full" + sslMode: "verify-ca" c4gh: secretName: c4gh - keyFile: c4gh.key - publicFile: c4gh.pub + keyFile: c4gh.sec.pem + publicFile: c4gh.sec.pub elixir: oidcdHost: "http://oidc-server:8080" jwkPath: "" @@ -51,7 +51,7 @@ global: storageType: s3 # existingClaim: test s3Url: "https://minio" - s3Bucket: "archive" + s3Bucket: "inbox" s3AccessKey: idDQBxLpXoM8l88l s3SecretKey: ABd6XCIvNWj7JULbrqBf8tB7q9BoHJSc s3CaFile: ca.crt @@ -64,9 +64,6 @@ global: logLevel: debug tls: enabled: true - caCert: - secretName: "ca-root" - secretKey: "ca.crt" auth: tls: secretName: "auth-certs" @@ -74,9 +71,6 @@ backup: deploy: true tls: secretName: "backup-certs" - certFile: "backup.crt" - keyFile: "backup.key" - caCert: "ca.crt" doa: tls: secretName: "doa-certs" diff --git a/dev_tools/scripts/create-s3-buckets.sh b/dev_tools/scripts/create-s3-buckets.sh index c85aad1c..0155550e 100755 --- a/dev_tools/scripts/create-s3-buckets.sh +++ b/dev_tools/scripts/create-s3-buckets.sh @@ -2,7 +2,7 @@ set -e if [ ! -f s3cmd.conf ]; then - cat >> "s3cmd.conf" <>"s3cmd.conf" <&2; return 1 ;; + esac + printf '%s.%s\n' "${signed_content}" "${sig}" +} + +iat=$(date +%s) +exp=$(date --date=tomorrow +%s) + +test_payload='{ + "at_hash": "J_fA458SPsXFV6lJQL1l-w", + "aud": "XC56EL11xx", + "email": "dummy.tester@example.org", + "exp": '"$exp"', + "iat": '"$iat"', + "iss": "http://sda-sda-svc-auth", + "kid": "d87f2d01d1a4abb16e1eb88f6561e5067f3a6430174b8fcd0b6bf61434d6c5c8", + "name": "Dummy Tester", + "sid": "1ad14eb5-9b51-40c0-a52a-154a5a3792d5", + "sub": "dummy" +}' + + +sign "$@" diff --git a/dev_tools/scripts/ssl.cnf b/dev_tools/scripts/ssl.cnf index f8b1dc5d..d6804a4c 100644 --- a/dev_tools/scripts/ssl.cnf +++ b/dev_tools/scripts/ssl.cnf @@ -182,7 +182,7 @@ subjectKeyIdentifier = hash authorityKeyIdentifier = keyid,issuer:always keyUsage = critical, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:inbox,DNS:inbox.default.svc.cluster.local,DNS:inbox.default +subjectAltName = DNS:sda-sda-svc-inbox,DNS:inbox.default.svc.cluster.local,DNS:inbox.default,DNS:localhost [ doa_cert ] # Extensions for server certificates (`man x509v3_config`). @@ -191,7 +191,7 @@ subjectKeyIdentifier = hash authorityKeyIdentifier = keyid,issuer:always keyUsage = critical, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:doa,DNS:doa.default.svc.cluster.local,DNS:doa.default +subjectAltName = DNS:sda-sda-svc-doa,DNS:doa.default.svc.cluster.local,DNS:doa.default,DNS:localhost [ download_cert ] # Extensions for server certificates (`man x509v3_config`). @@ -200,7 +200,7 @@ subjectKeyIdentifier = hash authorityKeyIdentifier = keyid,issuer:always keyUsage = critical, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:download,DNS:download.default.svc.cluster.local,DNS:download.default +subjectAltName = DNS:sda-sda-svc-download,DNS:download.default.svc.cluster.local,DNS:download.default,DNS:localhost [ auth_cert ] # Extensions for server certificates (`man x509v3_config`). @@ -209,7 +209,7 @@ subjectKeyIdentifier = hash authorityKeyIdentifier = keyid,issuer:always keyUsage = critical, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:auth,DNS:auth.default.svc.cluster.local,DNS:auth.default +subjectAltName = DNS:sda-sda-svc-auth,DNS:auth.default.svc.cluster.local,DNS:auth.default,DNS:localhost [ tester_cert ] # Extensions for server certificates (`man x509v3_config`). @@ -222,23 +222,14 @@ subjectAltName = DNS:tester,DNS:tester.default.svc.cluster.local,DNS:tester.defa # CEGA certificates -[ cega_users_cert ] +[ cega ] # Extensions for server certificates (`man x509v3_config`). basicConstraints = critical,CA:FALSE subjectKeyIdentifier = hash authorityKeyIdentifier = keyid,issuer:always keyUsage = critical, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:cega-users,DNS:cega-users.default.svc.cluster.local,DNS:cega-users.default - -[ cega_mq_cert ] -# Extensions for server certificates (`man x509v3_config`). -basicConstraints = critical,CA:FALSE -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always -keyUsage = critical, digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:cega-mq,DNS:cega-mq.default.svc.cluster.local,DNS:cega-mq.default +subjectAltName = DNS:cega-mq,DNS:cega-users [ crl_ext ] # Extension for CRLs (`man x509v3_config`). diff --git a/dev_tools/scripts/svc-setup.sh b/dev_tools/scripts/svc-setup.sh index 0c3c5225..fce2f766 100755 --- a/dev_tools/scripts/svc-setup.sh +++ b/dev_tools/scripts/svc-setup.sh @@ -6,19 +6,16 @@ basedir="sda-deploy-init/config" ## cega config and certs mkdir -p LocalEGA-helm/ega-charts/cega/config/certs cp -r dev_tools/cega/* LocalEGA-helm/ega-charts/cega/config/ -cp "${basedir}"/certs/ca.crt LocalEGA-helm/ega-charts/cega/config/certs/root.ca.crt -cp "${basedir}"/certs/cega-users.crt LocalEGA-helm/ega-charts/cega/config/certs/cega-users.ca.crt -cp "${basedir}"/certs/cega-users.key LocalEGA-helm/ega-charts/cega/config/certs/cega-users.ca.key -cp "${basedir}"/certs/cega-mq.crt LocalEGA-helm/ega-charts/cega/config/certs/cega-mq.crt -cp "${basedir}"/certs/cega-mq.key LocalEGA-helm/ega-charts/cega/config/certs/cega-mq.key +cp "${basedir}"/certs/ca.crt LocalEGA-helm/ega-charts/cega/config/certs/ca.crt +cp "${basedir}"/certs/cega.crt LocalEGA-helm/ega-charts/cega/config/certs/tls.crt +cp "${basedir}"/certs/cega.key LocalEGA-helm/ega-charts/cega/config/certs/tls.key ## sda-svc certs -kubectl create secret generic ca-root --from-file="${basedir}"/certs/ca.crt - for n in backup doa finalize inbox ingest intercept verify mapper auth tester download do - kubectl create secret tls $n-certs \ - --cert="${basedir}"/certs/$n.crt \ - --key="${basedir}"/certs/$n.key + kubectl create secret generic $n-certs \ + --from-file="${basedir}"/certs/ca.crt \ + --from-file=tls.crt="${basedir}"/certs/$n.crt \ + --from-file=tls.key="${basedir}"/certs/$n.key done diff --git a/dev_tools/scripts/wait-for-pods.sh b/dev_tools/scripts/wait-for-pods.sh index 1feff4eb..cbb89a13 100755 --- a/dev_tools/scripts/wait-for-pods.sh +++ b/dev_tools/scripts/wait-for-pods.sh @@ -1,22 +1,47 @@ #!/bin/bash set -e -if [ -n "$1" ]; then SVCNAME=$1; else SVCNAME="svc"; fi -if [ -n "$2" ]; then LABEL=$2; else LABEL="role"; fi -if [ -n "$3" ]; then NAMESPACE=$3; else NAMESPACE="default"; fi +base_list="backup download finalize inbox ingest mapper verify" -for p in $SVCNAME -do +if [ -n "$1" ]; then + case "$1" in + federated_s3_svc_list) + SVCNAME="$base_list auth intercept" + ;; + + federated_posix_svc_list) + SVCNAME="$base_list intercept" + ;; + + standalone_s3_svc_list) + SVCNAME="$base_list auth" + ;; + + standalone_posix_svc_list) + SVCNAME="$base_list" + ;; + esac +fi + +if [ -n "$2" ]; then + LABEL=$2 +else + LABEL="role" +fi +if [ -n "$3" ]; then + NAMESPACE=${3:-default} +fi + +for p in $SVCNAME; do RETRY_TIMES=0 - until kubectl get pods -n=$NAMESPACE -l=$LABEL=$p -o jsonpath='{..status.containerStatuses[*].ready}' | grep "true" - do - echo "waiting for $p to become ready"; - RETRY_TIMES=$((RETRY_TIMES+1)); - if [ $RETRY_TIMES -eq 30 ]; then - kubectl describe pod -n=$NAMESPACE -l=$LABEL=$p - kubectl logs -n=$NAMESPACE -l=$LABEL=$p - exit 1; + until kubectl get pods -n "$NAMESPACE" -l "$LABEL=$p" -o jsonpath='{..status.containerStatuses[*].ready}' | grep "true"; do + echo "waiting for $p to become ready" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + kubectl describe pod -n "$NAMESPACE" -l "$LABEL"="$p" + kubectl logs -n "$NAMESPACE" -l "$LABEL=$p" + exit 1 fi - sleep 10; + sleep 10 done done