From 661941dbf17c96241fb9674cdc6adcf15607f90d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 26 Jan 2024 16:35:14 -0400 Subject: [PATCH 01/70] feat: Split datatracker/celery start scripts (#6974) --- dev/build/celery-start.sh | 22 ++++++++++++++++++++++ dev/build/datatracker-start.sh | 10 ++++++++++ dev/build/start.sh | 28 +++++++++++++++++++--------- 3 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 dev/build/celery-start.sh create mode 100644 dev/build/datatracker-start.sh diff --git a/dev/build/celery-start.sh b/dev/build/celery-start.sh new file mode 100644 index 0000000000..c229defa2f --- /dev/null +++ b/dev/build/celery-start.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Run a celery worker +# +echo "Running Datatracker checks..." +./ietf/manage.py check + +cleanup () { + # Cleanly terminate the celery app by sending it a TERM, then waiting for it to exit. + if [[ -n "${celery_pid}" ]]; then + echo "Gracefully terminating celery worker. This may take a few minutes if tasks are in progress..." + kill -TERM "${celery_pid}" + wait "${celery_pid}" + fi +} + +trap 'trap "" TERM; cleanup' TERM + +# start celery in the background so we can trap the TERM signal +celery "$@" & +celery_pid=$! +wait "${celery_pid}" diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh new file mode 100644 index 0000000000..ef64ca7b30 --- /dev/null +++ b/dev/build/datatracker-start.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +echo "Running Datatracker checks..." +./ietf/manage.py check + +echo "Running Datatracker migrations..." +./ietf/manage.py migrate --settings=settings_local + +echo "Starting Datatracker..." +./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local diff --git a/dev/build/start.sh b/dev/build/start.sh index ef64ca7b30..f20d264beb 100644 --- a/dev/build/start.sh +++ b/dev/build/start.sh @@ -1,10 +1,20 @@ #!/bin/bash - -echo "Running Datatracker checks..." -./ietf/manage.py check - -echo "Running Datatracker migrations..." -./ietf/manage.py migrate --settings=settings_local - -echo "Starting Datatracker..." -./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local +# +# Environment config: +# +# CONTAINER_ROLE - datatracker, celery, or beat (defaults to datatracker) +# +case "${CONTAINER_ROLE:-datatracker}" in + datatracker) + exec ./datatracker-start.sh + ;; + celery) + exec ./celery-start.sh --app=ietf worker + ;; + beat) + exec ./celery-start.sh --app=ietf beat + ;; + *) + echo "Unknown role '${CONTAINER_ROLE}'" + exit 255 +esac From ff228a5913b01aeccab8926e2642e2fcc280bcdb Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 26 Jan 2024 15:01:38 -0600 Subject: [PATCH 02/70] feat: merge dt and celery container builds (#6973) * feat: merge dt and celery container builds * fix: mount /a instead of /assets * feat: use consolidated script * fix: get start.sh dependencies. Remove watchdog * fix: correct source path --- dev/build/Dockerfile | 43 ++++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 2ffce35495..e6173cd508 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,17 +1,26 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:latest -LABEL maintainer="IETF Tools Team " - -ENV DEBIAN_FRONTEND=noninteractive - -COPY . . -COPY ./dev/build/start.sh ./start.sh -RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt -RUN chmod +x start.sh && \ - chmod +x docker/scripts/app-create-dirs.sh && \ - sh ./docker/scripts/app-create-dirs.sh - -VOLUME [ "/assets" ] - -EXPOSE 8000 - -CMD ["./start.sh"] \ No newline at end of file +FROM ghcr.io/ietf-tools/datatracker-app-base:latest +LABEL maintainer="IETF Tools Team " + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get purge -y imagemagick imagemagick-6-common + +COPY . . +COPY ./dev/build/start.sh ./start.sh +COPY ./dev/build/datatracker-start.sh ./datatracker-start.sh +COPY ./dev/build/celery-start.sh ./celery-start.sh + +RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt + +RUN chmod +x datatracker-start.sh && \ + chmod +x celery-start.sh \ + chmod +x docker/scripts/app-create-dirs.sh && \ + sh ./docker/scripts/app-create-dirs.sh + +RUN mkdir -p /a + +VOLUME [ "/a" ] + +EXPOSE 8000 + +CMD ["./start.sh"] From c867d6f0da40721da32c5304e60b08616d1bed2b Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Fri, 26 Jan 2024 16:54:03 -0500 Subject: [PATCH 03/70] ci: helm values + deployment files for all 5 services (wip) --- helm/templates/_helpers.tpl | 127 +++- .../beat.yaml} | 1 + helm/templates/deployments/celery.yaml | 67 ++ helm/templates/deployments/datatracker.yaml | 67 ++ helm/templates/deployments/memcached.yaml | 67 ++ helm/templates/deployments/rabbitmq.yaml | 67 ++ helm/templates/ingress.yaml | 20 +- helm/templates/rbac/beat-serviceaccount.yaml | 12 + .../templates/rbac/celery-serviceaccount.yaml | 12 + .../rbac/datatracker-serviceaccount.yaml | 12 + .../rbac/memcached-serviceaccount.yaml | 12 + .../rbac/rabbitmq-serviceaccount.yaml | 12 + helm/templates/service.yaml | 6 +- helm/templates/serviceaccount.yaml | 12 - helm/values.yaml | 587 ++++++++++++++---- 15 files changed, 943 insertions(+), 138 deletions(-) rename helm/templates/{deployment.yaml => deployments/beat.yaml} (95%) create mode 100644 helm/templates/deployments/celery.yaml create mode 100644 helm/templates/deployments/datatracker.yaml create mode 100644 helm/templates/deployments/memcached.yaml create mode 100644 helm/templates/deployments/rabbitmq.yaml create mode 100644 helm/templates/rbac/beat-serviceaccount.yaml create mode 100644 helm/templates/rbac/celery-serviceaccount.yaml create mode 100644 helm/templates/rbac/datatracker-serviceaccount.yaml create mode 100644 helm/templates/rbac/memcached-serviceaccount.yaml create mode 100644 helm/templates/rbac/rabbitmq-serviceaccount.yaml delete mode 100644 helm/templates/serviceaccount.yaml diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 071e9b824c..0f6d455856 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -23,6 +23,91 @@ If release name contains chart name it will be used as a full name. {{- end }} {{- end }} +{{/* +Create a fully qualified datatracker name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.datatracker.fullname" -}} +{{- if .Values.datatracker.fullnameOverride -}} +{{- .Values.datatracker.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified celery name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.celery.fullname" -}} +{{- if .Values.celery.fullnameOverride -}} +{{- .Values.celery.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.celery.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.celery.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified celery name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.beat.fullname" -}} +{{- if .Values.beat.fullnameOverride -}} +{{- .Values.beat.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.beat.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.beat.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified rabbitmq name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.rabbitmq.fullname" -}} +{{- if .Values.rabbitmq.fullnameOverride -}} +{{- .Values.rabbitmq.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified memcached name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.memcached.fullname" -}} +{{- if .Values.memcached.fullnameOverride -}} +{{- .Values.memcached.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + {{/* Create chart name and version as used by the chart label. */}} @@ -53,10 +138,42 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{/* Create the name of the service account to use */}} -{{- define "datatracker.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "datatracker.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} +{{- define "datatracker.serviceAccountName.datatracker" -}} +{{- if .Values.serviceAccounts.datatracker.create -}} + {{ default (include "datatracker.datatracker.fullname" .) .Values.serviceAccounts.datatracker.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.datatracker.name }} +{{- end -}} {{- end }} + +{{- define "datatracker.serviceAccountName.celery" -}} +{{- if .Values.serviceAccounts.celery.create -}} + {{ default (include "datatracker.celery.fullname" .) .Values.serviceAccounts.celery.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.celery.name }} +{{- end -}} +{{- end }} + +{{- define "datatracker.serviceAccountName.beat" -}} +{{- if .Values.serviceAccounts.beat.create -}} + {{ default (include "datatracker.beat.fullname" .) .Values.serviceAccounts.beat.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.beat.name }} +{{- end -}} +{{- end }} + +{{- define "datatracker.serviceAccountName.rabbitmq" -}} +{{- if .Values.serviceAccounts.rabbitmq.create -}} + {{ default (include "datatracker.rabbitmq.fullname" .) .Values.serviceAccounts.rabbitmq.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.rabbitmq.name }} +{{- end -}} +{{- end }} + +{{- define "datatracker.serviceAccountName.memcached" -}} +{{- if .Values.serviceAccounts.memcached.create -}} + {{ default (include "datatracker.memcached.fullname" .) .Values.serviceAccounts.memcached.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.memcached.name }} +{{- end -}} {{- end }} \ No newline at end of file diff --git a/helm/templates/deployment.yaml b/helm/templates/deployments/beat.yaml similarity index 95% rename from helm/templates/deployment.yaml rename to helm/templates/deployments/beat.yaml index b47c41a970..41f76b7f4f 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployments/beat.yaml @@ -29,6 +29,7 @@ spec: image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} env: + CONTAINER_ROLE: beat {{- if .Values.env }} {{- toYaml .Values.env | nindent 12 }} {{- end }} diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml new file mode 100644 index 0000000000..f346783072 --- /dev/null +++ b/helm/templates/deployments/celery.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + env: + CONTAINER_ROLE: celery + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml .Values.startupProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml new file mode 100644 index 0000000000..1f403787e2 --- /dev/null +++ b/helm/templates/deployments/datatracker.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + env: + CONTAINER_ROLE: datatracker + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml .Values.startupProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/deployments/memcached.yaml b/helm/templates/deployments/memcached.yaml new file mode 100644 index 0000000000..41f76b7f4f --- /dev/null +++ b/helm/templates/deployments/memcached.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + env: + CONTAINER_ROLE: beat + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml .Values.startupProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/deployments/rabbitmq.yaml b/helm/templates/deployments/rabbitmq.yaml new file mode 100644 index 0000000000..41f76b7f4f --- /dev/null +++ b/helm/templates/deployments/rabbitmq.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + env: + CONTAINER_ROLE: beat + {{- if .Values.env }} + {{- toYaml .Values.env | nindent 12 }} + {{- end }} + {{- with .Values.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml .Values.startupProbe | nindent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index 8d9258cd83..68034601a5 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -1,9 +1,9 @@ -{{- if .Values.ingress.enabled -}} +{{- if .Values.datatracker.ingress.enabled -}} {{- $fullName := include "datatracker.fullname" . -}} {{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} +{{- if and .Values.datatracker.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class" .Values.datatracker.ingress.className}} {{- end }} {{- end }} {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} @@ -18,17 +18,17 @@ metadata: name: {{ $fullName }} labels: {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} + {{- with .Values.datatracker.ingress.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} + {{- if and .Values.datatracker.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.datatracker.ingress.className }} {{- end }} - {{- if .Values.ingress.tls }} + {{- if .Values.datatracker.ingress.tls }} tls: - {{- range .Values.ingress.tls }} + {{- range .Values.datatracker.ingress.tls }} - hosts: {{- range .hosts }} - {{ . | quote }} @@ -37,7 +37,7 @@ spec: {{- end }} {{- end }} rules: - {{- range .Values.ingress.hosts }} + {{- range .Values.datatracker.ingress.hosts }} - host: {{ .host | quote }} http: paths: diff --git a/helm/templates/rbac/beat-serviceaccount.yaml b/helm/templates/rbac/beat-serviceaccount.yaml new file mode 100644 index 0000000000..f9fa5830c3 --- /dev/null +++ b/helm/templates/rbac/beat-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.beat.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.beat" . }} + labels: + {{- include "datatracker.beat.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.beat.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/helm/templates/rbac/celery-serviceaccount.yaml b/helm/templates/rbac/celery-serviceaccount.yaml new file mode 100644 index 0000000000..267d1dce3e --- /dev/null +++ b/helm/templates/rbac/celery-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.celery.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.celery" . }} + labels: + {{- include "datatracker.celery.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.celery.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/helm/templates/rbac/datatracker-serviceaccount.yaml b/helm/templates/rbac/datatracker-serviceaccount.yaml new file mode 100644 index 0000000000..8f37799ba7 --- /dev/null +++ b/helm/templates/rbac/datatracker-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.datatracker.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.datatracker" . }} + labels: + {{- include "datatracker.datatracker.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.datatracker.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/helm/templates/rbac/memcached-serviceaccount.yaml b/helm/templates/rbac/memcached-serviceaccount.yaml new file mode 100644 index 0000000000..ba150a4065 --- /dev/null +++ b/helm/templates/rbac/memcached-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.memcached.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.memcached" . }} + labels: + {{- include "datatracker.memcached.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.memcached.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/helm/templates/rbac/rabbitmq-serviceaccount.yaml b/helm/templates/rbac/rabbitmq-serviceaccount.yaml new file mode 100644 index 0000000000..c9ce127482 --- /dev/null +++ b/helm/templates/rbac/rabbitmq-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.rabbitmq.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.rabbitmq" . }} + labels: + {{- include "datatracker.rabbitmq.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.rabbitmq.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} \ No newline at end of file diff --git a/helm/templates/service.yaml b/helm/templates/service.yaml index f1bdca0ad2..e7b310f98f 100644 --- a/helm/templates/service.yaml +++ b/helm/templates/service.yaml @@ -3,16 +3,16 @@ kind: Service metadata: name: {{include "datatracker.fullname" .}} labels: {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.service.annotations }} + {{- with .Values.datatracker.service.annotations }} annotations: {{- range $key, $value := . }} {{ $key }}: {{ $value | quote }} {{- end }} {{- end }} spec: - type: {{.Values.service.type}} + type: {{.Values.datatracker.service.type}} ports: - - port: {{ default "80" .Values.service.port}} + - port: {{ default "80" .Values.datatracker.service.port}} targetPort: http protocol: TCP name: http diff --git a/helm/templates/serviceaccount.yaml b/helm/templates/serviceaccount.yaml deleted file mode 100644 index 475fcd51f7..0000000000 --- a/helm/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml index 92efbce9dd..31f5e25cf8 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -2,117 +2,488 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. -replicaCount: 1 - -image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -livenessProbe: - httpGet: - path: /healthz - port: http - -readinessProbe: - httpGet: - path: /healthz - port: http - -startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - httpGet: - path: /healthz - port: http - -podAnnotations: {} -podLabels: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: datatracker.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi +## Define serviceAccount names for components. Defaults to component's fully qualified name. +## +serviceAccounts: + datatracker: + create: true + name: + annotations: {} + celery: + create: true + name: + annotations: {} + beat: + create: true + name: + annotations: {} + rabbitmq: + create: true + name: + annotations: {} + memcached: + create: true + name: + annotations: {} -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 +# ------------------------------------------------------------- +# DATATRACKER +# ------------------------------------------------------------- + +datatracker: + image: + repository: "ghcr.io/ietf-tools/datatracker" + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + # tag: "v1.1.0" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: datatracker.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + + livenessProbe: + httpGet: + path: /health + port: http + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + readinessProbe: + httpGet: + path: /health + port: http + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + httpGet: + path: /health + port: http + + # Additional volumes on the output Deployment definition. + volumes: [] + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: [] + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# CELERY +# ------------------------------------------------------------- + +celery: + image: + repository: "ghcr.io/ietf-tools/datatracker" + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + # tag: "v1.1.0" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + livenessProbe: + exec: + command: celery inspect ping + periodSeconds: 30 + timeoutSeconds: 5 + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: celery inspect ping + + # Additional volumes on the output Deployment definition. + volumes: [] + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: [] + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# BEAT +# ------------------------------------------------------------- + +beat: + image: + repository: "ghcr.io/ietf-tools/datatracker" + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + # tag: "v1.1.0" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + livenessProbe: + exec: + command: celery inspect ping + periodSeconds: 30 + timeoutSeconds: 5 -# Additional volumes on the output Deployment definition. -volumes: [] -# - name: foo -# secret: -# secretName: mysecret -# optional: false + podAnnotations: {} + podLabels: {} -# Additional volumeMounts on the output Deployment definition. -volumeMounts: [] -# - name: foo -# mountPath: "/etc/foo" -# readOnly: true + podSecurityContext: {} + # fsGroup: 2000 -nodeSelector: {} + replicaCount: 1 -tolerations: [] + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi -affinity: {} \ No newline at end of file + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: celery inspect ping + + # Additional volumes on the output Deployment definition. + volumes: [] + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: [] + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# RABBITMQ +# ------------------------------------------------------------- + +rabbitmq: + image: + repository: "ghcr.io/ietf-tools/datatracker-mq" + pullPolicy: IfNotPresent + tag: "latest" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + livenessProbe: + exec: + command: rabbitmq-diagnostics -q ping + periodSeconds: 30 + timeoutSeconds: 5 + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: rabbitmq-diagnostics -q ping + + # Additional volumes on the output Deployment definition. + volumes: [] + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: [] + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# MEMCACHED +# ------------------------------------------------------------- + +memcached: + image: + repository: "memcached" + pullPolicy: IfNotPresent + tag: "latest" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# COMMON +# ------------------------------------------------------------- + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 \ No newline at end of file From f980954336761cdc02c526c31fca6f6bb43c109b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 12 Mar 2024 18:01:26 -0300 Subject: [PATCH 04/70] ci: add missing "&&" --- dev/build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index e6173cd508..539dd88af7 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -13,7 +13,7 @@ COPY ./dev/build/celery-start.sh ./celery-start.sh RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt RUN chmod +x datatracker-start.sh && \ - chmod +x celery-start.sh \ + chmod +x celery-start.sh && \ chmod +x docker/scripts/app-create-dirs.sh && \ sh ./docker/scripts/app-create-dirs.sh From bf74f2db31a9ae5abd18de3cd2ed31b6e4663006 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 12 Mar 2024 18:25:50 -0300 Subject: [PATCH 05/70] ci: still need to chmod start.sh --- dev/build/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 539dd88af7..fe1b51f0bd 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -12,7 +12,8 @@ COPY ./dev/build/celery-start.sh ./celery-start.sh RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt -RUN chmod +x datatracker-start.sh && \ +RUN chmod +x start.sh && \ + chmod +x datatracker-start.sh && \ chmod +x celery-start.sh && \ chmod +x docker/scripts/app-create-dirs.sh && \ sh ./docker/scripts/app-create-dirs.sh From 6ed19bb2abdaa077f6004a0822980548f1880715 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 12 Mar 2024 18:40:07 -0300 Subject: [PATCH 06/70] ci: nearly deployable helm chart (wip) (#7176) * style: CRLF -> LF * build: Use AppVersion for image * chore: Helm YAML indent_size = 2 * ci: Deploy settings_local.py via helm * ci: misc progress on the helm chart (WIP) * ci: move configmap to env in values.yaml * ci: Refactor env and configmap formats * ci: merge fixup + helm debugging * ci: Fix rabbitmq.yaml and its values * ci: fix up other deployments * ci: fix copy error in beat.yaml and celery.yaml * ci: install correct images and volumes --- .editorconfig | 6 + helm/settings_local.py | 121 ++++++++++++++ helm/templates/_helpers.tpl | 2 +- helm/templates/configmap.yaml | 7 + helm/templates/deployments/beat.yaml | 139 ++++++++-------- helm/templates/deployments/celery.yaml | 45 ++--- helm/templates/deployments/datatracker.yaml | 47 +++--- helm/templates/deployments/memcached.yaml | 43 ++--- helm/templates/deployments/rabbitmq.yaml | 43 ++--- helm/templates/ingress.yaml | 4 +- helm/templates/rbac/beat-serviceaccount.yaml | 4 +- .../templates/rbac/celery-serviceaccount.yaml | 4 +- .../rbac/datatracker-serviceaccount.yaml | 4 +- .../rbac/memcached-serviceaccount.yaml | 4 +- .../rbac/rabbitmq-serviceaccount.yaml | 4 +- helm/values.yaml | 156 ++++++++++++++---- 16 files changed, 437 insertions(+), 196 deletions(-) create mode 100644 helm/settings_local.py create mode 100644 helm/templates/configmap.yaml diff --git a/.editorconfig b/.editorconfig index d6eafe8d8f..35c9168a28 100644 --- a/.editorconfig +++ b/.editorconfig @@ -50,3 +50,9 @@ indent_size = 2 [ietf/**.html] insert_final_newline = false + +# Settings for Helm chart yaml +# --------------------------------------------------------- +# Use 2-space indents +[helm/**.yaml] +indent_size = 2 \ No newline at end of file diff --git a/helm/settings_local.py b/helm/settings_local.py new file mode 100644 index 0000000000..3434b54205 --- /dev/null +++ b/helm/settings_local.py @@ -0,0 +1,121 @@ +# Copyright The IETF Trust 2007-2024, All Rights Reserved +# -*- coding: utf-8 -*- + +from ietf.settings import * # pyflakes:ignore + +ALLOWED_HOSTS = ['*'] + +DATABASES = { + "default": { + "HOST": os.environ.get("DBHOST", "db"), + "PORT": os.environ.get("DBPORT", "5432"), + "NAME": os.environ.get("DBNAME", "datatracker"), + "ENGINE": "django.db.backends.postgresql", + "USER": os.environ.get("DBUSER", "django"), + "PASSWORD": os.environ.get("DBPASS", ""), + }, +} + +SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY") + +CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL") + +IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" +IDSUBMIT_REPOSITORY_PATH = "/test/id/" +IDSUBMIT_STAGING_PATH = "/test/staging/" + +AGENDA_PATH = "/assets/www6s/proceedings/" +MEETINGHOST_LOGO_PATH = AGENDA_PATH + +USING_DEBUG_EMAIL_SERVER=True +EMAIL_HOST= "localhost" +EMAIL_PORT=2025 + +MEDIA_BASE_DIR = "/assets" +MEDIA_ROOT = MEDIA_BASE_DIR + "/media/" +MEDIA_URL = "/media/" + +PHOTOS_DIRNAME = "photo" +PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME + +SUBMIT_YANG_CATALOG_MODEL_DIR = "/assets/ietf-ftp/yang/catalogmod/" +SUBMIT_YANG_DRAFT_MODEL_DIR = "/assets/ietf-ftp/yang/draftmod/" +SUBMIT_YANG_INVAL_MODEL_DIR = "/assets/ietf-ftp/yang/invalmod/" +SUBMIT_YANG_IANA_MODEL_DIR = "/assets/ietf-ftp/yang/ianamod/" +SUBMIT_YANG_RFC_MODEL_DIR = "/assets/ietf-ftp/yang/rfcmod/" + +# Set INTERNAL_IPS for use within Docker. See https://knasmueller.net/fix-djangos-debug-toolbar-not-showing-inside-docker +import socket +hostname, _, ips = socket.gethostbyname_ex(socket.gethostname()) +INTERNAL_IPS = [".".join(ip.split(".")[:-1] + ["1"]) for ip in ips] + +# DEV_TEMPLATE_CONTEXT_PROCESSORS = [ +# 'ietf.context_processors.sql_debug', +# ] + +DOCUMENT_PATH_PATTERN = "/assets/ietfdata/doc/{doc.type_id}/" +INTERNET_DRAFT_PATH = "/assets/ietf-ftp/internet-drafts/" +RFC_PATH = "/assets/ietf-ftp/rfc/" +CHARTER_PATH = "/assets/ietf-ftp/charter/" +BOFREQ_PATH = "/assets/ietf-ftp/bofreq/" +CONFLICT_REVIEW_PATH = "/assets/ietf-ftp/conflict-reviews/" +STATUS_CHANGE_PATH = "/assets/ietf-ftp/status-changes/" +INTERNET_DRAFT_ARCHIVE_DIR = "/assets/archive/id" +INTERNET_ALL_DRAFTS_ARCHIVE_DIR = "/assets/archive/id" +BIBXML_BASE_PATH = "/assets/ietfdata/derived/bibxml" +IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH + +NOMCOM_PUBLIC_KEYS_DIR = "data/nomcom_keys/public_keys/" +SLIDE_STAGING_PATH = "/test/staging/" + +# todo check that de-gfm is in place +DE_GFM_BINARY = "/usr/local/bin/de-gfm" + +# OIDC configuration +SITE_URL = os.environ.get("OIDC_SITE_URL") + +# todo: parameterize memcached url in settings.py +MEMCACHED_HOST = os.environ.get("DTH_DATATRACKER_MEMCACHED_SERVICE_HOST", "127.0.0.1") +MEMCACHED_PORT = os.environ.get("DTH_DATATRACKER_MEMCACHED_SERVICE_PORT", "11211") +from ietf import __version__ +CACHES = { + "default": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + "VERSION": __version__, + "KEY_PREFIX": "ietf:dt", + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "sessions": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt", + }, + "htmlized": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/htmlized", + "OPTIONS": { + "MAX_ENTRIES": 100000, # 100,000 + }, + }, + "pdfized": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/pdfized", + "OPTIONS": { + "MAX_ENTRIES": 100000, # 100,000 + }, + }, + "slowpages": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/slowpages", + "OPTIONS": { + "MAX_ENTRIES": 5000, + }, + }, +} + +# Normally only set for debug, but needed until we have a real FS +DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 0f6d455856..4da0ff5df8 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -176,4 +176,4 @@ Create the name of the service account to use {{- else -}} {{ default "default" .Values.serviceAccounts.memcached.name }} {{- end -}} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml new file mode 100644 index 0000000000..904efe8622 --- /dev/null +++ b/helm/templates/configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: django-configmap +data: + settings_local.py: |- + {{- .Files.Get "settings_local.py" | indent 4 }} diff --git a/helm/templates/deployments/beat.yaml b/helm/templates/deployments/beat.yaml index 41f76b7f4f..bfb3e99b14 100644 --- a/helm/templates/deployments/beat.yaml +++ b/helm/templates/deployments/beat.yaml @@ -1,67 +1,72 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.fullname" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} - env: - CONTAINER_ROLE: beat - {{- if .Values.env }} - {{- toYaml .Values.env | nindent 12 }} - {{- end }} - {{- with .Values.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml .Values.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml .Values.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml .Values.startupProbe | nindent 12 }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.beat.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + {{- $podValues := .Values.beat }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with $podValues.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName.beat" . }} + securityContext: + {{- toYaml $podValues.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} + env: + - name: "CONTAINER_ROLE" + value: "beat" + {{- if .Values.env }} + {{- range $key, $val := .Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} + {{- end }} + {{- end }} + {{- with $podValues.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml $podValues.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml $podValues.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml $podValues.startupProbe | nindent 12 }} + resources: + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml index f346783072..ab12ce66e7 100644 --- a/helm/templates/deployments/celery.yaml +++ b/helm/templates/deployments/celery.yaml @@ -1,12 +1,13 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "datatracker.fullname" . }} + name: {{ include "datatracker.celery.fullname" . }} labels: {{- include "datatracker.labels" . | nindent 4 }} spec: - replicas: {{ .Values.replicaCount }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- $podValues := .Values.celery }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} @@ -15,25 +16,29 @@ spec: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} spec: - {{- with .Values.imagePullSecrets }} + {{- with $podValues.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + serviceAccountName: {{ include "datatracker.serviceAccountName.celery" . }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml $podValues.podSecurityContext | nindent 8 }} containers: - name: {{ .Chart.Name }} securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} env: - CONTAINER_ROLE: celery + - name: "CONTAINER_ROLE" + value: "celery" {{- if .Values.env }} - {{- toYaml .Values.env | nindent 12 }} + {{- range $key, $val := .Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} {{- end }} - {{- with .Values.volumeMounts }} + {{- end }} + {{- with $podValues.volumeMounts }} volumeMounts: {{- toYaml . | nindent 12 }} {{- end }} @@ -42,26 +47,26 @@ spec: containerPort: 8000 protocol: TCP livenessProbe: - {{- toYaml .Values.livenessProbe | nindent 12 }} + {{- toYaml $podValues.livenessProbe | nindent 12 }} readinessProbe: - {{- toYaml .Values.readinessProbe | nindent 12 }} + {{- toYaml $podValues.readinessProbe | nindent 12 }} startupProbe: - {{- toYaml .Values.startupProbe | nindent 12 }} + {{- toYaml $podValues.startupProbe | nindent 12 }} resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with $podValues.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with $podValues.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.volumes }} + {{- with $podValues.volumes }} volumes: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml index 1f403787e2..1ee4a7b6d0 100644 --- a/helm/templates/deployments/datatracker.yaml +++ b/helm/templates/deployments/datatracker.yaml @@ -1,12 +1,13 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "datatracker.fullname" . }} + name: {{ include "datatracker.datatracker.fullname" . }} labels: {{- include "datatracker.labels" . | nindent 4 }} spec: - replicas: {{ .Values.replicaCount }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- $podValues := .Values.datatracker }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} @@ -15,25 +16,29 @@ spec: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} spec: - {{- with .Values.imagePullSecrets }} + {{- with $podValues.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + serviceAccountName: {{ include "datatracker.serviceAccountName.datatracker" . }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml $podValues.podSecurityContext | nindent 8 }} containers: - name: {{ .Chart.Name }} securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion $podValues.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} env: - CONTAINER_ROLE: datatracker - {{- if .Values.env }} - {{- toYaml .Values.env | nindent 12 }} + - name: "CONTAINER_ROLE" + value: "datatracker" + {{- if $.Values.env }} + {{- range $key, $val := $.Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} {{- end }} - {{- with .Values.volumeMounts }} + {{- end }} + {{- with $podValues.volumeMounts }} volumeMounts: {{- toYaml . | nindent 12 }} {{- end }} @@ -42,26 +47,26 @@ spec: containerPort: 8000 protocol: TCP livenessProbe: - {{- toYaml .Values.livenessProbe | nindent 12 }} + {{- toYaml $podValues.livenessProbe | nindent 12 }} readinessProbe: - {{- toYaml .Values.readinessProbe | nindent 12 }} + {{- toYaml $podValues.readinessProbe | nindent 12 }} startupProbe: - {{- toYaml .Values.startupProbe | nindent 12 }} + {{- toYaml $podValues.startupProbe | nindent 12 }} resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with $podValues.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with $podValues.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.volumes }} + {{- with $podValues.volumes }} volumes: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/helm/templates/deployments/memcached.yaml b/helm/templates/deployments/memcached.yaml index 41f76b7f4f..02d4401832 100644 --- a/helm/templates/deployments/memcached.yaml +++ b/helm/templates/deployments/memcached.yaml @@ -1,12 +1,13 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "datatracker.fullname" . }} + name: {{ include "datatracker.memcached.fullname" . }} labels: {{- include "datatracker.labels" . | nindent 4 }} spec: - replicas: {{ .Values.replicaCount }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- $podValues := .Values.memcached }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} @@ -15,25 +16,27 @@ spec: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} spec: - {{- with .Values.imagePullSecrets }} + {{- with $podValues.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + serviceAccountName: {{ include "datatracker.serviceAccountName.memcached" . }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml $podValues.podSecurityContext | nindent 8 }} containers: - name: {{ .Chart.Name }} securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} env: - CONTAINER_ROLE: beat {{- if .Values.env }} - {{- toYaml .Values.env | nindent 12 }} + {{- range $key, $val := .Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} {{- end }} - {{- with .Values.volumeMounts }} + {{- end }} + {{- with $podValues.volumeMounts }} volumeMounts: {{- toYaml . | nindent 12 }} {{- end }} @@ -42,26 +45,26 @@ spec: containerPort: 8000 protocol: TCP livenessProbe: - {{- toYaml .Values.livenessProbe | nindent 12 }} + {{- toYaml $podValues.livenessProbe | nindent 12 }} readinessProbe: - {{- toYaml .Values.readinessProbe | nindent 12 }} + {{- toYaml $podValues.readinessProbe | nindent 12 }} startupProbe: - {{- toYaml .Values.startupProbe | nindent 12 }} + {{- toYaml $podValues.startupProbe | nindent 12 }} resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with $podValues.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with $podValues.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.volumes }} + {{- with $podValues.volumes }} volumes: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/helm/templates/deployments/rabbitmq.yaml b/helm/templates/deployments/rabbitmq.yaml index 41f76b7f4f..bf72d5b4b9 100644 --- a/helm/templates/deployments/rabbitmq.yaml +++ b/helm/templates/deployments/rabbitmq.yaml @@ -1,12 +1,13 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "datatracker.fullname" . }} + name: {{ include "datatracker.rabbitmq.fullname" . }} labels: {{- include "datatracker.labels" . | nindent 4 }} spec: - replicas: {{ .Values.replicaCount }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- $podValues := .Values.rabbitmq }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} @@ -15,25 +16,27 @@ spec: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} spec: - {{- with .Values.imagePullSecrets }} + {{- with $podValues.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName" . }} + serviceAccountName: {{ include "datatracker.serviceAccountName.rabbitmq" . }} securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- toYaml $podValues.podSecurityContext | nindent 8 }} containers: - name: {{ .Chart.Name }} securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.pullPolicy }} env: - CONTAINER_ROLE: beat {{- if .Values.env }} - {{- toYaml .Values.env | nindent 12 }} + {{- range $key, $val := .Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} {{- end }} - {{- with .Values.volumeMounts }} + {{- end }} + {{- with $podValues.volumeMounts }} volumeMounts: {{- toYaml . | nindent 12 }} {{- end }} @@ -42,26 +45,26 @@ spec: containerPort: 8000 protocol: TCP livenessProbe: - {{- toYaml .Values.livenessProbe | nindent 12 }} + {{- toYaml $podValues.livenessProbe | nindent 12 }} readinessProbe: - {{- toYaml .Values.readinessProbe | nindent 12 }} + {{- toYaml $podValues.readinessProbe | nindent 12 }} startupProbe: - {{- toYaml .Values.startupProbe | nindent 12 }} + {{- toYaml $podValues.startupProbe | nindent 12 }} resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.affinity }} + {{- with $podValues.affinity }} affinity: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.tolerations }} + {{- with $podValues.tolerations }} tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.volumes }} + {{- with $podValues.volumes }} volumes: {{- toYaml . | nindent 8 }} {{- end }} diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index 68034601a5..be9207658c 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -1,6 +1,6 @@ {{- if .Values.datatracker.ingress.enabled -}} {{- $fullName := include "datatracker.fullname" . -}} -{{- $svcPort := .Values.service.port -}} +{{- $svcPort := .Values.datatracker.service.port -}} {{- if and .Values.datatracker.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} {{- if not (hasKey .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class") }} {{- $_ := set .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class" .Values.datatracker.ingress.className}} @@ -58,4 +58,4 @@ spec: {{- end }} {{- end }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/templates/rbac/beat-serviceaccount.yaml b/helm/templates/rbac/beat-serviceaccount.yaml index f9fa5830c3..85331d76dd 100644 --- a/helm/templates/rbac/beat-serviceaccount.yaml +++ b/helm/templates/rbac/beat-serviceaccount.yaml @@ -4,9 +4,9 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.beat" . }} labels: - {{- include "datatracker.beat.labels" . | nindent 4 }} + {{- include "datatracker.labels" . | nindent 4 }} {{- with .Values.serviceAccounts.beat.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm/templates/rbac/celery-serviceaccount.yaml b/helm/templates/rbac/celery-serviceaccount.yaml index 267d1dce3e..8f8876f08d 100644 --- a/helm/templates/rbac/celery-serviceaccount.yaml +++ b/helm/templates/rbac/celery-serviceaccount.yaml @@ -4,9 +4,9 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.celery" . }} labels: - {{- include "datatracker.celery.labels" . | nindent 4 }} + {{- include "datatracker.labels" . | nindent 4 }} {{- with .Values.serviceAccounts.celery.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm/templates/rbac/datatracker-serviceaccount.yaml b/helm/templates/rbac/datatracker-serviceaccount.yaml index 8f37799ba7..67b38cf0f2 100644 --- a/helm/templates/rbac/datatracker-serviceaccount.yaml +++ b/helm/templates/rbac/datatracker-serviceaccount.yaml @@ -4,9 +4,9 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.datatracker" . }} labels: - {{- include "datatracker.datatracker.labels" . | nindent 4 }} + {{- include "datatracker.labels" . | nindent 4 }} {{- with .Values.serviceAccounts.datatracker.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm/templates/rbac/memcached-serviceaccount.yaml b/helm/templates/rbac/memcached-serviceaccount.yaml index ba150a4065..6674dccff7 100644 --- a/helm/templates/rbac/memcached-serviceaccount.yaml +++ b/helm/templates/rbac/memcached-serviceaccount.yaml @@ -4,9 +4,9 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.memcached" . }} labels: - {{- include "datatracker.memcached.labels" . | nindent 4 }} + {{- include "datatracker.labels" . | nindent 4 }} {{- with .Values.serviceAccounts.memcached.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm/templates/rbac/rabbitmq-serviceaccount.yaml b/helm/templates/rbac/rabbitmq-serviceaccount.yaml index c9ce127482..e0b81d11f7 100644 --- a/helm/templates/rbac/rabbitmq-serviceaccount.yaml +++ b/helm/templates/rbac/rabbitmq-serviceaccount.yaml @@ -4,9 +4,9 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.rabbitmq" . }} labels: - {{- include "datatracker.rabbitmq.labels" . | nindent 4 }} + {{- include "datatracker.labels" . | nindent 4 }} {{- with .Values.serviceAccounts.rabbitmq.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/helm/values.yaml b/helm/values.yaml index 31f5e25cf8..2c549382ba 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -7,23 +7,23 @@ serviceAccounts: datatracker: create: true - name: + name: datatracker annotations: {} celery: create: true - name: + name: celery annotations: {} beat: create: true - name: + name: beat annotations: {} rabbitmq: create: true - name: + name: rabbitmq annotations: {} memcached: create: true - name: + name: memcached annotations: {} # ------------------------------------------------------------- @@ -31,6 +31,7 @@ serviceAccounts: # ------------------------------------------------------------- datatracker: + name: datatracker image: repository: "ghcr.io/ietf-tools/datatracker" pullPolicy: IfNotPresent @@ -59,7 +60,8 @@ datatracker: livenessProbe: httpGet: - path: /health + # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint + path: /submit/tool-instructions/ port: http podAnnotations: {} @@ -68,10 +70,11 @@ datatracker: podSecurityContext: {} # fsGroup: 2000 - readinessProbe: - httpGet: - path: /health - port: http +#readinessProbe: +# httpGet: +# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint +# path: /submit/tool-instructions/ +# port: http replicaCount: 1 @@ -117,18 +120,36 @@ datatracker: successThreshold: 1 failureThreshold: 60 httpGet: - path: /health + # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint + path: /submit/tool-instructions/ port: http # Additional volumes on the output Deployment definition. - volumes: [] + volumes: + - name: settings-local-volume + configMap: + name: django-configmap + - name: cache-volume + emptyDir: + sizeLimit: 1Gi + - name: staging-volume + emptyDir: + sizeLimit: 1Gi # - name: foo # secret: # secretName: mysecret # optional: false # Additional volumeMounts on the output Deployment definition. - volumeMounts: [] + volumeMounts: + - name: settings-local-volume + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + readOnly: true + - name: cache-volume + mountPath: "/a/cache" + - name: staging-volume + mountPath: "/test/staging" # - name: foo # mountPath: "/etc/foo" # readOnly: true @@ -144,6 +165,7 @@ datatracker: # ------------------------------------------------------------- celery: + name: celery image: repository: "ghcr.io/ietf-tools/datatracker" pullPolicy: IfNotPresent @@ -156,7 +178,10 @@ celery: livenessProbe: exec: - command: celery inspect ping + command: + - celery + - inspect + - ping periodSeconds: 30 timeoutSeconds: 5 @@ -210,17 +235,37 @@ celery: successThreshold: 1 failureThreshold: 60 exec: - command: celery inspect ping + command: + - celery + - inspect + - ping # Additional volumes on the output Deployment definition. - volumes: [] + volumes: + - name: settings-local-volume + configMap: + name: django-configmap + - name: cache-volume + emptyDir: + sizeLimit: 1Gi + - name: staging-volume + emptyDir: + sizeLimit: 1Gi # - name: foo # secret: # secretName: mysecret # optional: false # Additional volumeMounts on the output Deployment definition. - volumeMounts: [] + volumeMounts: + - name: settings-local-volume + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + readOnly: true + - name: cache-volume + mountPath: "/a/cache" + - name: staging-volume + mountPath: "/test/staging" # - name: foo # mountPath: "/etc/foo" # readOnly: true @@ -236,6 +281,7 @@ celery: # ------------------------------------------------------------- beat: + name: beat image: repository: "ghcr.io/ietf-tools/datatracker" pullPolicy: IfNotPresent @@ -248,7 +294,10 @@ beat: livenessProbe: exec: - command: celery inspect ping + command: + - celery + - inspect + - ping periodSeconds: 30 timeoutSeconds: 5 @@ -302,17 +351,37 @@ beat: successThreshold: 1 failureThreshold: 60 exec: - command: celery inspect ping + command: + - celery + - inspect + - ping # Additional volumes on the output Deployment definition. - volumes: [] + volumes: + - name: settings-local-volume + configMap: + name: django-configmap + - name: cache-volume + emptyDir: + sizeLimit: 1Gi + - name: staging-volume + emptyDir: + sizeLimit: 1Gi # - name: foo # secret: # secretName: mysecret # optional: false # Additional volumeMounts on the output Deployment definition. - volumeMounts: [] + volumeMounts: + - name: settings-local-volume + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + readOnly: true + - name: cache-volume + mountPath: "/a/cache" + - name: staging-volume + mountPath: "/test/staging" # - name: foo # mountPath: "/etc/foo" # readOnly: true @@ -328,6 +397,7 @@ beat: # ------------------------------------------------------------- rabbitmq: + name: rabbitmq image: repository: "ghcr.io/ietf-tools/datatracker-mq" pullPolicy: IfNotPresent @@ -339,7 +409,10 @@ rabbitmq: livenessProbe: exec: - command: rabbitmq-diagnostics -q ping + command: + - rabbitmq-diagnostics + - -q + - ping periodSeconds: 30 timeoutSeconds: 5 @@ -393,20 +466,23 @@ rabbitmq: successThreshold: 1 failureThreshold: 60 exec: - command: rabbitmq-diagnostics -q ping + command: + - rabbitmq-diagnostics + - -q + - ping # Additional volumes on the output Deployment definition. volumes: [] - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. volumeMounts: [] - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true tolerations: [] @@ -419,10 +495,11 @@ rabbitmq: # ------------------------------------------------------------- memcached: + name: memcached image: repository: "memcached" pullPolicy: IfNotPresent - tag: "latest" + tag: "1.6-alpine" imagePullSecrets: [] nameOverride: "" @@ -458,7 +535,7 @@ memcached: service: type: ClusterIP - port: 80 + port: 11211 serviceAccount: # Specifies whether a service account should be created @@ -486,4 +563,13 @@ autoscaling: minReplicas: 1 maxReplicas: 100 targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 \ No newline at end of file + # targetMemoryUtilizationPercentage: 80 + +env: + # n.b., these are debug values / non-secret secrets + # DBHOST: "host.minikube.internal" + # DBPORT: "5432" + DBNAME: "datatracker" + DBUSER: "django" + DBPASS: "RkTkDPFnKpko" + DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" From dca77db2507d1d93f32dad420485807d68d42ebd Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 12 Mar 2024 18:55:27 -0300 Subject: [PATCH 07/70] ci: package and push helm chart --- .github/workflows/build.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd65da036c..3ac2441a84 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -248,6 +248,17 @@ jobs: push: true tags: ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }} + - name: Package and Push Chart + env: + VERSION: 2.0.${{ github.run_number }} + run: | + echo "Setting version ${{ env.PKG_VERSION }}..." + sed -i -r -e "s|^version: .*$|version: '${{ env.PKG_VERSION }}'|" -e "s|^appVersion: .*$|appVersion: '${{ env.PKG_VERSION }}'|" helm/Chart.yaml + helm plugin install https://github.com/chartmuseum/helm-push.git + helm repo add chartmuseum https://charts.ietf.org + helm cm-push --version="${{ env.PKG_VERSION }}" --username="${{ secrets.HELM_REPO_USERNAME }}" --password="${{ secrets.HELM_REPO_PASSWORD }}" helm/ chartmuseum + helm repo remove chartmuseum + - name: Update CHANGELOG id: changelog uses: Requarks/changelog-action@v1 From cd225018f9fdbe8427f2edc1b873c0c5c174e2d0 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Sat, 16 Mar 2024 01:18:23 +1000 Subject: [PATCH 08/70] ci: Memcached service (#7185) * ci: service.yaml -> services/datatracker.yaml * ci: memcached service * ci: simplify memcached service name * ci: Use correct service.type for memcached --- helm/settings_local.py | 4 +-- helm/templates/deployments/memcached.yaml | 4 +-- .../datatracker.yaml} | 36 +++++++++---------- helm/templates/services/memcached.yaml | 19 ++++++++++ 4 files changed, 41 insertions(+), 22 deletions(-) rename helm/templates/{service.yaml => services/datatracker.yaml} (96%) create mode 100644 helm/templates/services/memcached.yaml diff --git a/helm/settings_local.py b/helm/settings_local.py index 3434b54205..993c8c3276 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -75,8 +75,8 @@ SITE_URL = os.environ.get("OIDC_SITE_URL") # todo: parameterize memcached url in settings.py -MEMCACHED_HOST = os.environ.get("DTH_DATATRACKER_MEMCACHED_SERVICE_HOST", "127.0.0.1") -MEMCACHED_PORT = os.environ.get("DTH_DATATRACKER_MEMCACHED_SERVICE_PORT", "11211") +MEMCACHED_HOST = os.environ.get(f"MEMCACHED_SERVICE_HOST", "127.0.0.1") +MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") from ietf import __version__ CACHES = { "default": { diff --git a/helm/templates/deployments/memcached.yaml b/helm/templates/deployments/memcached.yaml index 02d4401832..36ef484666 100644 --- a/helm/templates/deployments/memcached.yaml +++ b/helm/templates/deployments/memcached.yaml @@ -41,8 +41,8 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} ports: - - name: http - containerPort: 8000 + - name: memcached + containerPort: 11211 protocol: TCP livenessProbe: {{- toYaml $podValues.livenessProbe | nindent 12 }} diff --git a/helm/templates/service.yaml b/helm/templates/services/datatracker.yaml similarity index 96% rename from helm/templates/service.yaml rename to helm/templates/services/datatracker.yaml index e7b310f98f..2fbea5826f 100644 --- a/helm/templates/service.yaml +++ b/helm/templates/services/datatracker.yaml @@ -1,19 +1,19 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{include "datatracker.fullname" .}} - labels: {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.datatracker.service.annotations }} - annotations: - {{- range $key, $value := . }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} -spec: - type: {{.Values.datatracker.service.type}} - ports: - - port: {{ default "80" .Values.datatracker.service.port}} - targetPort: http - protocol: TCP - name: http +apiVersion: v1 +kind: Service +metadata: + name: {{include "datatracker.fullname" .}} + labels: {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.datatracker.service.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} +spec: + type: {{.Values.datatracker.service.type}} + ports: + - port: {{ default "80" .Values.datatracker.service.port}} + targetPort: http + protocol: TCP + name: http selector: {{- include "datatracker.selectorLabels" . | nindent 4}} \ No newline at end of file diff --git a/helm/templates/services/memcached.yaml b/helm/templates/services/memcached.yaml new file mode 100644 index 0000000000..1337571b04 --- /dev/null +++ b/helm/templates/services/memcached.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: memcached + labels: {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.memcached.service.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} +spec: + type: {{.Values.memcached.service.type}} + ports: + - port: {{ default "11211" .Values.memcached.service.port}} + targetPort: memcached + protocol: TCP + name: memcached + selector: {{- include "datatracker.selectorLabels" . | nindent 4}} From f91ea458138b96a85a27a97e96c8af7ed70345e6 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Sat, 16 Mar 2024 21:27:16 -0500 Subject: [PATCH 09/70] chore: CRLF -> LF (#7214) --- helm/.helmignore | 44 +- helm/Chart.yaml | 44 +- helm/templates/_helpers.tpl | 358 ++--- helm/templates/deployments/celery.yaml | 144 +-- helm/templates/deployments/datatracker.yaml | 144 +-- helm/templates/deployments/memcached.yaml | 140 +- helm/templates/deployments/rabbitmq.yaml | 140 +- helm/templates/hpa.yaml | 62 +- helm/templates/ingress.yaml | 122 +- helm/templates/rbac/beat-serviceaccount.yaml | 24 +- .../templates/rbac/celery-serviceaccount.yaml | 24 +- .../rbac/datatracker-serviceaccount.yaml | 24 +- .../rbac/memcached-serviceaccount.yaml | 24 +- .../rbac/rabbitmq-serviceaccount.yaml | 24 +- helm/values.yaml | 1150 ++++++++--------- 15 files changed, 1234 insertions(+), 1234 deletions(-) diff --git a/helm/.helmignore b/helm/.helmignore index 2252590f2e..691fa13d6a 100644 --- a/helm/.helmignore +++ b/helm/.helmignore @@ -1,23 +1,23 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj .vscode/ \ No newline at end of file diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 1c19834c06..844087fa69 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -1,23 +1,23 @@ -apiVersion: v2 -name: datatracker -description: The day-to-day front-end to the IETF database for people who work on IETF standards. -home: https://datatracker.ietf.org -sources: - - https://github.com/ietf-tools/datatracker -maintainers: - - name: IETF Tools Team - email: tools-discuss@ietf.org - url: https://github.com/ietf-tools - -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. +apiVersion: v2 +name: datatracker +description: The day-to-day front-end to the IETF database for people who work on IETF standards. +home: https://datatracker.ietf.org +sources: + - https://github.com/ietf-tools/datatracker +maintainers: + - name: IETF Tools Team + email: tools-discuss@ietf.org + url: https://github.com/ietf-tools + +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 1.0.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. appVersion: "1.0.0" \ No newline at end of file diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 4da0ff5df8..268f0e75dc 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -1,179 +1,179 @@ -{{/* - Expand the name of the chart. - */}} -{{- define "datatracker.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "datatracker.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create a fully qualified datatracker name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.datatracker.fullname" -}} -{{- if .Values.datatracker.fullnameOverride -}} -{{- .Values.datatracker.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified celery name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.celery.fullname" -}} -{{- if .Values.celery.fullnameOverride -}} -{{- .Values.celery.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.celery.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.celery.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified celery name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.beat.fullname" -}} -{{- if .Values.beat.fullnameOverride -}} -{{- .Values.beat.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.beat.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.beat.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified rabbitmq name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.rabbitmq.fullname" -}} -{{- if .Values.rabbitmq.fullnameOverride -}} -{{- .Values.rabbitmq.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified memcached name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.memcached.fullname" -}} -{{- if .Values.memcached.fullnameOverride -}} -{{- .Values.memcached.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "datatracker.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "datatracker.labels" -}} -helm.sh/chart: {{ include "datatracker.chart" . }} -{{ include "datatracker.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "datatracker.selectorLabels" -}} -app.kubernetes.io/name: {{ include "datatracker.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "datatracker.serviceAccountName.datatracker" -}} -{{- if .Values.serviceAccounts.datatracker.create -}} - {{ default (include "datatracker.datatracker.fullname" .) .Values.serviceAccounts.datatracker.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.datatracker.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.celery" -}} -{{- if .Values.serviceAccounts.celery.create -}} - {{ default (include "datatracker.celery.fullname" .) .Values.serviceAccounts.celery.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.celery.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.beat" -}} -{{- if .Values.serviceAccounts.beat.create -}} - {{ default (include "datatracker.beat.fullname" .) .Values.serviceAccounts.beat.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.beat.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.rabbitmq" -}} -{{- if .Values.serviceAccounts.rabbitmq.create -}} - {{ default (include "datatracker.rabbitmq.fullname" .) .Values.serviceAccounts.rabbitmq.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.rabbitmq.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.memcached" -}} -{{- if .Values.serviceAccounts.memcached.create -}} - {{ default (include "datatracker.memcached.fullname" .) .Values.serviceAccounts.memcached.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.memcached.name }} -{{- end -}} -{{- end }} +{{/* + Expand the name of the chart. + */}} +{{- define "datatracker.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "datatracker.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create a fully qualified datatracker name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.datatracker.fullname" -}} +{{- if .Values.datatracker.fullnameOverride -}} +{{- .Values.datatracker.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified celery name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.celery.fullname" -}} +{{- if .Values.celery.fullnameOverride -}} +{{- .Values.celery.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.celery.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.celery.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified celery name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.beat.fullname" -}} +{{- if .Values.beat.fullnameOverride -}} +{{- .Values.beat.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.beat.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.beat.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified rabbitmq name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.rabbitmq.fullname" -}} +{{- if .Values.rabbitmq.fullnameOverride -}} +{{- .Values.rabbitmq.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a fully qualified memcached name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "datatracker.memcached.fullname" -}} +{{- if .Values.memcached.fullnameOverride -}} +{{- .Values.memcached.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "datatracker.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "datatracker.labels" -}} +helm.sh/chart: {{ include "datatracker.chart" . }} +{{ include "datatracker.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "datatracker.selectorLabels" -}} +app.kubernetes.io/name: {{ include "datatracker.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "datatracker.serviceAccountName.datatracker" -}} +{{- if .Values.serviceAccounts.datatracker.create -}} + {{ default (include "datatracker.datatracker.fullname" .) .Values.serviceAccounts.datatracker.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.datatracker.name }} +{{- end -}} +{{- end }} + +{{- define "datatracker.serviceAccountName.celery" -}} +{{- if .Values.serviceAccounts.celery.create -}} + {{ default (include "datatracker.celery.fullname" .) .Values.serviceAccounts.celery.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.celery.name }} +{{- end -}} +{{- end }} + +{{- define "datatracker.serviceAccountName.beat" -}} +{{- if .Values.serviceAccounts.beat.create -}} + {{ default (include "datatracker.beat.fullname" .) .Values.serviceAccounts.beat.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.beat.name }} +{{- end -}} +{{- end }} + +{{- define "datatracker.serviceAccountName.rabbitmq" -}} +{{- if .Values.serviceAccounts.rabbitmq.create -}} + {{ default (include "datatracker.rabbitmq.fullname" .) .Values.serviceAccounts.rabbitmq.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.rabbitmq.name }} +{{- end -}} +{{- end }} + +{{- define "datatracker.serviceAccountName.memcached" -}} +{{- if .Values.serviceAccounts.memcached.create -}} + {{ default (include "datatracker.memcached.fullname" .) .Values.serviceAccounts.memcached.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.memcached.name }} +{{- end -}} +{{- end }} diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml index ab12ce66e7..8dc1643fff 100644 --- a/helm/templates/deployments/celery.yaml +++ b/helm/templates/deployments/celery.yaml @@ -1,72 +1,72 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.celery.fullname" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} -spec: - {{- $podValues := .Values.celery }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.celery" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} - env: - - name: "CONTAINER_ROLE" - value: "celery" - {{- if .Values.env }} - {{- range $key, $val := .Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.celery.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + {{- $podValues := .Values.celery }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with $podValues.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName.celery" . }} + securityContext: + {{- toYaml $podValues.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} + env: + - name: "CONTAINER_ROLE" + value: "celery" + {{- if .Values.env }} + {{- range $key, $val := .Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} + {{- end }} + {{- end }} + {{- with $podValues.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml $podValues.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml $podValues.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml $podValues.startupProbe | nindent 12 }} + resources: + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml index 1ee4a7b6d0..dd4d8e4d82 100644 --- a/helm/templates/deployments/datatracker.yaml +++ b/helm/templates/deployments/datatracker.yaml @@ -1,72 +1,72 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.datatracker.fullname" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} -spec: - {{- $podValues := .Values.datatracker }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.datatracker" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion $podValues.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} - env: - - name: "CONTAINER_ROLE" - value: "datatracker" - {{- if $.Values.env }} - {{- range $key, $val := $.Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.datatracker.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + {{- $podValues := .Values.datatracker }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with $podValues.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName.datatracker" . }} + securityContext: + {{- toYaml $podValues.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion $podValues.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} + env: + - name: "CONTAINER_ROLE" + value: "datatracker" + {{- if $.Values.env }} + {{- range $key, $val := $.Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} + {{- end }} + {{- end }} + {{- with $podValues.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml $podValues.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml $podValues.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml $podValues.startupProbe | nindent 12 }} + resources: + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/deployments/memcached.yaml b/helm/templates/deployments/memcached.yaml index 36ef484666..cbc7b3c35a 100644 --- a/helm/templates/deployments/memcached.yaml +++ b/helm/templates/deployments/memcached.yaml @@ -1,70 +1,70 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.memcached.fullname" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} -spec: - {{- $podValues := .Values.memcached }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.memcached" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} - env: - {{- if .Values.env }} - {{- range $key, $val := .Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: memcached - containerPort: 11211 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.memcached.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + {{- $podValues := .Values.memcached }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with $podValues.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName.memcached" . }} + securityContext: + {{- toYaml $podValues.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} + env: + {{- if .Values.env }} + {{- range $key, $val := .Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} + {{- end }} + {{- end }} + {{- with $podValues.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: memcached + containerPort: 11211 + protocol: TCP + livenessProbe: + {{- toYaml $podValues.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml $podValues.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml $podValues.startupProbe | nindent 12 }} + resources: + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/deployments/rabbitmq.yaml b/helm/templates/deployments/rabbitmq.yaml index bf72d5b4b9..a44a84a252 100644 --- a/helm/templates/deployments/rabbitmq.yaml +++ b/helm/templates/deployments/rabbitmq.yaml @@ -1,70 +1,70 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.rabbitmq.fullname" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} -spec: - {{- $podValues := .Values.rabbitmq }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.rabbitmq" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.pullPolicy }} - env: - {{- if .Values.env }} - {{- range $key, $val := .Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "datatracker.rabbitmq.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + {{- $podValues := .Values.rabbitmq }} + replicas: {{ $podValues.replicaCount }} + revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "datatracker.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "datatracker.selectorLabels" . | nindent 8 }} + spec: + {{- with $podValues.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "datatracker.serviceAccountName.rabbitmq" . }} + securityContext: + {{- toYaml $podValues.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml $podValues.securityContext | nindent 12 }} + image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.image.pullPolicy }} + env: + {{- if .Values.env }} + {{- range $key, $val := .Values.env }} + - name: {{ $key | quote }} + value: {{ $val | quote }} + {{- end }} + {{- end }} + {{- with $podValues.volumeMounts }} + volumeMounts: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8000 + protocol: TCP + livenessProbe: + {{- toYaml $podValues.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml $podValues.readinessProbe | nindent 12 }} + startupProbe: + {{- toYaml $podValues.startupProbe | nindent 12 }} + resources: + {{- toYaml $podValues.resources | nindent 12 }} + {{- with $podValues.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with $podValues.volumes }} + volumes: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/templates/hpa.yaml b/helm/templates/hpa.yaml index 518f7e23ab..5dd889b509 100644 --- a/helm/templates/hpa.yaml +++ b/helm/templates/hpa.yaml @@ -1,32 +1,32 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "datatracker.fullname" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "datatracker.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "datatracker.fullname" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "datatracker.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index be9207658c..c9405ef389 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -1,61 +1,61 @@ -{{- if .Values.datatracker.ingress.enabled -}} -{{- $fullName := include "datatracker.fullname" . -}} -{{- $svcPort := .Values.datatracker.service.port -}} -{{- if and .Values.datatracker.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class" .Values.datatracker.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.datatracker.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.datatracker.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.datatracker.ingress.className }} - {{- end }} - {{- if .Values.datatracker.ingress.tls }} - tls: - {{- range .Values.datatracker.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.datatracker.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} +{{- if .Values.datatracker.ingress.enabled -}} +{{- $fullName := include "datatracker.fullname" . -}} +{{- $svcPort := .Values.datatracker.service.port -}} +{{- if and .Values.datatracker.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class" .Values.datatracker.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.datatracker.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.datatracker.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.datatracker.ingress.className }} + {{- end }} + {{- if .Values.datatracker.ingress.tls }} + tls: + {{- range .Values.datatracker.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.datatracker.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/templates/rbac/beat-serviceaccount.yaml b/helm/templates/rbac/beat-serviceaccount.yaml index 85331d76dd..bd955aed88 100644 --- a/helm/templates/rbac/beat-serviceaccount.yaml +++ b/helm/templates/rbac/beat-serviceaccount.yaml @@ -1,12 +1,12 @@ -{{- if .Values.serviceAccounts.beat.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.beat" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.serviceAccounts.beat.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} +{{- if .Values.serviceAccounts.beat.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.beat" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.beat.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/helm/templates/rbac/celery-serviceaccount.yaml b/helm/templates/rbac/celery-serviceaccount.yaml index 8f8876f08d..105b979c60 100644 --- a/helm/templates/rbac/celery-serviceaccount.yaml +++ b/helm/templates/rbac/celery-serviceaccount.yaml @@ -1,12 +1,12 @@ -{{- if .Values.serviceAccounts.celery.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.celery" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.serviceAccounts.celery.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} +{{- if .Values.serviceAccounts.celery.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.celery" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.celery.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/helm/templates/rbac/datatracker-serviceaccount.yaml b/helm/templates/rbac/datatracker-serviceaccount.yaml index 67b38cf0f2..7786443ada 100644 --- a/helm/templates/rbac/datatracker-serviceaccount.yaml +++ b/helm/templates/rbac/datatracker-serviceaccount.yaml @@ -1,12 +1,12 @@ -{{- if .Values.serviceAccounts.datatracker.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.datatracker" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.serviceAccounts.datatracker.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} +{{- if .Values.serviceAccounts.datatracker.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.datatracker" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.datatracker.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/helm/templates/rbac/memcached-serviceaccount.yaml b/helm/templates/rbac/memcached-serviceaccount.yaml index 6674dccff7..7f01615b1d 100644 --- a/helm/templates/rbac/memcached-serviceaccount.yaml +++ b/helm/templates/rbac/memcached-serviceaccount.yaml @@ -1,12 +1,12 @@ -{{- if .Values.serviceAccounts.memcached.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.memcached" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.serviceAccounts.memcached.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} +{{- if .Values.serviceAccounts.memcached.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.memcached" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.memcached.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/helm/templates/rbac/rabbitmq-serviceaccount.yaml b/helm/templates/rbac/rabbitmq-serviceaccount.yaml index e0b81d11f7..015e098da3 100644 --- a/helm/templates/rbac/rabbitmq-serviceaccount.yaml +++ b/helm/templates/rbac/rabbitmq-serviceaccount.yaml @@ -1,12 +1,12 @@ -{{- if .Values.serviceAccounts.rabbitmq.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.rabbitmq" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.serviceAccounts.rabbitmq.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} +{{- if .Values.serviceAccounts.rabbitmq.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "datatracker.serviceAccountName.rabbitmq" . }} + labels: + {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.serviceAccounts.rabbitmq.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/helm/values.yaml b/helm/values.yaml index 2c549382ba..cfdef165a2 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -1,575 +1,575 @@ -# Default values for datatracker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -## Define serviceAccount names for components. Defaults to component's fully qualified name. -## -serviceAccounts: - datatracker: - create: true - name: datatracker - annotations: {} - celery: - create: true - name: celery - annotations: {} - beat: - create: true - name: beat - annotations: {} - rabbitmq: - create: true - name: rabbitmq - annotations: {} - memcached: - create: true - name: memcached - annotations: {} - -# ------------------------------------------------------------- -# DATATRACKER -# ------------------------------------------------------------- - -datatracker: - name: datatracker - image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: datatracker.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - - livenessProbe: - httpGet: - # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint - path: /submit/tool-instructions/ - port: http - - podAnnotations: {} - podLabels: {} - - podSecurityContext: {} - # fsGroup: 2000 - -#readinessProbe: -# httpGet: -# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint -# path: /submit/tool-instructions/ -# port: http - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - service: - type: ClusterIP - port: 80 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - httpGet: - # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint - path: /submit/tool-instructions/ - port: http - - # Additional volumes on the output Deployment definition. - volumes: - - name: settings-local-volume - configMap: - name: django-configmap - - name: cache-volume - emptyDir: - sizeLimit: 1Gi - - name: staging-volume - emptyDir: - sizeLimit: 1Gi - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: settings-local-volume - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - readOnly: true - - name: cache-volume - mountPath: "/a/cache" - - name: staging-volume - mountPath: "/test/staging" - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# CELERY -# ------------------------------------------------------------- - -celery: - name: celery - image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - livenessProbe: - exec: - command: - - celery - - inspect - - ping - periodSeconds: 30 - timeoutSeconds: 5 - - podAnnotations: {} - podLabels: {} - - podSecurityContext: {} - # fsGroup: 2000 - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - service: - type: ClusterIP - port: 80 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - exec: - command: - - celery - - inspect - - ping - - # Additional volumes on the output Deployment definition. - volumes: - - name: settings-local-volume - configMap: - name: django-configmap - - name: cache-volume - emptyDir: - sizeLimit: 1Gi - - name: staging-volume - emptyDir: - sizeLimit: 1Gi - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: settings-local-volume - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - readOnly: true - - name: cache-volume - mountPath: "/a/cache" - - name: staging-volume - mountPath: "/test/staging" - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# BEAT -# ------------------------------------------------------------- - -beat: - name: beat - image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - livenessProbe: - exec: - command: - - celery - - inspect - - ping - periodSeconds: 30 - timeoutSeconds: 5 - - podAnnotations: {} - podLabels: {} - - podSecurityContext: {} - # fsGroup: 2000 - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - service: - type: ClusterIP - port: 80 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - exec: - command: - - celery - - inspect - - ping - - # Additional volumes on the output Deployment definition. - volumes: - - name: settings-local-volume - configMap: - name: django-configmap - - name: cache-volume - emptyDir: - sizeLimit: 1Gi - - name: staging-volume - emptyDir: - sizeLimit: 1Gi - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: settings-local-volume - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - readOnly: true - - name: cache-volume - mountPath: "/a/cache" - - name: staging-volume - mountPath: "/test/staging" - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# RABBITMQ -# ------------------------------------------------------------- - -rabbitmq: - name: rabbitmq - image: - repository: "ghcr.io/ietf-tools/datatracker-mq" - pullPolicy: IfNotPresent - tag: "latest" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - livenessProbe: - exec: - command: - - rabbitmq-diagnostics - - -q - - ping - periodSeconds: 30 - timeoutSeconds: 5 - - podAnnotations: {} - podLabels: {} - - podSecurityContext: {} - # fsGroup: 2000 - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - service: - type: ClusterIP - port: 80 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - exec: - command: - - rabbitmq-diagnostics - - -q - - ping - - # Additional volumes on the output Deployment definition. - volumes: [] - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: [] - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# MEMCACHED -# ------------------------------------------------------------- - -memcached: - name: memcached - image: - repository: "memcached" - pullPolicy: IfNotPresent - tag: "1.6-alpine" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - podAnnotations: {} - podLabels: {} - - podSecurityContext: {} - # fsGroup: 2000 - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - service: - type: ClusterIP - port: 11211 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# COMMON -# ------------------------------------------------------------- - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -env: - # n.b., these are debug values / non-secret secrets - # DBHOST: "host.minikube.internal" - # DBPORT: "5432" - DBNAME: "datatracker" - DBUSER: "django" - DBPASS: "RkTkDPFnKpko" - DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" +# Default values for datatracker. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +## Define serviceAccount names for components. Defaults to component's fully qualified name. +## +serviceAccounts: + datatracker: + create: true + name: datatracker + annotations: {} + celery: + create: true + name: celery + annotations: {} + beat: + create: true + name: beat + annotations: {} + rabbitmq: + create: true + name: rabbitmq + annotations: {} + memcached: + create: true + name: memcached + annotations: {} + +# ------------------------------------------------------------- +# DATATRACKER +# ------------------------------------------------------------- + +datatracker: + name: datatracker + image: + repository: "ghcr.io/ietf-tools/datatracker" + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + # tag: "v1.1.0" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: datatracker.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + + livenessProbe: + httpGet: + # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint + path: /submit/tool-instructions/ + port: http + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + +#readinessProbe: +# httpGet: +# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint +# path: /submit/tool-instructions/ +# port: http + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + httpGet: + # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint + path: /submit/tool-instructions/ + port: http + + # Additional volumes on the output Deployment definition. + volumes: + - name: settings-local-volume + configMap: + name: django-configmap + - name: cache-volume + emptyDir: + sizeLimit: 1Gi + - name: staging-volume + emptyDir: + sizeLimit: 1Gi + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: + - name: settings-local-volume + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + readOnly: true + - name: cache-volume + mountPath: "/a/cache" + - name: staging-volume + mountPath: "/test/staging" + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# CELERY +# ------------------------------------------------------------- + +celery: + name: celery + image: + repository: "ghcr.io/ietf-tools/datatracker" + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + # tag: "v1.1.0" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + livenessProbe: + exec: + command: + - celery + - inspect + - ping + periodSeconds: 30 + timeoutSeconds: 5 + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: + - celery + - inspect + - ping + + # Additional volumes on the output Deployment definition. + volumes: + - name: settings-local-volume + configMap: + name: django-configmap + - name: cache-volume + emptyDir: + sizeLimit: 1Gi + - name: staging-volume + emptyDir: + sizeLimit: 1Gi + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: + - name: settings-local-volume + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + readOnly: true + - name: cache-volume + mountPath: "/a/cache" + - name: staging-volume + mountPath: "/test/staging" + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# BEAT +# ------------------------------------------------------------- + +beat: + name: beat + image: + repository: "ghcr.io/ietf-tools/datatracker" + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + # tag: "v1.1.0" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + livenessProbe: + exec: + command: + - celery + - inspect + - ping + periodSeconds: 30 + timeoutSeconds: 5 + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: + - celery + - inspect + - ping + + # Additional volumes on the output Deployment definition. + volumes: + - name: settings-local-volume + configMap: + name: django-configmap + - name: cache-volume + emptyDir: + sizeLimit: 1Gi + - name: staging-volume + emptyDir: + sizeLimit: 1Gi + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: + - name: settings-local-volume + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + readOnly: true + - name: cache-volume + mountPath: "/a/cache" + - name: staging-volume + mountPath: "/test/staging" + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# RABBITMQ +# ------------------------------------------------------------- + +rabbitmq: + name: rabbitmq + image: + repository: "ghcr.io/ietf-tools/datatracker-mq" + pullPolicy: IfNotPresent + tag: "latest" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + livenessProbe: + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + periodSeconds: 30 + timeoutSeconds: 5 + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 80 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: + - rabbitmq-diagnostics + - -q + - ping + + # Additional volumes on the output Deployment definition. + volumes: [] + # - name: foo + # secret: + # secretName: mysecret + # optional: false + + # Additional volumeMounts on the output Deployment definition. + volumeMounts: [] + # - name: foo + # mountPath: "/etc/foo" + # readOnly: true + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# MEMCACHED +# ------------------------------------------------------------- + +memcached: + name: memcached + image: + repository: "memcached" + pullPolicy: IfNotPresent + tag: "1.6-alpine" + + imagePullSecrets: [] + nameOverride: "" + fullnameOverride: "" + + podAnnotations: {} + podLabels: {} + + podSecurityContext: {} + # fsGroup: 2000 + + replicaCount: 1 + + resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + + service: + type: ClusterIP + port: 11211 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + + tolerations: [] + + nodeSelector: {} + + affinity: {} + +# ------------------------------------------------------------- +# COMMON +# ------------------------------------------------------------- + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +env: + # n.b., these are debug values / non-secret secrets + # DBHOST: "host.minikube.internal" + # DBPORT: "5432" + DBNAME: "datatracker" + DBUSER: "django" + DBPASS: "RkTkDPFnKpko" + DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" From c6372992c609797e0f771e31b32469221df51dda Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Sun, 17 Mar 2024 12:35:16 +1000 Subject: [PATCH 10/70] ci: use datatracker image values for celery/beat (#7213) * ci: use datatracker image values for celery/beat * ci: refactor exec command lists as JSON --------- Co-authored-by: Robert Sparks --- helm/templates/deployments/beat.yaml | 4 +-- helm/templates/deployments/celery.yaml | 4 +-- helm/values.yaml | 44 ++++++++------------------ 3 files changed, 18 insertions(+), 34 deletions(-) diff --git a/helm/templates/deployments/beat.yaml b/helm/templates/deployments/beat.yaml index bfb3e99b14..6f7409fb9c 100644 --- a/helm/templates/deployments/beat.yaml +++ b/helm/templates/deployments/beat.yaml @@ -27,8 +27,8 @@ spec: - name: {{ .Chart.Name }} securityContext: {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} + image: "{{ default $.Values.datatracker.image.repository $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" + imagePullPolicy: {{ default "IfNotPresent" (default $.Values.datatracker.image.imagePullPolicy $podValues.image.imagePullPolicy) }} env: - name: "CONTAINER_ROLE" value: "beat" diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml index 8dc1643fff..f1f043abbd 100644 --- a/helm/templates/deployments/celery.yaml +++ b/helm/templates/deployments/celery.yaml @@ -27,8 +27,8 @@ spec: - name: {{ .Chart.Name }} securityContext: {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} + image: "{{ default $.Values.datatracker.image.repository $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" + imagePullPolicy: {{ default "IfNotPresent" (default $.Values.datatracker.image.imagePullPolicy $podValues.image.imagePullPolicy) }} env: - name: "CONTAINER_ROLE" value: "celery" diff --git a/helm/values.yaml b/helm/values.yaml index cfdef165a2..68b9344c02 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -166,9 +166,10 @@ datatracker: celery: name: celery - image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent + image: {} + # defaults to datatracker settings if not specified separately + #repository: "ghcr.io/ietf-tools/datatracker" + #pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. # tag: "v1.1.0" @@ -178,10 +179,7 @@ celery: livenessProbe: exec: - command: - - celery - - inspect - - ping + command: ["celery", "inspect", "ping"] periodSeconds: 30 timeoutSeconds: 5 @@ -235,10 +233,7 @@ celery: successThreshold: 1 failureThreshold: 60 exec: - command: - - celery - - inspect - - ping + command: ["celery", "inspect", "ping"] # Additional volumes on the output Deployment definition. volumes: @@ -282,9 +277,10 @@ celery: beat: name: beat - image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent + image: {} + # defaults to datatracker settings if not specified separately + # repository: "ghcr.io/ietf-tools/datatracker" + # pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. # tag: "v1.1.0" @@ -294,10 +290,7 @@ beat: livenessProbe: exec: - command: - - celery - - inspect - - ping + command: ["celery", "inspect", "ping"] periodSeconds: 30 timeoutSeconds: 5 @@ -351,10 +344,7 @@ beat: successThreshold: 1 failureThreshold: 60 exec: - command: - - celery - - inspect - - ping + command: ["celery", "inspect", "ping"] # Additional volumes on the output Deployment definition. volumes: @@ -409,10 +399,7 @@ rabbitmq: livenessProbe: exec: - command: - - rabbitmq-diagnostics - - -q - - ping + command: ["rabbitmq-diagnostics", "-q", "ping"] periodSeconds: 30 timeoutSeconds: 5 @@ -466,10 +453,7 @@ rabbitmq: successThreshold: 1 failureThreshold: 60 exec: - command: - - rabbitmq-diagnostics - - -q - - ping + command: ["rabbitmq-diagnostics", "-q", "ping"] # Additional volumes on the output Deployment definition. volumes: [] From 0faf4d868cfdceab203a488f70b1b9393eda4172 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 16 Mar 2024 23:05:17 -0400 Subject: [PATCH 11/70] ci: Update Dockerfile --- dev/build/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index fe1b51f0bd..81215c7bfa 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -18,6 +18,9 @@ RUN chmod +x start.sh && \ chmod +x docker/scripts/app-create-dirs.sh && \ sh ./docker/scripts/app-create-dirs.sh +RUN yarn build && \ + yarn legacy:build + RUN mkdir -p /a VOLUME [ "/a" ] From f6db3e8e1b4535eaba32ad5d758af8f4d9b95e55 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 16 Mar 2024 23:11:41 -0400 Subject: [PATCH 12/70] ci: Update Dockerfile --- dev/build/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 81215c7bfa..e9ebf3bcba 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -18,7 +18,8 @@ RUN chmod +x start.sh && \ chmod +x docker/scripts/app-create-dirs.sh && \ sh ./docker/scripts/app-create-dirs.sh -RUN yarn build && \ +RUN yarn rebuild && \ + yarn build && \ yarn legacy:build RUN mkdir -p /a From c987bacc95c75077febffa505242a26fd45800ac Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 19 Mar 2024 14:24:39 +1000 Subject: [PATCH 13/70] ci: Rabbitmq progress (wip) (#7233) * ci: rabbitmq service (wip) * ci: customize rabbitmq config --- helm/templates/configmap.yaml | 56 +++++++++++++++++++ helm/templates/services/rabbitmq.yaml | 20 +++++++ .../rabbitmq.yaml | 6 +- helm/values.yaml | 18 ++++-- 4 files changed, 91 insertions(+), 9 deletions(-) create mode 100644 helm/templates/services/rabbitmq.yaml rename helm/templates/{deployments => statefulsets}/rabbitmq.yaml (96%) diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 904efe8622..89983b5f07 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -5,3 +5,59 @@ metadata: data: settings_local.py: |- {{- .Files.Get "settings_local.py" | indent 4 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: rabbitmq-configmap +data: + definitions.json: |- + { + "permissions": [ + { + "configure": ".*", + "read": ".*", + "user": "datatracker", + "vhost": "dt", + "write": ".*" + } + ], + "users": [ + { + "hashing_algorithm": "rabbit_password_hashing_sha256", + "limits": {}, + "name": "datatracker", + "password_hash": "HJxcItcpXtBN+R/CH7dUelfKBOvdUs3AWo82SBw2yLMSguzb", + "tags": [] + } + ], + "vhosts": [ + { + "limits": [], + "metadata": { + "description": "", + "tags": [] + }, + "name": "dt" + } + ] + } + rabbitmq.conf: |- + # prevent guest from logging in over tcp + loopback_users.guest = true + + # load saved definitions + load_definitions = /etc/rabbitmq/definitions.json + + # Ensure that enough disk is available to flush to disk. To do this, need to limit the + # memory available to the container to something reasonable. See + # https://www.rabbitmq.com/production-checklist.html#monitoring-and-resource-usage + # for recommendations. + + # 1-1.5 times the memory available to the container is adequate for disk limit + disk_free_limit.absolute = 6000MB + + # This should be ~40% of the memory available to the container. Use an + # absolute number because relative will be proprtional to the full machine + # memory. + vm_memory_high_watermark.absolute = 1600MB diff --git a/helm/templates/services/rabbitmq.yaml b/helm/templates/services/rabbitmq.yaml new file mode 100644 index 0000000000..a23c4eaef5 --- /dev/null +++ b/helm/templates/services/rabbitmq.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq + labels: {{- include "datatracker.labels" . | nindent 4 }} + {{- with .Values.rabbitmq.service.annotations }} + annotations: + {{- range $key, $value := . }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} +spec: + type: {{.Values.rabbitmq.service.type}} + clusterIP: None # headless service + ports: + - port: {{ default "5672" .Values.rabbitmq.service.port}} + targetPort: amqp + protocol: TCP + name: amqp + selector: {{- include "datatracker.selectorLabels" . | nindent 4}} diff --git a/helm/templates/deployments/rabbitmq.yaml b/helm/templates/statefulsets/rabbitmq.yaml similarity index 96% rename from helm/templates/deployments/rabbitmq.yaml rename to helm/templates/statefulsets/rabbitmq.yaml index a44a84a252..cad4f16502 100644 --- a/helm/templates/deployments/rabbitmq.yaml +++ b/helm/templates/statefulsets/rabbitmq.yaml @@ -1,5 +1,5 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ include "datatracker.rabbitmq.fullname" . }} labels: @@ -41,8 +41,8 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} ports: - - name: http - containerPort: 8000 + - name: amqp + containerPort: 5672 protocol: TCP livenessProbe: {{- toYaml $podValues.livenessProbe | nindent 12 }} diff --git a/helm/values.yaml b/helm/values.yaml index 68b9344c02..dc5c24f065 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -387,11 +387,11 @@ beat: # ------------------------------------------------------------- rabbitmq: - name: rabbitmq + name: "rabbitmq" image: - repository: "ghcr.io/ietf-tools/datatracker-mq" + repository: "rabbitmq" pullPolicy: IfNotPresent - tag: "latest" + tag: "3.13" imagePullSecrets: [] nameOverride: "" @@ -433,7 +433,7 @@ rabbitmq: service: type: ClusterIP - port: 80 + port: 5672 serviceAccount: # Specifies whether a service account should be created @@ -456,14 +456,19 @@ rabbitmq: command: ["rabbitmq-diagnostics", "-q", "ping"] # Additional volumes on the output Deployment definition. - volumes: [] + volumes: + - name: "rabbitmq-config" + configMap: + name: "rabbitmq-configmap" # - name: foo # secret: # secretName: mysecret # optional: false # Additional volumeMounts on the output Deployment definition. - volumeMounts: [] + volumeMounts: + - name: "rabbitmq-config" + mountPath: "/etc/rabbitmq" # - name: foo # mountPath: "/etc/foo" # readOnly: true @@ -557,3 +562,4 @@ env: DBUSER: "django" DBPASS: "RkTkDPFnKpko" DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" + CELERY_BROKER_URL: "amqp://datatracker:frog@rabbitmq/dt" From f1e6c3729ff1cedde3da9cd007aa06313c57dd7c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 20 Mar 2024 05:21:38 +1000 Subject: [PATCH 14/70] ci: give rabbitmq a persistent volume claim (#7235) * ci: give rabbitmq a persistent volume claim This relies on minikube's dynamic creation of persistent volumes - expect changes likely needed for production deployment. * ci: that's not an f-string * ci: todo is todone --- helm/settings_local.py | 3 +-- helm/templates/persistentvolumeclaims.yaml | 12 ++++++++++++ helm/values.yaml | 7 ++++++- 3 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 helm/templates/persistentvolumeclaims.yaml diff --git a/helm/settings_local.py b/helm/settings_local.py index 993c8c3276..b07b559c44 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -68,14 +68,13 @@ NOMCOM_PUBLIC_KEYS_DIR = "data/nomcom_keys/public_keys/" SLIDE_STAGING_PATH = "/test/staging/" -# todo check that de-gfm is in place DE_GFM_BINARY = "/usr/local/bin/de-gfm" # OIDC configuration SITE_URL = os.environ.get("OIDC_SITE_URL") # todo: parameterize memcached url in settings.py -MEMCACHED_HOST = os.environ.get(f"MEMCACHED_SERVICE_HOST", "127.0.0.1") +MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") from ietf import __version__ CACHES = { diff --git a/helm/templates/persistentvolumeclaims.yaml b/helm/templates/persistentvolumeclaims.yaml new file mode 100644 index 0000000000..edf69aebcc --- /dev/null +++ b/helm/templates/persistentvolumeclaims.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: "rabbitmq-data-claim" +spec: +# storageClassName: "local" +# volumeName: "" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi diff --git a/helm/values.yaml b/helm/values.yaml index dc5c24f065..c64105abb1 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -37,7 +37,7 @@ datatracker: pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. # tag: "v1.1.0" - + imagePullSecrets: [] nameOverride: "" fullnameOverride: "" @@ -457,6 +457,9 @@ rabbitmq: # Additional volumes on the output Deployment definition. volumes: + - name: "rabbitmq-data" + persistentVolumeClaim: + claimName: "rabbitmq-data-claim" - name: "rabbitmq-config" configMap: name: "rabbitmq-configmap" @@ -467,6 +470,8 @@ rabbitmq: # Additional volumeMounts on the output Deployment definition. volumeMounts: + - name: "rabbitmq-data" + mountPath: "/var/lib/rabbitmq/mnesia" - name: "rabbitmq-config" mountPath: "/etc/rabbitmq" # - name: foo From b36ff61805e6d0a71b95ef26c57ae65760ce8074 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 19 Mar 2024 19:49:00 -0500 Subject: [PATCH 15/70] feat: use gunicorn (#7215) * feat: use gunicorn * fix: let gunicorn emit logs to stdout/stderr * fix: log to stdout/stderr in json format * fix: run collectstatic for the local copy of the statics --- dev/build/datatracker-start.sh | 16 ++++++++++++- ietf/settings.py | 28 +++++++++++------------ ietf/utils/log.py | 41 ++++++++-------------------------- requirements.txt | 1 + 4 files changed, 39 insertions(+), 47 deletions(-) diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index ef64ca7b30..988b8f5124 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -6,5 +6,19 @@ echo "Running Datatracker checks..." echo "Running Datatracker migrations..." ./ietf/manage.py migrate --settings=settings_local +echo "Running collectstatic..." +./ietf/manage.py collectstatic + echo "Starting Datatracker..." -./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local + +gunicorn \ + --workers 53 \ + --max-requests 32768 \ + --timeout 180 \ + --bind :8000 \ + --log-level info \ + ietf.wsgi:application + + # Leaving this here as a reminder to set up the env in the chart + # Remove this once that's complete. + #--env SCOUT_NAME=Datatracker \ diff --git a/ietf/settings.py b/ietf/settings.py index cd8c0700a1..b80f668f8e 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -236,7 +236,7 @@ def skip_unreadable_post(record): # 'loggers': { 'django': { - 'handlers': ['debug_console', 'mail_admins'], + 'handlers': ['console', 'mail_admins',], 'level': 'INFO', }, 'django.request': { @@ -248,13 +248,17 @@ def skip_unreadable_post(record): 'level': 'INFO', }, 'django.security': { - 'handlers': ['debug_console', ], + 'handlers': ['console', ], + 'level': 'INFO', + }, + 'oidc_provider': { + 'handlers': ['debug_console', ], + 'level': 'DEBUG', + }, + 'datatracker': { + 'handlers': ['console', ], 'level': 'INFO', }, - 'oidc_provider': { - 'handlers': ['debug_console', ], - 'level': 'DEBUG', - }, }, # # No logger filters @@ -263,14 +267,7 @@ def skip_unreadable_post(record): 'console': { 'level': 'DEBUG', 'class': 'logging.StreamHandler', - 'formatter': 'plain', - }, - 'syslog': { - 'level': 'DEBUG', - 'class': 'logging.handlers.SysLogHandler', - 'facility': 'user', - 'formatter': 'plain', - 'address': '/dev/log', + 'formatter': 'json', }, 'debug_console': { # Active only when DEBUG=True @@ -325,6 +322,9 @@ def skip_unreadable_post(record): 'style': '{', 'format': '{levelname}: {name}:{lineno}: {message}', }, + 'json' : { + '()': 'pythonjsonlogger.jsonlogger.JsonFormatter' + } }, } diff --git a/ietf/utils/log.py b/ietf/utils/log.py index d5a54e5516..324e8937f9 100644 --- a/ietf/utils/log.py +++ b/ietf/utils/log.py @@ -10,30 +10,19 @@ import traceback from typing import Callable # pyflakes:ignore - -try: - import syslog - logfunc = syslog.syslog # type: Callable -except ImportError: # import syslog will fail on Windows boxes - logging.basicConfig(filename='tracker.log',level=logging.INFO) - logfunc = logging.info - pass - from django.conf import settings +from pythonjsonlogger import jsonlogger import debug # pyflakes:ignore -formatter = logging.Formatter('{levelname}: {name}:{lineno}: {message}', style='{') +formatter = jsonlogger.JsonFormatter for name, level in settings.UTILS_LOGGER_LEVELS.items(): logger = logging.getLogger(name) if not logger.hasHandlers(): debug.say(' Adding handlers to logger %s' % logger.name) - handlers = [ - logging.StreamHandler(), - logging.handlers.SysLogHandler(address='/dev/log', - facility=logging.handlers.SysLogHandler.LOG_USER), - ] + logging.StreamHandler(), + ] for h in handlers: h.setFormatter(formatter) h.setLevel(level) @@ -56,20 +45,9 @@ def getcaller(): return (pmodule, pclass, pfunction, pfile, pline) def log(msg, e=None): - "Uses syslog by preference. Logs the given calling point and message." - global logfunc - def _flushfunc(): - pass - _logfunc = logfunc - if settings.SERVER_MODE == 'test': - if getattr(settings, 'show_logging', False) is True: - _logfunc = debug.say - _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition) - else: + "Logs the given calling point and message to the logging framework's datatracker handler at severity INFO" + if settings.SERVER_MODE == 'test' and not getattr(settings, 'show_logging',False): return - elif settings.DEBUG == True: - _logfunc = debug.say - _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition) if not isinstance(msg, str): msg = msg.encode('unicode_escape') try: @@ -82,11 +60,8 @@ def _flushfunc(): where = " in " + func + "()" except IndexError: file, line, where = "/", 0, "" - _flushfunc() - _logfunc("ietf%s(%d)%s: %s" % (file, line, where, msg)) - -logger = logging.getLogger('django') + logging.getLogger("datatracker").info(msg=msg, extra = {"file":file, "line":line, "where":where}) def exc_parts(): @@ -124,6 +99,7 @@ def assertion(statement, state=True, note=None): This acts like an assertion. It uses the django logger in order to send the failed assertion and a backtrace as for an internal server error. """ + logger = logging.getLogger("django") # Note this is a change - before this would have gone to "django" frame = inspect.currentframe().f_back value = eval(statement, frame.f_globals, frame.f_locals) if bool(value) != bool(state): @@ -148,6 +124,7 @@ def assertion(statement, state=True, note=None): def unreachable(date="(unknown)"): "Raises an assertion or sends traceback to admins if executed." + logger = logging.getLogger("django") frame = inspect.currentframe().f_back if settings.DEBUG is True or settings.SERVER_MODE == 'test': raise AssertionError("Arrived at code in %s() which was marked unreachable on %s." % (frame.f_code.co_name, date)) diff --git a/requirements.txt b/requirements.txt index 9231e400a7..36c0f5d4e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,6 +53,7 @@ pyopenssl>=22.0.0 # Used by urllib3.contrib, which is used by PyQuery but not pyquery>=1.4.3 python-dateutil>=2.8.2 types-python-dateutil>=2.8.2 +python-json-logger>=2.0.7 python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form failures pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=1.6 # from TastyPie From ad34104f689f0dda6c6d3423c11b5050d12d41cd Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 20 Mar 2024 16:13:26 +1000 Subject: [PATCH 16/70] ci: Deploy rabbitmq with a password (#7239) --- helm/settings_local.py | 6 +++++- helm/values.yaml | 14 +++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index b07b559c44..26d8ca8e05 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -18,7 +18,11 @@ SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY") -CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL") +CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( + host=os.environ.get("RABBITMQ_HOSTNAME", "rabbitmq"), + password=os.environ.get("CELERY_PASSWORD", ""), + queue=os.environ.get("RABBITMQ_QUEUE", "dt") +) IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" IDSUBMIT_REPOSITORY_PATH = "/test/id/" diff --git a/helm/values.yaml b/helm/values.yaml index c64105abb1..2902b346bc 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -179,7 +179,7 @@ celery: livenessProbe: exec: - command: ["celery", "inspect", "ping"] + command: ["celery", "-A", "ietf", "inspect", "ping"] periodSeconds: 30 timeoutSeconds: 5 @@ -233,7 +233,7 @@ celery: successThreshold: 1 failureThreshold: 60 exec: - command: ["celery", "inspect", "ping"] + command: ["celery", "-A", "ietf", "inspect", "ping"] # Additional volumes on the output Deployment definition. volumes: @@ -290,7 +290,7 @@ beat: livenessProbe: exec: - command: ["celery", "inspect", "ping"] + command: ["celery", "-A", "ietf", "inspect", "ping"] periodSeconds: 30 timeoutSeconds: 5 @@ -344,7 +344,7 @@ beat: successThreshold: 1 failureThreshold: 60 exec: - command: ["celery", "inspect", "ping"] + command: ["celery", "-A", "ietf", "inspect", "ping"] # Additional volumes on the output Deployment definition. volumes: @@ -389,9 +389,9 @@ beat: rabbitmq: name: "rabbitmq" image: - repository: "rabbitmq" + repository: "ghcr.io/ietf-tools/datatracker-mq" pullPolicy: IfNotPresent - tag: "3.13" + tag: "3.12-alpine" imagePullSecrets: [] nameOverride: "" @@ -567,4 +567,4 @@ env: DBUSER: "django" DBPASS: "RkTkDPFnKpko" DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" - CELERY_BROKER_URL: "amqp://datatracker:frog@rabbitmq/dt" + CELERY_PASSWORD: "this-is-a-secret" From 1c90789b4989059f5a0612b8bca920bb8050ec0e Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 20 Mar 2024 16:40:36 -0500 Subject: [PATCH 17/70] chore: reduce worker count and disable live checks for the datatracker pod (#7240) --- dev/build/datatracker-start.sh | 2 +- helm/values.yaml | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index 988b8f5124..c04834b97b 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -12,7 +12,7 @@ echo "Running collectstatic..." echo "Starting Datatracker..." gunicorn \ - --workers 53 \ + --workers 9 \ --max-requests 32768 \ --timeout 180 \ --bind :8000 \ diff --git a/helm/values.yaml b/helm/values.yaml index 2902b346bc..df75ebb8bc 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -58,11 +58,11 @@ datatracker: # hosts: # - chart-example.local - livenessProbe: - httpGet: - # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint - path: /submit/tool-instructions/ - port: http +# livenessProbe: +# httpGet: +# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint +# path: /submit/tool-instructions/ +# port: http podAnnotations: {} podLabels: {} @@ -113,16 +113,16 @@ datatracker: # If not set and create is true, a name is generated using the fullname template name: "" - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - httpGet: - # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint - path: /submit/tool-instructions/ - port: http +# startupProbe: +# initialDelaySeconds: 15 +# periodSeconds: 5 +# timeoutSeconds: 5 +# successThreshold: 1 +# failureThreshold: 60 +# httpGet: +# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint +# path: /submit/tool-instructions/ +# port: http # Additional volumes on the output Deployment definition. volumes: From 7a238a363e5bb645c535aab96e281736177c2c84 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 26 Mar 2024 09:03:07 -0600 Subject: [PATCH 18/70] ci: collectstatic --no-input (#7252) --- dev/build/datatracker-start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index c04834b97b..1e4c5f52b3 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -7,7 +7,7 @@ echo "Running Datatracker migrations..." ./ietf/manage.py migrate --settings=settings_local echo "Running collectstatic..." -./ietf/manage.py collectstatic +./ietf/manage.py collectstatic --no-input echo "Starting Datatracker..." From e3d02904808b968f02765c83632b16435835945d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 27 Mar 2024 16:02:59 -0300 Subject: [PATCH 19/70] ci: install libreoffice (#7262) Use bullseye-backports to get something more recent (ietfa has 7.3.6.2; bullseye has 7.0.x.x; bullseye-backports has 7.4.7.2) --- dev/build/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index e9ebf3bcba..f9ff3414e3 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -5,6 +5,11 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get purge -y imagemagick imagemagick-6-common +# Install libreoffice (needed via PPT2PDF_COMMAND) +RUN echo "deb http://deb.debian.org/debian bullseye-backports main" > /etc/apt/sources.list.d/bullseye-backports.list && \ + apt-get update && \ + apt-get -qyt bullseye-backports install libreoffice-nogui + COPY . . COPY ./dev/build/start.sh ./start.sh COPY ./dev/build/datatracker-start.sh ./datatracker-start.sh From f58bbc3caa251279b7103669c3feeb4ddfb0226e Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 27 Mar 2024 16:04:30 -0300 Subject: [PATCH 20/70] ci: parameterize / update settings (#7248) * ci: parameterize gunicorn in datatracker-start.sh * fix: typo * ci: update settings_local for helm chart * ci: Add todo comment * ci: Drop redundant USE_TZ setting * ci: Require secrets in production * ci: fix indentation * style: Black * ci: memcached cfg from env in settings.py * ci: set SITE_URL in settings.py * refactor: /www/htpasswd -> /a/www/htpasswd (it's a symlink on production) * refactor: Remove obsolete SECR_ settings * refactor: SECR_MAX_UPLOAD_SIZE -> DATATRACKER_... * refactor: SECR_PPT2PDF_COMMAND -> PPT2PDF_COMMAND * ci: Fix up helm/settings_local * ci: Remove commented-out settings * ci: Refactor/improve env var guards * ci: More env refactoring / guards --- dev/build/datatracker-start.sh | 8 +- helm/settings_local.py | 224 ++++++++++++++++++++++++--------- helm/values.yaml | 37 +++++- ietf/meeting/utils.py | 4 +- ietf/secr/meetings/tests.py | 20 +-- ietf/settings.py | 115 +++++++++-------- ietf/utils/validators.py | 11 +- 7 files changed, 275 insertions(+), 144 deletions(-) diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index 1e4c5f52b3..da355175ff 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -12,11 +12,11 @@ echo "Running collectstatic..." echo "Starting Datatracker..." gunicorn \ - --workers 9 \ - --max-requests 32768 \ - --timeout 180 \ + --workers ${DATATRACKER_GUNICORN_WORKERS:-9} \ + --max-requests ${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768} \ + --timeout ${DATATRACKER_GUNICORN_TIMEOUT:-180} \ --bind :8000 \ - --log-level info \ + --log-level ${DATATRACKER_GUNICORN_LOG_LEVEL:-info} \ ietf.wsgi:application # Leaving this here as a reminder to set up the env in the chart diff --git a/helm/settings_local.py b/helm/settings_local.py index 26d8ca8e05..92e7bce1a4 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -1,22 +1,100 @@ # Copyright The IETF Trust 2007-2024, All Rights Reserved # -*- coding: utf-8 -*- +from base64 import b64decode +from email.utils import parseaddr + +from ietf import __release_hash__ from ietf.settings import * # pyflakes:ignore -ALLOWED_HOSTS = ['*'] + +# Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env! +SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development") + +# Secrets +_SECRET_KEY = os.environ.get("DATATRACKER_DJANGO_SECRET_KEY", None) +if _SECRET_KEY is not None: + SECRET_KEY = _SECRET_KEY +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set in production") + +_NOMCOM_APP_SECRET_B64 = os.environ.get("DATATRACKER_NOMCOM_APP_SECRET_B64", None) +if _NOMCOM_APP_SECRET_B64 is not None: + NOMCOM_APP_SECRET = b64decode(_NOMCOM_APP_SECRET_B64) +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_NOMCOM_APP_SECRET_B64 must be set in production") + +_IANA_SYNC_PASSWORD = os.environ.get("DATATRACKER_IANA_SYNC_PASSWORD", None) +if _IANA_SYNC_PASSWORD is not None: + IANA_SYNC_PASSWORD = _IANA_SYNC_PASSWORD +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set in production") + +_RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD", None) +if _RFC_EDITOR_SYNC_PASSWORD is not None: + RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD") +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD must be set in production") + +_YOUTUBE_API_KEY = os.environ.get("DATATRACKER_YOUTUBE_API_KEY", None) +if _YOUTUBE_API_KEY is not None: + YOUTUBE_API_KEY = _YOUTUBE_API_KEY +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_YOUTUBE_API_KEY must be set in production") + +_GITHUB_BACKUP_API_KEY = os.environ.get("DATATRACKER_GITHUB_BACKUP_API_KEY", None) +if _GITHUB_BACKUP_API_KEY is not None: + GITHUB_BACKUP_API_KEY = _GITHUB_BACKUP_API_KEY +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set in production") + +_API_KEY_TYPE = os.environ.get("DATATRACKER_API_KEY_TYPE", None) +if _API_KEY_TYPE is not None: + API_KEY_TYPE = _API_KEY_TYPE +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set in production") + +_API_PUBLIC_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PUBLIC_KEY_PEM_B64", None) +if _API_PUBLIC_KEY_PEM_B64 is not None: + API_PUBLIC_KEY_PEM = b64decode(_API_PUBLIC_KEY_PEM_B64) +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set in production") + +_API_PRIVATE_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PRIVATE_KEY_PEM_B64", None) +if _API_PRIVATE_KEY_PEM_B64 is not None: + API_PRIVATE_KEY_PEM = b64decode(_API_PRIVATE_KEY_PEM_B64) +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set in production") + +# Set DEBUG if DATATRACKER_DEBUG env var is the word "true" +DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true" + +# DATATRACKER_ALLOWED_HOSTS env var is a comma-separated list of allowed hosts +_allowed_hosts_str = os.environ.get("DATATRACKER_ALLOWED_HOSTS", None) +if _allowed_hosts_str is not None: + ALLOWED_HOSTS = [h.strip() for h in _allowed_hosts_str.split(",")] DATABASES = { "default": { - "HOST": os.environ.get("DBHOST", "db"), - "PORT": os.environ.get("DBPORT", "5432"), - "NAME": os.environ.get("DBNAME", "datatracker"), + "HOST": os.environ.get("DATATRACKER_DBHOST", "db"), + "PORT": os.environ.get("DATATRACKER_DBPORT", "5432"), + "NAME": os.environ.get("DATATRACKER_DBNAME", "datatracker"), "ENGINE": "django.db.backends.postgresql", - "USER": os.environ.get("DBUSER", "django"), - "PASSWORD": os.environ.get("DBPASS", ""), + "USER": os.environ.get("DATATRACKER_DBUSER", "django"), + "PASSWORD": os.environ.get("DATATRACKER_DBPASS", ""), }, } -SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY") +# DATATRACKER_ADMINS is a newline-delimited list of addresses parseable by email.utils.parseaddr +_ADMINS = os.environ.get("DATATRACKER_ADMINS", None) +if _ADMINS is not None: + ADMINS = [parseaddr(admin) for admin in _ADMINS.split("\n")] +elif SERVER_MODE == "production": + raise RuntimeError("DATATRACKER_ADMINS must be set in production") + +USING_DEBUG_EMAIL_SERVER = os.environ.get("DATATRACKER_EMAIL_DEBUG", "false").lower() == "true" +EMAIL_HOST = os.environ.get("DATATRACKER_EMAIL_HOST", "localhost") +EMAIL_PORT = int(os.environ.get("DATATRACKER_EMAIL_PORT", "2025")) CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( host=os.environ.get("RABBITMQ_HOSTNAME", "rabbitmq"), @@ -24,62 +102,99 @@ queue=os.environ.get("RABBITMQ_QUEUE", "dt") ) -IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" -IDSUBMIT_REPOSITORY_PATH = "/test/id/" -IDSUBMIT_STAGING_PATH = "/test/staging/" +IANA_SYNC_USERNAME = "ietfsync" +IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" +IANA_SYNC_PROTOCOLS_URL = "http://www.iana.org/protocols/" -AGENDA_PATH = "/assets/www6s/proceedings/" -MEETINGHOST_LOGO_PATH = AGENDA_PATH +RFC_EDITOR_NOTIFICATION_URL = "http://www.rfc-editor.org/parser/parser.php" -USING_DEBUG_EMAIL_SERVER=True -EMAIL_HOST= "localhost" -EMAIL_PORT=2025 +STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://registration.ietf.org/{number}/attendees/?apikey=redacted' -MEDIA_BASE_DIR = "/assets" -MEDIA_ROOT = MEDIA_BASE_DIR + "/media/" -MEDIA_URL = "/media/" +#FIRST_CUTOFF_DAYS = 12 +#SECOND_CUTOFF_DAYS = 12 +#SUBMISSION_CUTOFF_DAYS = 26 +#SUBMISSION_CORRECTION_DAYS = 57 +MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26 +MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 54 + +HTPASSWD_COMMAND = "/usr/bin/htpasswd2" + +_MEETECHO_CLIENT_ID = os.environ.get("DATATRACKER_MEETECHO_CLIENT_ID", None) +_MEETECHO_CLIENT_SECRET = os.environ.get("DATATRACKER_MEETECHO_CLIENT_SECRET", None) +if _MEETECHO_CLIENT_ID is not None and _MEETECHO_CLIENT_SECRET is not None: + MEETECHO_API_CONFIG = { + "api_base": os.environ.get( + "DATATRACKER_MEETECHO_API_BASE", + "https://meetings.conf.meetecho.com/api/v1/", + ), + "client_id": _MEETECHO_CLIENT_ID, + "client_secret": _MEETECHO_CLIENT_SECRET, + "request_timeout": 3.01, # python-requests doc recommend slightly > a multiple of 3 seconds + } +elif SERVER_MODE == "production": + raise RuntimeError( + "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set in production" + ) + +APP_API_TOKENS = { + "ietf.api.views.directauth": ["redacted",], + "ietf.api.views.email_aliases": ["redacted"], + "ietf.api.views.active_email_list": ["redacted"], +} +EMAIL_COPY_TO = "" + +# Until we teach the datatracker to look beyond cloudflare for this check +IDSUBMIT_MAX_DAILY_SAME_SUBMITTER = 5000 + +# Leave DATATRACKER_MATOMO_SITE_ID unset to disable Matomo reporting +if "DATATRACKER_MATOMO_SITE_ID" in os.environ: + MATOMO_DOMAIN_PATH = os.environ.get("DATATRACKER_MATOMO_DOMAIN_PATH", "analytics.ietf.org") + MATOMO_SITE_ID = os.environ.get("DATATRACKER_MATOMO_SITE_ID") + MATOMO_DISABLE_COOKIES = True + +# Leave DATATRACKER_SCOUT_KEY unset to disable Scout APM agent +_SCOUT_KEY = os.environ.get("DATATRACKER_SCOUT_KEY", None) +if _SCOUT_KEY is not None: + if SERVER_MODE == "production": + PROD_PRE_APPS = ["scout_apm.django", ] + else: + DEV_PRE_APPS = ["scout_apm.django", ] + SCOUT_MONITOR = True + SCOUT_KEY = _SCOUT_KEY + SCOUT_NAME = "Datatracker" + SCOUT_ERRORS_ENABLED = True + SCOUT_SHUTDOWN_MESSAGE_ENABLED = False + SCOUT_CORE_AGENT_DIR = "/a/core-agent/1.4.0" + SCOUT_CORE_AGENT_FULL_NAME = "scout_apm_core-v1.4.0-x86_64-unknown-linux-musl" + SCOUT_CORE_AGENT_SOCKET_PATH = "tcp://{host}:{port}".format( + host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "scout"), + port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "16590"), + ), + SCOUT_CORE_AGENT_DOWNLOAD = False + SCOUT_CORE_AGENT_LAUNCH = False + SCOUT_REVISION_SHA = __release_hash__[:7] + +# Path to the email alias lists. Used by ietf.utils.aliases +DRAFT_ALIASES_PATH = "/a/postfix/draft-aliases" +DRAFT_VIRTUAL_PATH = "/a/postfix/draft-virtual" +GROUP_ALIASES_PATH = "/a/postfix/group-aliases" +GROUP_VIRTUAL_PATH = "/a/postfix/group-virtual" + +# Set these to the same as "production" in settings.py, whether production mode or not +MEDIA_ROOT = "/a/www/www6s/lib/dt/media/" +MEDIA_URL = "https://www.ietf.org/lib/dt/media/" PHOTOS_DIRNAME = "photo" PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME -SUBMIT_YANG_CATALOG_MODEL_DIR = "/assets/ietf-ftp/yang/catalogmod/" -SUBMIT_YANG_DRAFT_MODEL_DIR = "/assets/ietf-ftp/yang/draftmod/" -SUBMIT_YANG_INVAL_MODEL_DIR = "/assets/ietf-ftp/yang/invalmod/" -SUBMIT_YANG_IANA_MODEL_DIR = "/assets/ietf-ftp/yang/ianamod/" -SUBMIT_YANG_RFC_MODEL_DIR = "/assets/ietf-ftp/yang/rfcmod/" - -# Set INTERNAL_IPS for use within Docker. See https://knasmueller.net/fix-djangos-debug-toolbar-not-showing-inside-docker -import socket -hostname, _, ips = socket.gethostbyname_ex(socket.gethostname()) -INTERNAL_IPS = [".".join(ip.split(".")[:-1] + ["1"]) for ip in ips] - -# DEV_TEMPLATE_CONTEXT_PROCESSORS = [ -# 'ietf.context_processors.sql_debug', -# ] - -DOCUMENT_PATH_PATTERN = "/assets/ietfdata/doc/{doc.type_id}/" -INTERNET_DRAFT_PATH = "/assets/ietf-ftp/internet-drafts/" -RFC_PATH = "/assets/ietf-ftp/rfc/" -CHARTER_PATH = "/assets/ietf-ftp/charter/" -BOFREQ_PATH = "/assets/ietf-ftp/bofreq/" -CONFLICT_REVIEW_PATH = "/assets/ietf-ftp/conflict-reviews/" -STATUS_CHANGE_PATH = "/assets/ietf-ftp/status-changes/" -INTERNET_DRAFT_ARCHIVE_DIR = "/assets/archive/id" -INTERNET_ALL_DRAFTS_ARCHIVE_DIR = "/assets/archive/id" -BIBXML_BASE_PATH = "/assets/ietfdata/derived/bibxml" -IDSUBMIT_REPOSITORY_PATH = INTERNET_DRAFT_PATH - -NOMCOM_PUBLIC_KEYS_DIR = "data/nomcom_keys/public_keys/" -SLIDE_STAGING_PATH = "/test/staging/" +# Normally only set for debug, but needed until we have a real FS +DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') +# Binaries that are different in the docker image DE_GFM_BINARY = "/usr/local/bin/de-gfm" +IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" -# OIDC configuration -SITE_URL = os.environ.get("OIDC_SITE_URL") - -# todo: parameterize memcached url in settings.py -MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") -MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") +# Duplicating production cache from settings.py and using it whether we're in production mode or not from ietf import __version__ CACHES = { "default": { @@ -119,6 +234,3 @@ }, }, } - -# Normally only set for debug, but needed until we have a real FS -DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') diff --git a/helm/values.yaml b/helm/values.yaml index df75ebb8bc..3eedd0e1f6 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -561,10 +561,35 @@ autoscaling: env: # n.b., these are debug values / non-secret secrets - # DBHOST: "host.minikube.internal" - # DBPORT: "5432" - DBNAME: "datatracker" - DBUSER: "django" - DBPASS: "RkTkDPFnKpko" - DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" + DATATRACKER_SERVER_MODE: "development" # defaults to "production" + DATATRACKER_ADMINS: |- + Robert Sparks + Ryan Cross + Kesara Rathnayake + Jennifer Richards + Nicolas Giard + DATATRACKER_ALLOWED_HOSTS: "*" # empty for production + # DATATRACKER_DATATRACKER_DEBUG: "false" + # DATATRACKER_DBHOST: "db" + # DATATRACKER_DBPORT: "5432" + # DATATRACKER_DBNAME: "datatracker" + # DATATRACKER_DBUSER: "django" + DATATRACKER_DBPASS: "RkTkDPFnKpko" + DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" + DATATRACKER_EMAIL_DEBUG: "true" + DATATRACKER_EMAIL_HOST: "localhost" + DATATRACKER_EMAIL_PORT: "2025" + # DATATRACKER_NOMCOM_APP_SECRET_B64: "" + DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" + DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" + DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" + DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" + # DATATRACKER_API_KEY_TYPE: "ES265" + # DATATRACKER_API_PUBLIC_KEY_PEM_B64: " settings.SECR_MAX_UPLOAD_SIZE: - raise ValidationError('Please keep filesize under %s. Requested upload size was %s' % (filesizeformat(settings.SECR_MAX_UPLOAD_SIZE), filesizeformat(file.size))) + if size > settings.DATATRACKER_MAX_UPLOAD_SIZE: + raise ValidationError( + "Please keep filesize under {}. Requested upload size was {}".format( + filesizeformat(settings.DATATRACKE_MAX_UPLOAD_SIZE), + filesizeformat(file.size) + ) + ) + def validate_mime_type(file, valid, missing_ok=False): try: From 253ba1dfbd86fdf8368a2cfc96f393e82011af9f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 28 Mar 2024 20:08:45 -0300 Subject: [PATCH 21/70] fix: mypy/flakes lint --- ietf/secr/meetings/tests.py | 3 --- ietf/utils/log.py | 2 +- ietf/utils/validators.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ietf/secr/meetings/tests.py b/ietf/secr/meetings/tests.py index 9eed6b94f6..a241d2b5eb 100644 --- a/ietf/secr/meetings/tests.py +++ b/ietf/secr/meetings/tests.py @@ -3,14 +3,11 @@ import datetime -import os -import shutil from pyquery import PyQuery import debug # pyflakes:ignore -from django.conf import settings from django.urls import reverse from django.utils import timezone diff --git a/ietf/utils/log.py b/ietf/utils/log.py index 324e8937f9..7c2e233f9b 100644 --- a/ietf/utils/log.py +++ b/ietf/utils/log.py @@ -24,7 +24,7 @@ logging.StreamHandler(), ] for h in handlers: - h.setFormatter(formatter) + h.setFormatter(formatter) # type: ignore h.setLevel(level) logger.addHandler(h) debug.say(" Setting %s logging level to %s" % (logger.name, level)) diff --git a/ietf/utils/validators.py b/ietf/utils/validators.py index 83814b24a5..8fe989df99 100644 --- a/ietf/utils/validators.py +++ b/ietf/utils/validators.py @@ -73,7 +73,7 @@ def validate_file_size(file, missing_ok=False): if size > settings.DATATRACKER_MAX_UPLOAD_SIZE: raise ValidationError( "Please keep filesize under {}. Requested upload size was {}".format( - filesizeformat(settings.DATATRACKE_MAX_UPLOAD_SIZE), + filesizeformat(settings.DATATRACKER_MAX_UPLOAD_SIZE), filesizeformat(file.size) ) ) From 8872e31e3045327ce85925e9dec56a63167aaf5c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 2 Apr 2024 11:02:45 -0300 Subject: [PATCH 22/70] ci: Get APP_API_TOKENS from env (#7271) * ci: Get APP_API_TOKENS from env * ci: chart fixup * ci: Remove canned APP_API_TOKENs (setec astronomy) --- helm/settings_local.py | 13 ++++++++----- helm/values.yaml | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index 92e7bce1a4..27ac845959 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -3,6 +3,7 @@ from base64 import b64decode from email.utils import parseaddr +import json from ietf import __release_hash__ from ietf.settings import * # pyflakes:ignore @@ -136,11 +137,11 @@ "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set in production" ) -APP_API_TOKENS = { - "ietf.api.views.directauth": ["redacted",], - "ietf.api.views.email_aliases": ["redacted"], - "ietf.api.views.active_email_list": ["redacted"], -} +_APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None) +if _APP_API_TOKENS_JSON is not None: + APP_API_TOKENS = json.loads(_APP_API_TOKENS_JSON) +else: + APP_API_TOKENS = {} EMAIL_COPY_TO = "" @@ -195,6 +196,8 @@ IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" # Duplicating production cache from settings.py and using it whether we're in production mode or not +MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") +MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") from ietf import __version__ CACHES = { "default": { diff --git a/helm/values.yaml b/helm/values.yaml index 3eedd0e1f6..d09df4d65b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -593,3 +593,4 @@ env: # DATATRACKER_MATOMO_SITE_ID: "7" # must be present to enable Matomo # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org" CELERY_PASSWORD: "this-is-a-secret" + # DATATRACKER_APP_API_TOKENS_JSON: "" From 46e521ea9ac581a38033bf8503b404aa70b7f206 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 2 Apr 2024 17:51:15 -0300 Subject: [PATCH 23/70] ci: shared PersistentVolumeClaim on /a (#7283) * ci: shared PersistentVolumeClaim on /a * ci: Increase storage request, add comment --- helm/templates/persistentvolumeclaims.yaml | 14 ++++ helm/values.yaml | 81 ++++++++++++++-------- 2 files changed, 65 insertions(+), 30 deletions(-) diff --git a/helm/templates/persistentvolumeclaims.yaml b/helm/templates/persistentvolumeclaims.yaml index edf69aebcc..db636fb05e 100644 --- a/helm/templates/persistentvolumeclaims.yaml +++ b/helm/templates/persistentvolumeclaims.yaml @@ -1,5 +1,19 @@ apiVersion: v1 kind: PersistentVolumeClaim +metadata: + name: "datatracker-shared-volume-claim" + # Note: This is the /a shared volume. The spec should be adjusted to ensure it always + # matches the correct EBS volume in production. For the moment, it just requests a + # large volume. +spec: + accessModes: + - ReadWriteMany # or ReadWriteOnce and force datatracker/celery/beat to a single node + resources: + requests: + storage: 600Gi # adjust to something satisfied by the /a PersistentVolume +--- +apiVersion: v1 +kind: PersistentVolumeClaim metadata: name: "rabbitmq-data-claim" spec: diff --git a/helm/values.yaml b/helm/values.yaml index d09df4d65b..90ff438b29 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -129,12 +129,16 @@ datatracker: - name: settings-local-volume configMap: name: django-configmap - - name: cache-volume - emptyDir: - sizeLimit: 1Gi - - name: staging-volume - emptyDir: - sizeLimit: 1Gi + - name: datatracker-shared-volume + persistentVolumeClaim: + claimName: "datatracker-shared-volume-claim" +# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume +# - name: cache-volume +# emptyDir: +# sizeLimit: 1Gi +# - name: staging-volume +# emptyDir: +# sizeLimit: 1Gi # - name: foo # secret: # secretName: mysecret @@ -146,10 +150,13 @@ datatracker: mountPath: /workspace/ietf/settings_local.py subPath: settings_local.py readOnly: true - - name: cache-volume - mountPath: "/a/cache" - - name: staging-volume - mountPath: "/test/staging" + - name: datatracker-shared-volume + mountPath: /a +# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume +# - name: cache-volume +# mountPath: "/a/cache" +# - name: staging-volume +# mountPath: "/test/staging" # - name: foo # mountPath: "/etc/foo" # readOnly: true @@ -240,12 +247,16 @@ celery: - name: settings-local-volume configMap: name: django-configmap - - name: cache-volume - emptyDir: - sizeLimit: 1Gi - - name: staging-volume - emptyDir: - sizeLimit: 1Gi + - name: datatracker-shared-volume + persistentVolumeClaim: + claimName: "datatracker-shared-volume-claim" +# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume +# - name: cache-volume +# emptyDir: +# sizeLimit: 1Gi +# - name: staging-volume +# emptyDir: +# sizeLimit: 1Gi # - name: foo # secret: # secretName: mysecret @@ -257,10 +268,13 @@ celery: mountPath: /workspace/ietf/settings_local.py subPath: settings_local.py readOnly: true - - name: cache-volume - mountPath: "/a/cache" - - name: staging-volume - mountPath: "/test/staging" + - name: datatracker-shared-volume + mountPath: /a +# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume +# - name: cache-volume +# mountPath: "/a/cache" +# - name: staging-volume +# mountPath: "/test/staging" # - name: foo # mountPath: "/etc/foo" # readOnly: true @@ -351,12 +365,16 @@ beat: - name: settings-local-volume configMap: name: django-configmap - - name: cache-volume - emptyDir: - sizeLimit: 1Gi - - name: staging-volume - emptyDir: - sizeLimit: 1Gi + - name: datatracker-shared-volume + persistentVolumeClaim: + claimName: "datatracker-shared-volume-claim" +# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume +# - name: cache-volume +# emptyDir: +# sizeLimit: 1Gi +# - name: staging-volume +# emptyDir: +# sizeLimit: 1Gi # - name: foo # secret: # secretName: mysecret @@ -368,10 +386,13 @@ beat: mountPath: /workspace/ietf/settings_local.py subPath: settings_local.py readOnly: true - - name: cache-volume - mountPath: "/a/cache" - - name: staging-volume - mountPath: "/test/staging" + - name: datatracker-shared-volume + mountPath: /a +# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume +# - name: cache-volume +# mountPath: "/a/cache" +# - name: staging-volume +# mountPath: "/test/staging" # - name: foo # mountPath: "/etc/foo" # readOnly: true From c4a2ea98223db4fb8b92469a5d0f60f332ed7a98 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 3 Apr 2024 16:02:27 -0300 Subject: [PATCH 24/70] ci: Adjust secrets in values.yaml (#7288) * ci: Require secrets, even in "development" * ci: More secrets-requiring * ci: Strip whitespace out before b64 decoding * ci: Adjust values.yaml * ci: Comment in values.yaml --- helm/settings_local.py | 58 +++++++++++++++++++++---------------- helm/values.yaml | 66 +++++++++++++++++++++++++++++------------- 2 files changed, 79 insertions(+), 45 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index 27ac845959..eb960839b6 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -9,6 +9,11 @@ from ietf.settings import * # pyflakes:ignore +def _remove_whitespace_and_b64decode(s): + """Helper to strip out whitespace and base64 decode""" + return b64decode("".join(s.split())) + + # Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env! SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development") @@ -16,56 +21,56 @@ _SECRET_KEY = os.environ.get("DATATRACKER_DJANGO_SECRET_KEY", None) if _SECRET_KEY is not None: SECRET_KEY = _SECRET_KEY -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set in production") +else: + raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set") _NOMCOM_APP_SECRET_B64 = os.environ.get("DATATRACKER_NOMCOM_APP_SECRET_B64", None) if _NOMCOM_APP_SECRET_B64 is not None: - NOMCOM_APP_SECRET = b64decode(_NOMCOM_APP_SECRET_B64) -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_NOMCOM_APP_SECRET_B64 must be set in production") + NOMCOM_APP_SECRET = _remove_whitespace_and_b64decode(_NOMCOM_APP_SECRET_B64) +else: + raise RuntimeError("DATATRACKER_NOMCOM_APP_SECRET_B64 must be set") _IANA_SYNC_PASSWORD = os.environ.get("DATATRACKER_IANA_SYNC_PASSWORD", None) if _IANA_SYNC_PASSWORD is not None: IANA_SYNC_PASSWORD = _IANA_SYNC_PASSWORD -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set in production") +else: + raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set") _RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD", None) if _RFC_EDITOR_SYNC_PASSWORD is not None: RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD") -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD must be set in production") +else: + raise RuntimeError("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD must be set") _YOUTUBE_API_KEY = os.environ.get("DATATRACKER_YOUTUBE_API_KEY", None) if _YOUTUBE_API_KEY is not None: YOUTUBE_API_KEY = _YOUTUBE_API_KEY -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_YOUTUBE_API_KEY must be set in production") +else: + raise RuntimeError("DATATRACKER_YOUTUBE_API_KEY must be set") _GITHUB_BACKUP_API_KEY = os.environ.get("DATATRACKER_GITHUB_BACKUP_API_KEY", None) if _GITHUB_BACKUP_API_KEY is not None: GITHUB_BACKUP_API_KEY = _GITHUB_BACKUP_API_KEY -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set in production") +else: + raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set") _API_KEY_TYPE = os.environ.get("DATATRACKER_API_KEY_TYPE", None) if _API_KEY_TYPE is not None: API_KEY_TYPE = _API_KEY_TYPE -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set in production") +else: + raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set") _API_PUBLIC_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PUBLIC_KEY_PEM_B64", None) if _API_PUBLIC_KEY_PEM_B64 is not None: - API_PUBLIC_KEY_PEM = b64decode(_API_PUBLIC_KEY_PEM_B64) -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set in production") + API_PUBLIC_KEY_PEM = _remove_whitespace_and_b64decode(_API_PUBLIC_KEY_PEM_B64) +else: + raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set") _API_PRIVATE_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PRIVATE_KEY_PEM_B64", None) if _API_PRIVATE_KEY_PEM_B64 is not None: - API_PRIVATE_KEY_PEM = b64decode(_API_PRIVATE_KEY_PEM_B64) -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set in production") + API_PRIVATE_KEY_PEM = _remove_whitespace_and_b64decode(_API_PRIVATE_KEY_PEM_B64) +else: + raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") # Set DEBUG if DATATRACKER_DEBUG env var is the word "true" DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true" @@ -73,7 +78,7 @@ # DATATRACKER_ALLOWED_HOSTS env var is a comma-separated list of allowed hosts _allowed_hosts_str = os.environ.get("DATATRACKER_ALLOWED_HOSTS", None) if _allowed_hosts_str is not None: - ALLOWED_HOSTS = [h.strip() for h in _allowed_hosts_str.split(",")] + ALLOWED_HOSTS = [h.strip() for h in _allowed_hosts_str.split("\n")] DATABASES = { "default": { @@ -90,16 +95,19 @@ _ADMINS = os.environ.get("DATATRACKER_ADMINS", None) if _ADMINS is not None: ADMINS = [parseaddr(admin) for admin in _ADMINS.split("\n")] -elif SERVER_MODE == "production": - raise RuntimeError("DATATRACKER_ADMINS must be set in production") +else: + raise RuntimeError("DATATRACKER_ADMINS must be set") USING_DEBUG_EMAIL_SERVER = os.environ.get("DATATRACKER_EMAIL_DEBUG", "false").lower() == "true" EMAIL_HOST = os.environ.get("DATATRACKER_EMAIL_HOST", "localhost") EMAIL_PORT = int(os.environ.get("DATATRACKER_EMAIL_PORT", "2025")) +_celery_password = os.environ.get("CELERY_PASSWORD", None) +if _celery_password is None: + raise RuntimeError("CELERY_PASSWORD must be set") CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( host=os.environ.get("RABBITMQ_HOSTNAME", "rabbitmq"), - password=os.environ.get("CELERY_PASSWORD", ""), + password=_celery_password, queue=os.environ.get("RABBITMQ_QUEUE", "dt") ) diff --git a/helm/values.yaml b/helm/values.yaml index 90ff438b29..27094f5464 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -582,36 +582,62 @@ autoscaling: env: # n.b., these are debug values / non-secret secrets - DATATRACKER_SERVER_MODE: "development" # defaults to "production" + DATATRACKER_SERVER_MODE: "development" # development for staging, production for production DATATRACKER_ADMINS: |- Robert Sparks Ryan Cross Kesara Rathnayake Jennifer Richards Nicolas Giard - DATATRACKER_ALLOWED_HOSTS: "*" # empty for production + DATATRACKER_ALLOWED_HOSTS: ".ietf.org" # newline-separated list also allowed # DATATRACKER_DATATRACKER_DEBUG: "false" + + # DB access details - needs to be filled in # DATATRACKER_DBHOST: "db" # DATATRACKER_DBPORT: "5432" # DATATRACKER_DBNAME: "datatracker" - # DATATRACKER_DBUSER: "django" - DATATRACKER_DBPASS: "RkTkDPFnKpko" - DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" - DATATRACKER_EMAIL_DEBUG: "true" - DATATRACKER_EMAIL_HOST: "localhost" - DATATRACKER_EMAIL_PORT: "2025" - # DATATRACKER_NOMCOM_APP_SECRET_B64: "" - DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" - DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" - DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" - DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" - # DATATRACKER_API_KEY_TYPE: "ES265" - # DATATRACKER_API_PUBLIC_KEY_PEM_B64: "" + + CELERY_PASSWORD: "this-is-a-secret" # secret + + DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret From 206a4bb7490da944d48d6777515232f4e745b16d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 11 Apr 2024 11:56:44 -0300 Subject: [PATCH 25/70] ci: clean up labels and service values (#7314) * ci: Remove unused service values * ci: Reorg labels --- helm/templates/_helpers.tpl | 6 +++--- helm/templates/deployments/beat.yaml | 5 ++++- helm/templates/deployments/celery.yaml | 5 ++++- helm/templates/deployments/datatracker.yaml | 5 ++++- helm/templates/deployments/memcached.yaml | 5 ++++- helm/templates/hpa.yaml | 4 ++-- helm/templates/ingress.yaml | 2 +- helm/templates/rbac/beat-serviceaccount.yaml | 2 +- helm/templates/rbac/celery-serviceaccount.yaml | 2 +- helm/templates/rbac/datatracker-serviceaccount.yaml | 2 +- helm/templates/rbac/memcached-serviceaccount.yaml | 2 +- helm/templates/rbac/rabbitmq-serviceaccount.yaml | 2 +- helm/templates/services/datatracker.yaml | 6 ++++-- helm/templates/services/memcached.yaml | 6 ++++-- helm/templates/services/rabbitmq.yaml | 6 ++++-- helm/templates/statefulsets/rabbitmq.yaml | 5 ++++- helm/values.yaml | 8 -------- 17 files changed, 43 insertions(+), 30 deletions(-) diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index 268f0e75dc..503b3f1bb8 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -118,20 +118,20 @@ Create chart name and version as used by the chart label. {{/* Common labels */}} -{{- define "datatracker.labels" -}} +{{- define "datatracker.commonLabels" -}} helm.sh/chart: {{ include "datatracker.chart" . }} -{{ include "datatracker.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/part-of: {{ include "datatracker.name" . | default "datatracker" }} {{- end }} {{/* Selector labels */}} {{- define "datatracker.selectorLabels" -}} -app.kubernetes.io/name: {{ include "datatracker.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} diff --git a/helm/templates/deployments/beat.yaml b/helm/templates/deployments/beat.yaml index 6f7409fb9c..83347481cd 100644 --- a/helm/templates/deployments/beat.yaml +++ b/helm/templates/deployments/beat.yaml @@ -3,7 +3,8 @@ kind: Deployment metadata: name: {{ include "datatracker.beat.fullname" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} + app.kubernetes.io/name: {{ .Values.beat.name }} spec: {{- $podValues := .Values.beat }} replicas: {{ $podValues.replicaCount }} @@ -11,10 +12,12 @@ spec: selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} + app.kubernetes.io/name: {{ $podValues.name }} template: metadata: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} + app.kubernetes.io/name: {{ $podValues.name }} spec: {{- with $podValues.imagePullSecrets }} imagePullSecrets: diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml index f1f043abbd..1fb176009a 100644 --- a/helm/templates/deployments/celery.yaml +++ b/helm/templates/deployments/celery.yaml @@ -3,7 +3,8 @@ kind: Deployment metadata: name: {{ include "datatracker.celery.fullname" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} + app.kubernetes.io/name: {{ .Values.celery.name }} spec: {{- $podValues := .Values.celery }} replicas: {{ $podValues.replicaCount }} @@ -11,10 +12,12 @@ spec: selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} + app.kubernetes.io/name: {{ $podValues.name }} template: metadata: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} + app.kubernetes.io/name: {{ $podValues.name }} spec: {{- with $podValues.imagePullSecrets }} imagePullSecrets: diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml index dd4d8e4d82..0cc5ab6173 100644 --- a/helm/templates/deployments/datatracker.yaml +++ b/helm/templates/deployments/datatracker.yaml @@ -3,7 +3,8 @@ kind: Deployment metadata: name: {{ include "datatracker.datatracker.fullname" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} + app.kubernetes.io/name: {{ .Values.datatracker.name }} spec: {{- $podValues := .Values.datatracker }} replicas: {{ $podValues.replicaCount }} @@ -11,10 +12,12 @@ spec: selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} + app.kubernetes.io/name: {{ $podValues.name }} template: metadata: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} + app.kubernetes.io/name: {{ $podValues.name }} spec: {{- with $podValues.imagePullSecrets }} imagePullSecrets: diff --git a/helm/templates/deployments/memcached.yaml b/helm/templates/deployments/memcached.yaml index cbc7b3c35a..4bc35927d8 100644 --- a/helm/templates/deployments/memcached.yaml +++ b/helm/templates/deployments/memcached.yaml @@ -3,7 +3,8 @@ kind: Deployment metadata: name: {{ include "datatracker.memcached.fullname" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} + app.kubernetes.io/name: {{ .Values.memcached.name }} spec: {{- $podValues := .Values.memcached }} replicas: {{ $podValues.replicaCount }} @@ -11,10 +12,12 @@ spec: selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} + app.kubernetes.io/name: {{ $podValues.name }} template: metadata: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} + app.kubernetes.io/name: {{ $podValues.name }} spec: {{- with $podValues.imagePullSecrets }} imagePullSecrets: diff --git a/helm/templates/hpa.yaml b/helm/templates/hpa.yaml index 5dd889b509..646000bdcb 100644 --- a/helm/templates/hpa.yaml +++ b/helm/templates/hpa.yaml @@ -4,7 +4,7 @@ kind: HorizontalPodAutoscaler metadata: name: {{ include "datatracker.fullname" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} spec: scaleTargetRef: apiVersion: apps/v1 @@ -29,4 +29,4 @@ spec: type: Utilization averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml index c9405ef389..3b1a9febc5 100644 --- a/helm/templates/ingress.yaml +++ b/helm/templates/ingress.yaml @@ -17,7 +17,7 @@ kind: Ingress metadata: name: {{ $fullName }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.datatracker.ingress.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/templates/rbac/beat-serviceaccount.yaml b/helm/templates/rbac/beat-serviceaccount.yaml index bd955aed88..9834013756 100644 --- a/helm/templates/rbac/beat-serviceaccount.yaml +++ b/helm/templates/rbac/beat-serviceaccount.yaml @@ -4,7 +4,7 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.beat" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.serviceAccounts.beat.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/templates/rbac/celery-serviceaccount.yaml b/helm/templates/rbac/celery-serviceaccount.yaml index 105b979c60..e27673e385 100644 --- a/helm/templates/rbac/celery-serviceaccount.yaml +++ b/helm/templates/rbac/celery-serviceaccount.yaml @@ -4,7 +4,7 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.celery" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.serviceAccounts.celery.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/templates/rbac/datatracker-serviceaccount.yaml b/helm/templates/rbac/datatracker-serviceaccount.yaml index 7786443ada..d64f1523da 100644 --- a/helm/templates/rbac/datatracker-serviceaccount.yaml +++ b/helm/templates/rbac/datatracker-serviceaccount.yaml @@ -4,7 +4,7 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.datatracker" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.serviceAccounts.datatracker.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/templates/rbac/memcached-serviceaccount.yaml b/helm/templates/rbac/memcached-serviceaccount.yaml index 7f01615b1d..cc8d1a1212 100644 --- a/helm/templates/rbac/memcached-serviceaccount.yaml +++ b/helm/templates/rbac/memcached-serviceaccount.yaml @@ -4,7 +4,7 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.memcached" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.serviceAccounts.memcached.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/templates/rbac/rabbitmq-serviceaccount.yaml b/helm/templates/rbac/rabbitmq-serviceaccount.yaml index 015e098da3..58c89b2dcf 100644 --- a/helm/templates/rbac/rabbitmq-serviceaccount.yaml +++ b/helm/templates/rbac/rabbitmq-serviceaccount.yaml @@ -4,7 +4,7 @@ kind: ServiceAccount metadata: name: {{ include "datatracker.serviceAccountName.rabbitmq" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.serviceAccounts.rabbitmq.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/templates/services/datatracker.yaml b/helm/templates/services/datatracker.yaml index 2fbea5826f..201f5a3b4b 100644 --- a/helm/templates/services/datatracker.yaml +++ b/helm/templates/services/datatracker.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: name: {{include "datatracker.fullname" .}} - labels: {{- include "datatracker.labels" . | nindent 4 }} + labels: {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.datatracker.service.annotations }} annotations: {{- range $key, $value := . }} @@ -16,4 +16,6 @@ spec: targetPort: http protocol: TCP name: http - selector: {{- include "datatracker.selectorLabels" . | nindent 4}} \ No newline at end of file + selector: + {{- include "datatracker.selectorLabels" . | nindent 4}} + app.kubernetes.io/name: {{ .Values.datatracker.name }} diff --git a/helm/templates/services/memcached.yaml b/helm/templates/services/memcached.yaml index 1337571b04..572c382b5b 100644 --- a/helm/templates/services/memcached.yaml +++ b/helm/templates/services/memcached.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: name: memcached - labels: {{- include "datatracker.labels" . | nindent 4 }} + labels: {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.memcached.service.annotations }} annotations: {{- range $key, $value := . }} @@ -16,4 +16,6 @@ spec: targetPort: memcached protocol: TCP name: memcached - selector: {{- include "datatracker.selectorLabels" . | nindent 4}} + selector: + {{- include "datatracker.selectorLabels" . | nindent 4}} + app.kubernetes.io/name: {{ .Values.memcached.name }} diff --git a/helm/templates/services/rabbitmq.yaml b/helm/templates/services/rabbitmq.yaml index a23c4eaef5..e67cb31694 100644 --- a/helm/templates/services/rabbitmq.yaml +++ b/helm/templates/services/rabbitmq.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: name: rabbitmq - labels: {{- include "datatracker.labels" . | nindent 4 }} + labels: {{- include "datatracker.commonLabels" . | nindent 4 }} {{- with .Values.rabbitmq.service.annotations }} annotations: {{- range $key, $value := . }} @@ -17,4 +17,6 @@ spec: targetPort: amqp protocol: TCP name: amqp - selector: {{- include "datatracker.selectorLabels" . | nindent 4}} + selector: + {{- include "datatracker.selectorLabels" . | nindent 4}} + app.kubernetes.io/name: {{ .Values.rabbitmq.name }} diff --git a/helm/templates/statefulsets/rabbitmq.yaml b/helm/templates/statefulsets/rabbitmq.yaml index cad4f16502..dcffe17671 100644 --- a/helm/templates/statefulsets/rabbitmq.yaml +++ b/helm/templates/statefulsets/rabbitmq.yaml @@ -3,7 +3,8 @@ kind: StatefulSet metadata: name: {{ include "datatracker.rabbitmq.fullname" . }} labels: - {{- include "datatracker.labels" . | nindent 4 }} + {{- include "datatracker.commonLabels" . | nindent 4 }} + app.kubernetes.io/name: {{ .Values.rabbitmq.name }} spec: {{- $podValues := .Values.rabbitmq }} replicas: {{ $podValues.replicaCount }} @@ -11,10 +12,12 @@ spec: selector: matchLabels: {{- include "datatracker.selectorLabels" . | nindent 6 }} + app.kubernetes.io/name: {{ $podValues.name }} template: metadata: labels: {{- include "datatracker.selectorLabels" . | nindent 8 }} + app.kubernetes.io/name: {{ $podValues.name }} spec: {{- with $podValues.imagePullSecrets }} imagePullSecrets: diff --git a/helm/values.yaml b/helm/values.yaml index 27094f5464..6e79be5e85 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -218,10 +218,6 @@ celery: # runAsNonRoot: true # runAsUser: 1000 - service: - type: ClusterIP - port: 80 - serviceAccount: # Specifies whether a service account should be created create: true @@ -336,10 +332,6 @@ beat: # runAsNonRoot: true # runAsUser: 1000 - service: - type: ClusterIP - port: 80 - serviceAccount: # Specifies whether a service account should be created create: true From e99286f400da7e0f3d49f84423e44134fef7bf88 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 12 Apr 2024 11:57:58 -0300 Subject: [PATCH 26/70] ci: use values for PVClaims, STATIC_URL (#7317) * ci: Expose PV parameters as values * ci: complain about missing value in dev mode * ci: Allow override of STATIC_URL --- helm/settings_local.py | 9 +++++-- helm/templates/persistentvolumeclaims.yaml | 29 +++++++++++++++------- helm/values.yaml | 26 ++++++++++++++++++- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index eb960839b6..0a6f8a3b53 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -140,9 +140,9 @@ def _remove_whitespace_and_b64decode(s): "client_secret": _MEETECHO_CLIENT_SECRET, "request_timeout": 3.01, # python-requests doc recommend slightly > a multiple of 3 seconds } -elif SERVER_MODE == "production": +else: raise RuntimeError( - "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set in production" + "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set" ) _APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None) @@ -190,6 +190,11 @@ def _remove_whitespace_and_b64decode(s): GROUP_ALIASES_PATH = "/a/postfix/group-aliases" GROUP_VIRTUAL_PATH = "/a/postfix/group-virtual" +# Repoint STATIC_URL - hack to let staging access statics without hosting them itself +_STATIC_URL = os.environ.get("DATATRACKER_STATIC_URL", None) +if _STATIC_URL is not None: + STATIC_URL = _STATIC_URL + # Set these to the same as "production" in settings.py, whether production mode or not MEDIA_ROOT = "/a/www/www6s/lib/dt/media/" MEDIA_URL = "https://www.ietf.org/lib/dt/media/" diff --git a/helm/templates/persistentvolumeclaims.yaml b/helm/templates/persistentvolumeclaims.yaml index db636fb05e..54e75925a9 100644 --- a/helm/templates/persistentvolumeclaims.yaml +++ b/helm/templates/persistentvolumeclaims.yaml @@ -2,25 +2,36 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: "datatracker-shared-volume-claim" - # Note: This is the /a shared volume. The spec should be adjusted to ensure it always - # matches the correct EBS volume in production. For the moment, it just requests a - # large volume. spec: + {{- with .Values.persistentVolumes.datatrackerSharedVolume }} + storageClassName: {{ .storageClassName | quote }} + {{- if .volumeName }} + volumeName: {{ .volumeName | quote }} + {{- end }} accessModes: - - ReadWriteMany # or ReadWriteOnce and force datatracker/celery/beat to a single node + {{- range .accessModes }} + - {{ . | quote }} + {{- end }} resources: requests: - storage: 600Gi # adjust to something satisfied by the /a PersistentVolume + storage: {{ .storage | quote }} + {{- end }} --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: "rabbitmq-data-claim" spec: -# storageClassName: "local" -# volumeName: "" + {{- with .Values.persistentVolumes.rabbitmqDataVolume }} + storageClassName: {{ .storageClassName | quote }} + {{- if .volumeName }} + volumeName: {{ .volumeName | quote }} + {{- end }} accessModes: - - ReadWriteOnce + {{- range .accessModes }} + - {{ . | quote }} + {{- end }} resources: requests: - storage: 8Gi + storage: {{ .storage | quote }} + {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml index 6e79be5e85..7170e63d0b 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -561,6 +561,26 @@ memcached: affinity: {} +# ------------------------------------------------------------- +# PERSISTENT VOLUMES +# ------------------------------------------------------------- + +persistentVolumes: + datatrackerSharedVolume: + # Note: This is the /a shared volume + volumeName: "datatracker-shared-volume" + accessModes: + - "ReadWriteMany" # or ReadWriteOnce and force datatracker/celery/beat to a single node + storageClassName: "" # Empty string means do not use default storage class + storage: "600Gi" # actual PersistentVolume must be at least this big or the PVC will not bind + + rabbitmqDataVolume: + volumeName: "rabbitmq-data-volume" + accessModes: + - "ReadWriteOnce" + storageClassName: "" # Empty string means do not use default storage class + storage: "8Gi" # actual PersistentVolume must be at least this big or the PVC will not bind + # ------------------------------------------------------------- # COMMON # ------------------------------------------------------------- @@ -592,7 +612,11 @@ env: # DATATRACKER_DBPASS: "RkTkDPFnKpko" # secret DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret - + + # Set this to point testing / staging at the production statics server until we + # sort that out + DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" + # DATATRACKER_EMAIL_DEBUG: "true" # Outgoing email details From 39d2199a746a8867d8da429fcdcea9d76f3ae25b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 12 Apr 2024 17:17:25 -0300 Subject: [PATCH 27/70] ci: allow override of csrf_trusted_origins --- helm/settings_local.py | 17 +++++++++++++---- helm/values.yaml | 4 ++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index 0a6f8a3b53..18010a6867 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -14,6 +14,11 @@ def _remove_whitespace_and_b64decode(s): return b64decode("".join(s.split())) +def _multiline_to_list(s): + """Helper to split at newlines and conver to list""" + return [item.strip() for item in s.split("\n")] + + # Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env! SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development") @@ -78,7 +83,7 @@ def _remove_whitespace_and_b64decode(s): # DATATRACKER_ALLOWED_HOSTS env var is a comma-separated list of allowed hosts _allowed_hosts_str = os.environ.get("DATATRACKER_ALLOWED_HOSTS", None) if _allowed_hosts_str is not None: - ALLOWED_HOSTS = [h.strip() for h in _allowed_hosts_str.split("\n")] + ALLOWED_HOSTS = _multiline_to_list(_allowed_hosts_str) DATABASES = { "default": { @@ -92,9 +97,9 @@ def _remove_whitespace_and_b64decode(s): } # DATATRACKER_ADMINS is a newline-delimited list of addresses parseable by email.utils.parseaddr -_ADMINS = os.environ.get("DATATRACKER_ADMINS", None) -if _ADMINS is not None: - ADMINS = [parseaddr(admin) for admin in _ADMINS.split("\n")] +_admins_str = os.environ.get("DATATRACKER_ADMINS", None) +if _admins_str is not None: + ADMINS = [parseaddr(admin) for admin in _multiline_to_list(_admins_str)] else: raise RuntimeError("DATATRACKER_ADMINS must be set") @@ -250,3 +255,7 @@ def _remove_whitespace_and_b64decode(s): }, }, } + +_csrf_trusted_origins_str = os.environ.get("DATATRACKER_CSRF_TRUSTED_ORIGINS") +if _csrf_trusted_origins_str is not None: + CSRF_TRUSTED_ORIGINS = _multiline_to_list(_csrf_trusted_origins_str) diff --git a/helm/values.yaml b/helm/values.yaml index 7170e63d0b..5e11dfa709 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -657,3 +657,7 @@ env: CELERY_PASSWORD: "this-is-a-secret" # secret DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret + + # use this to override default - one entry per line + # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- + # https://datatracker.staging.ietf.org From 8a4fcf7024292660adf161277d61349b5061ee40 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 12 Apr 2024 17:17:46 -0300 Subject: [PATCH 28/70] ci: comment out DATATRACKER_STATIC_URL env --- helm/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/values.yaml b/helm/values.yaml index 5e11dfa709..ed568094c2 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -612,10 +612,10 @@ env: # DATATRACKER_DBPASS: "RkTkDPFnKpko" # secret DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret - + # Set this to point testing / staging at the production statics server until we # sort that out - DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" + # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" # DATATRACKER_EMAIL_DEBUG: "true" From 48f908020f2ef0d486db054c79ff22aea17a6b9e Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 12 Apr 2024 18:22:06 -0300 Subject: [PATCH 29/70] ci: collect statics when building image --- dev/build/Dockerfile | 2 ++ dev/build/datatracker-start.sh | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index f9ff3414e3..bc46637f3a 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -27,6 +27,8 @@ RUN yarn rebuild && \ yarn build && \ yarn legacy:build +RUN echo "yes" | /bin/bash dev/build/collectstatics.sh + RUN mkdir -p /a VOLUME [ "/a" ] diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index da355175ff..5995409ef6 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -6,9 +6,6 @@ echo "Running Datatracker checks..." echo "Running Datatracker migrations..." ./ietf/manage.py migrate --settings=settings_local -echo "Running collectstatic..." -./ietf/manage.py collectstatic --no-input - echo "Starting Datatracker..." gunicorn \ From ebaf6b56805334554a67bf6f1fecc0875381753f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 15 Apr 2024 11:08:36 -0300 Subject: [PATCH 30/70] ci: quotes around shell parameters --- bin/mrun.py | 30 ++++++++++++++++++ bin/test-crawl | 57 ++++++++++++++++++++++++++-------- dev/build/datatracker-start.sh | 8 ++--- 3 files changed, 78 insertions(+), 17 deletions(-) create mode 100644 bin/mrun.py diff --git a/bin/mrun.py b/bin/mrun.py new file mode 100644 index 0000000000..7f30d5702f --- /dev/null +++ b/bin/mrun.py @@ -0,0 +1,30 @@ +from os import fork, kill, waitpid +from signal import SIGTERM +from subprocess import run +from sys import exit +from time import sleep + +num_processes = 50 + +children = [] + +def be_a_child(): + run(["bin/test-crawl"]) + + +for _ in range(num_processes): + sleep(0.5) + pid = fork() + if pid == 0: + be_a_child() + exit() + children.append(pid) + +input("Enter to exit...") +for pid in children: + kill(pid, SIGTERM) + +for pid in children: + waitpid(pid, 0) + +print("Done") diff --git a/bin/test-crawl b/bin/test-crawl index 9b1d5280d5..feafe87b30 100755 --- a/bin/test-crawl +++ b/bin/test-crawl @@ -66,16 +66,16 @@ args = parser.parse_args() # Import Django, call setup() os.environ.setdefault("DJANGO_SETTINGS_MODULE", args.settings or "ietf.settings_testcrawl") -import django -import django.test -import django.core.checks -from django.conf import settings -from django.utils import timezone - -django.setup() +# import django +# import django.test +# import django.core.checks +# from django.conf import settings +# from django.utils import timezone +# +# django.setup() # This needs to come after we set up sys path to include the local django -import debug # pyflakes:ignore +# import debug # pyflakes:ignore # prevent memory from leaking when settings.DEBUG=True # from django.db import connection @@ -84,7 +84,7 @@ import debug # pyflakes:ignore # pass # connection.queries = DontSaveQueries() -from ietf.name.models import DocTypeName +# from ietf.name.models import DocTypeName from ietf.utils.html import unescape from ietf.utils.test_utils import unicontent from ietf.utils.test_runner import start_vnu_server, vnu_validate, vnu_fmt_message, vnu_filter_message @@ -257,7 +257,7 @@ def nowstr(): t = time.time_ns()/(10**9) if nowtime != t: nowtime = t - nowstrn = timezone.now().strftime('%H:%M:%S').encode() + nowstrn = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%H:%M:%S').encode() return nowstrn b_exthost = re.sub(b'https?', b'', args.exthost.encode()) if args.exthost else None @@ -408,7 +408,8 @@ parser = html5lib.HTMLParser(strict=True) # because they aren't under our control, such as uploaded group agendas. validated_urls = {'/meeting/nn/agenda/foo/': True, } -doc_types = [ t.slug for t in DocTypeName.objects.all() ] +# doc_types = [ t.slug for t in DocTypeName.objects.all() ] +doc_types = ["agenda", "bcp", "bluesheets", "bofreq", "charter", "draft", "rfc"] # partial list errors = 0 warnings = 0 @@ -416,7 +417,36 @@ count = 0 start_time = datetime.datetime.now() -client = django.test.Client(Accept='text/html,text/plain,application/json') +# client = django.test.Client(Accept='text/html,text/plain,application/json') +class ExtResponse: + charset = "utf8" + def __init__(self, requests_response): + self._requests_response = requests_response + + def __getattr__(self, item): + return getattr(self._requests_response, item) + + def __getitem__(self, item): + return self._requests_response.headers[item] + + +class ExtClient: + base_url = "http://localhost" + accept_header = "text/html,text/plain,application/json" + + def get(self, url, **kwargs): + kwargs.pop("secure", None) + headers = kwargs.setdefault("headers", {}) + headers["accept"] = self.accept_header + return ExtResponse( + requests.get( + urllib.parse.urljoin(self.base_url, url), + **kwargs, + ) + ) + + +client = ExtClient() logfile = None if args.logfile: @@ -450,7 +480,8 @@ if __name__ == "__main__": do_exit(1) # Run django system checks and checks from ietf.checks: - error_list = django.core.checks.run_checks() + # error_list = django.core.checks.run_checks() + error_list = [] silenced = [] for i in range(len(error_list)): if error_list[i].id in settings.SILENCED_SYSTEM_CHECKS: diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index 5995409ef6..7a83ff5334 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -9,11 +9,11 @@ echo "Running Datatracker migrations..." echo "Starting Datatracker..." gunicorn \ - --workers ${DATATRACKER_GUNICORN_WORKERS:-9} \ - --max-requests ${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768} \ - --timeout ${DATATRACKER_GUNICORN_TIMEOUT:-180} \ + --workers "${DATATRACKER_GUNICORN_WORKERS:-9}" \ + --max-requests "${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768}" \ + --timeout "${DATATRACKER_GUNICORN_TIMEOUT:-180}" \ --bind :8000 \ - --log-level ${DATATRACKER_GUNICORN_LOG_LEVEL:-info} \ + --log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \ ietf.wsgi:application # Leaving this here as a reminder to set up the env in the chart From c81b5f7088a37b039eb6f40d57e67cdb234545a0 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 15 Apr 2024 11:16:23 -0300 Subject: [PATCH 31/70] ci: nindent instead of indent --- helm/templates/configmap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 89983b5f07..e0261f0f49 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -4,7 +4,7 @@ metadata: name: django-configmap data: settings_local.py: |- - {{- .Files.Get "settings_local.py" | indent 4 }} + {{- .Files.Get "settings_local.py" | nindent 4 }} --- apiVersion: v1 kind: ConfigMap From ac5155122c61965f5eda57067ad227ae4e513533 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 15 Apr 2024 11:19:17 -0300 Subject: [PATCH 32/70] ci: label PVCs --- helm/templates/persistentvolumeclaims.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/helm/templates/persistentvolumeclaims.yaml b/helm/templates/persistentvolumeclaims.yaml index 54e75925a9..412de9d02d 100644 --- a/helm/templates/persistentvolumeclaims.yaml +++ b/helm/templates/persistentvolumeclaims.yaml @@ -2,6 +2,9 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: "datatracker-shared-volume-claim" + labels: + {{- include "datatracker.commonLabels" . | nindent 4 }} + app.kubernetes.io/name: {{ .Values.datatracker.name }} spec: {{- with .Values.persistentVolumes.datatrackerSharedVolume }} storageClassName: {{ .storageClassName | quote }} @@ -21,6 +24,9 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: "rabbitmq-data-claim" + labels: + {{- include "datatracker.commonLabels" . | nindent 4 }} + app.kubernetes.io/name: {{ .Values.rabbitmq.name }} spec: {{- with .Values.persistentVolumes.rabbitmqDataVolume }} storageClassName: {{ .storageClassName | quote }} From 64441be3301f428d133f30ee684ed3ba981a6f9e Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 15 Apr 2024 13:23:54 -0300 Subject: [PATCH 33/70] chore: Remove accidentally committed file --- bin/mrun.py | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 bin/mrun.py diff --git a/bin/mrun.py b/bin/mrun.py deleted file mode 100644 index 7f30d5702f..0000000000 --- a/bin/mrun.py +++ /dev/null @@ -1,30 +0,0 @@ -from os import fork, kill, waitpid -from signal import SIGTERM -from subprocess import run -from sys import exit -from time import sleep - -num_processes = 50 - -children = [] - -def be_a_child(): - run(["bin/test-crawl"]) - - -for _ in range(num_processes): - sleep(0.5) - pid = fork() - if pid == 0: - be_a_child() - exit() - children.append(pid) - -input("Enter to exit...") -for pid in children: - kill(pid, SIGTERM) - -for pid in children: - waitpid(pid, 0) - -print("Done") From ffcf74bd03ac1505281094b2c93c0e9679b55778 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 15 Apr 2024 15:28:10 -0300 Subject: [PATCH 34/70] chore: Remove accidental commits... (#7329) --- bin/test-crawl | 57 ++++++++++++-------------------------------------- 1 file changed, 13 insertions(+), 44 deletions(-) diff --git a/bin/test-crawl b/bin/test-crawl index feafe87b30..9b1d5280d5 100755 --- a/bin/test-crawl +++ b/bin/test-crawl @@ -66,16 +66,16 @@ args = parser.parse_args() # Import Django, call setup() os.environ.setdefault("DJANGO_SETTINGS_MODULE", args.settings or "ietf.settings_testcrawl") -# import django -# import django.test -# import django.core.checks -# from django.conf import settings -# from django.utils import timezone -# -# django.setup() +import django +import django.test +import django.core.checks +from django.conf import settings +from django.utils import timezone + +django.setup() # This needs to come after we set up sys path to include the local django -# import debug # pyflakes:ignore +import debug # pyflakes:ignore # prevent memory from leaking when settings.DEBUG=True # from django.db import connection @@ -84,7 +84,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", args.settings or "ietf.settings_ # pass # connection.queries = DontSaveQueries() -# from ietf.name.models import DocTypeName +from ietf.name.models import DocTypeName from ietf.utils.html import unescape from ietf.utils.test_utils import unicontent from ietf.utils.test_runner import start_vnu_server, vnu_validate, vnu_fmt_message, vnu_filter_message @@ -257,7 +257,7 @@ def nowstr(): t = time.time_ns()/(10**9) if nowtime != t: nowtime = t - nowstrn = datetime.datetime.now(tz=datetime.timezone.utc).strftime('%H:%M:%S').encode() + nowstrn = timezone.now().strftime('%H:%M:%S').encode() return nowstrn b_exthost = re.sub(b'https?', b'', args.exthost.encode()) if args.exthost else None @@ -408,8 +408,7 @@ parser = html5lib.HTMLParser(strict=True) # because they aren't under our control, such as uploaded group agendas. validated_urls = {'/meeting/nn/agenda/foo/': True, } -# doc_types = [ t.slug for t in DocTypeName.objects.all() ] -doc_types = ["agenda", "bcp", "bluesheets", "bofreq", "charter", "draft", "rfc"] # partial list +doc_types = [ t.slug for t in DocTypeName.objects.all() ] errors = 0 warnings = 0 @@ -417,36 +416,7 @@ count = 0 start_time = datetime.datetime.now() -# client = django.test.Client(Accept='text/html,text/plain,application/json') -class ExtResponse: - charset = "utf8" - def __init__(self, requests_response): - self._requests_response = requests_response - - def __getattr__(self, item): - return getattr(self._requests_response, item) - - def __getitem__(self, item): - return self._requests_response.headers[item] - - -class ExtClient: - base_url = "http://localhost" - accept_header = "text/html,text/plain,application/json" - - def get(self, url, **kwargs): - kwargs.pop("secure", None) - headers = kwargs.setdefault("headers", {}) - headers["accept"] = self.accept_header - return ExtResponse( - requests.get( - urllib.parse.urljoin(self.base_url, url), - **kwargs, - ) - ) - - -client = ExtClient() +client = django.test.Client(Accept='text/html,text/plain,application/json') logfile = None if args.logfile: @@ -480,8 +450,7 @@ if __name__ == "__main__": do_exit(1) # Run django system checks and checks from ietf.checks: - # error_list = django.core.checks.run_checks() - error_list = [] + error_list = django.core.checks.run_checks() silenced = [] for i in range(len(error_list)): if error_list[i].id in settings.SILENCED_SYSTEM_CHECKS: From 90dc303293411812d96068e22c04fedda1877dde Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 16 Apr 2024 17:03:50 -0300 Subject: [PATCH 35/70] ci: Add scoutapm sidecar container (#7330) * ci: Add scoutapm sidecar container * ci: Configure ScoutAPM via helm --- helm/settings_local.py | 8 +++----- helm/templates/deployments/datatracker.yaml | 7 +++++++ helm/values.yaml | 10 ++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index 18010a6867..af27e17f7a 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -176,14 +176,12 @@ def _multiline_to_list(s): DEV_PRE_APPS = ["scout_apm.django", ] SCOUT_MONITOR = True SCOUT_KEY = _SCOUT_KEY - SCOUT_NAME = "Datatracker" + SCOUT_NAME = os.environ.get("DATATRACKER_SCOUT_NAME", "Datatracker") SCOUT_ERRORS_ENABLED = True SCOUT_SHUTDOWN_MESSAGE_ENABLED = False - SCOUT_CORE_AGENT_DIR = "/a/core-agent/1.4.0" - SCOUT_CORE_AGENT_FULL_NAME = "scout_apm_core-v1.4.0-x86_64-unknown-linux-musl" SCOUT_CORE_AGENT_SOCKET_PATH = "tcp://{host}:{port}".format( - host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "scout"), - port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "16590"), + host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "localhost"), + port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "6590"), ), SCOUT_CORE_AGENT_DOWNLOAD = False SCOUT_CORE_AGENT_LAUNCH = False diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml index 0cc5ab6173..d625872973 100644 --- a/helm/templates/deployments/datatracker.yaml +++ b/helm/templates/deployments/datatracker.yaml @@ -57,6 +57,13 @@ spec: {{- toYaml $podValues.startupProbe | nindent 12 }} resources: {{- toYaml $podValues.resources | nindent 12 }} + {{- if $podValues.scoutapm }} + initContainers: + - name: "scoutapm" + image: "{{ $podValues.scoutapm.image.repository }}:{{ default "latest" $podValues.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.scoutapm.image.imagePullPolicy }} + restartPolicy: "Always" + {{- end }} {{- with $podValues.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/helm/values.yaml b/helm/values.yaml index ed568094c2..564f8c0258 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -166,6 +166,12 @@ datatracker: nodeSelector: {} affinity: {} + + # Set this to enable a Scout APM Core Agent sidecar + scoutapm: + image: + repository: "scoutapp/scoutapm" + tag: "version-1.4.0" # ------------------------------------------------------------- # CELERY @@ -661,3 +667,7 @@ env: # use this to override default - one entry per line # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- # https://datatracker.staging.ietf.org + + # Scout configuration + DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" + DATATRACKER_SCOUT_NAME: "StagingDatatracker" From d14cbd10e9578d63a6a3e610b8f5b437d4d57fcb Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 18 Apr 2024 10:42:53 -0300 Subject: [PATCH 36/70] ci: refactor scout sidecar (#7338) * ci: remove stray comma in settings_local.py * ci: move scout sidecar to containers initContainers sidecars not supported until kubernetes 1.29 --- helm/settings_local.py | 2 +- helm/templates/deployments/datatracker.yaml | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index af27e17f7a..2840ca6b98 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -182,7 +182,7 @@ def _multiline_to_list(s): SCOUT_CORE_AGENT_SOCKET_PATH = "tcp://{host}:{port}".format( host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "localhost"), port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "6590"), - ), + ) SCOUT_CORE_AGENT_DOWNLOAD = False SCOUT_CORE_AGENT_LAUNCH = False SCOUT_REVISION_SHA = __release_hash__[:7] diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml index d625872973..31f15a5b74 100644 --- a/helm/templates/deployments/datatracker.yaml +++ b/helm/templates/deployments/datatracker.yaml @@ -27,6 +27,17 @@ spec: securityContext: {{- toYaml $podValues.podSecurityContext | nindent 8 }} containers: + {{- if $podValues.scoutapm }} + - name: "scoutapm" + image: "{{ $podValues.scoutapm.image.repository }}:{{ default "latest" $podValues.scoutapm.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" $podValues.scoutapm.image.imagePullPolicy }} + livenessProbe: + exec: + command: + - "sh" + - "-c" + - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + {{- end }} - name: {{ .Chart.Name }} securityContext: {{- toYaml $podValues.securityContext | nindent 12 }} @@ -57,13 +68,6 @@ spec: {{- toYaml $podValues.startupProbe | nindent 12 }} resources: {{- toYaml $podValues.resources | nindent 12 }} - {{- if $podValues.scoutapm }} - initContainers: - - name: "scoutapm" - image: "{{ $podValues.scoutapm.image.repository }}:{{ default "latest" $podValues.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.scoutapm.image.imagePullPolicy }} - restartPolicy: "Always" - {{- end }} {{- with $podValues.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} From 6ccde89a68ac5b5d4325940ff39536c6457dd6c1 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Apr 2024 14:21:20 -0300 Subject: [PATCH 37/70] ci: do not re-run yarn / collectstatics (#7353) --- dev/build/Dockerfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index bc46637f3a..267dbc587c 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -23,12 +23,6 @@ RUN chmod +x start.sh && \ chmod +x docker/scripts/app-create-dirs.sh && \ sh ./docker/scripts/app-create-dirs.sh -RUN yarn rebuild && \ - yarn build && \ - yarn legacy:build - -RUN echo "yes" | /bin/bash dev/build/collectstatics.sh - RUN mkdir -p /a VOLUME [ "/a" ] From 26f2306316b2b626c5b674b37910cf60dac71572 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Apr 2024 14:59:45 -0300 Subject: [PATCH 38/70] ci: add scout container to celery pod (#7354) * ci: Add scout container to celery pod * ci: Refactor scoutapm settings in values.yaml --- helm/templates/deployments/celery.yaml | 11 +++++++++++ helm/templates/deployments/datatracker.yaml | 6 +++--- helm/values.yaml | 15 +++++++++------ 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml index 1fb176009a..d697fcdc9d 100644 --- a/helm/templates/deployments/celery.yaml +++ b/helm/templates/deployments/celery.yaml @@ -27,6 +27,17 @@ spec: securityContext: {{- toYaml $podValues.podSecurityContext | nindent 8 }} containers: + {{- if .Values.scoutapm }} + - name: "scoutapm" + image: "{{ .Values.scoutapm.image.repository }}:{{ default "latest" .Values.scoutapm.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" .Values.scoutapm.image.imagePullPolicy }} + livenessProbe: + exec: + command: + - "sh" + - "-c" + - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + {{- end }} - name: {{ .Chart.Name }} securityContext: {{- toYaml $podValues.securityContext | nindent 12 }} diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml index 31f15a5b74..85e7556e93 100644 --- a/helm/templates/deployments/datatracker.yaml +++ b/helm/templates/deployments/datatracker.yaml @@ -27,10 +27,10 @@ spec: securityContext: {{- toYaml $podValues.podSecurityContext | nindent 8 }} containers: - {{- if $podValues.scoutapm }} + {{- if .Values.scoutapm }} - name: "scoutapm" - image: "{{ $podValues.scoutapm.image.repository }}:{{ default "latest" $podValues.scoutapm.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.scoutapm.image.imagePullPolicy }} + image: "{{ .Values.scoutapm.image.repository }}:{{ default "latest" .Values.scoutapm.image.tag }}" + imagePullPolicy: {{ default "IfNotPresent" .Values.scoutapm.image.imagePullPolicy }} livenessProbe: exec: command: diff --git a/helm/values.yaml b/helm/values.yaml index 564f8c0258..1a47d7f639 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -166,12 +166,6 @@ datatracker: nodeSelector: {} affinity: {} - - # Set this to enable a Scout APM Core Agent sidecar - scoutapm: - image: - repository: "scoutapp/scoutapm" - tag: "version-1.4.0" # ------------------------------------------------------------- # CELERY @@ -567,6 +561,15 @@ memcached: affinity: {} +# ------------------------------------------------------------- +# SCOUT APM SETTINGS +# ------------------------------------------------------------- +# Set this to enable a Scout APM Core Agent sidecar +scoutapm: + image: + repository: "scoutapp/scoutapm" + tag: "version-1.4.0" + # ------------------------------------------------------------- # PERSISTENT VOLUMES # ------------------------------------------------------------- From b50e60b05fb5e3e58040898ef64b8b65ad02ee63 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Apr 2024 17:03:22 -0300 Subject: [PATCH 39/70] ci: default to static.ietf.org in helm chart --- helm/settings_local.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/settings_local.py b/helm/settings_local.py index 2840ca6b98..53609afa79 100644 --- a/helm/settings_local.py +++ b/helm/settings_local.py @@ -193,10 +193,10 @@ def _multiline_to_list(s): GROUP_ALIASES_PATH = "/a/postfix/group-aliases" GROUP_VIRTUAL_PATH = "/a/postfix/group-virtual" -# Repoint STATIC_URL - hack to let staging access statics without hosting them itself -_STATIC_URL = os.environ.get("DATATRACKER_STATIC_URL", None) -if _STATIC_URL is not None: - STATIC_URL = _STATIC_URL +STATIC_URL = os.environ.get("DATATRACKER_STATIC_URL", None) +if STATIC_URL is None: + from ietf import __version__ + STATIC_URL = f"https://static.ietf.org/dt/{__version__}/" # Set these to the same as "production" in settings.py, whether production mode or not MEDIA_ROOT = "/a/www/www6s/lib/dt/media/" From 9379bbad7d163bccca86bf35454d8ceea9767e60 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Apr 2024 18:16:19 -0300 Subject: [PATCH 40/70] ci: remove reminder comment --- dev/build/datatracker-start.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index 7a83ff5334..390b46af86 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -15,7 +15,3 @@ gunicorn \ --bind :8000 \ --log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \ ietf.wsgi:application - - # Leaving this here as a reminder to set up the env in the chart - # Remove this once that's complete. - #--env SCOUT_NAME=Datatracker \ From 2c9c61d8780f8f7d7c4e25fd8a168cead0e3ef91 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 23 Apr 2024 22:34:24 -0300 Subject: [PATCH 41/70] ci: more memcached mem; set securityContext (#7356) --- helm/templates/deployments/memcached.yaml | 1 + helm/values.yaml | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/helm/templates/deployments/memcached.yaml b/helm/templates/deployments/memcached.yaml index 4bc35927d8..7bf21c1d79 100644 --- a/helm/templates/deployments/memcached.yaml +++ b/helm/templates/deployments/memcached.yaml @@ -31,6 +31,7 @@ spec: securityContext: {{- toYaml $podValues.securityContext | nindent 12 }} image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" + args: ["-m", "$(MEMCACHED_MEM_LIMIT)"] imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} env: {{- if .Values.env }} diff --git a/helm/values.yaml b/helm/values.yaml index 1a47d7f639..dae391aaaa 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -515,9 +515,6 @@ memcached: podAnnotations: {} podLabels: {} - podSecurityContext: {} - # fsGroup: 2000 - replicaCount: 1 resources: {} @@ -532,13 +529,17 @@ memcached: # cpu: 100m # memory: 128Mi - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 + podSecurityContext: + runAsNonRoot: true + runAsUser: 11211 + runAsGroup: 11211 + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true service: type: ClusterIP @@ -674,3 +675,5 @@ env: # Scout configuration DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" DATATRACKER_SCOUT_NAME: "StagingDatatracker" + + MEMCACHED_MEM_LIMIT: "1024" From 30a4a5a77b9322c1de42697a6eabb51179adcb52 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 24 Apr 2024 20:21:22 -0300 Subject: [PATCH 42/70] ci: run rabbitmq as non-root (#7362) * ci: securityContext for rabbitmq * ci: logs from rabbitmq as json to console * ci: tmp volume for rabbitmq Needed since rootfs is now read-only * ci: fix permissions on /var/lib/rabbitmq vol Rabbitmq needs to be able to write to the fs at /var/lib/rabbitmq. It may be possible to get rid of the initContainer and use fsGroup in the pod securityContext to manage this, but that does not work for the hostVolume mounts I use for dev. The solution here moves the actual mount to the rabbitmq/ directory in the rabbitmq-data volume and uses an initContainer to set the permissions on that. That should work for any volume type. --- helm/templates/configmap.yaml | 6 ++++ helm/templates/statefulsets/rabbitmq.yaml | 14 ++++++++ helm/values.yaml | 39 +++++++++++++---------- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index e0261f0f49..06cda4a04b 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -61,3 +61,9 @@ data: # absolute number because relative will be proprtional to the full machine # memory. vm_memory_high_watermark.absolute = 1600MB + + # Logging + log.file = false + log.console = true + log.console.level = info + log.console.formatter = json diff --git a/helm/templates/statefulsets/rabbitmq.yaml b/helm/templates/statefulsets/rabbitmq.yaml index dcffe17671..6cb2cc8367 100644 --- a/helm/templates/statefulsets/rabbitmq.yaml +++ b/helm/templates/statefulsets/rabbitmq.yaml @@ -26,6 +26,20 @@ spec: serviceAccountName: {{ include "datatracker.serviceAccountName.rabbitmq" . }} securityContext: {{- toYaml $podValues.podSecurityContext | nindent 8 }} + initContainers: + - name: init-rabbitmq + image: busybox:stable + command: + - "sh" + - "-c" + - "mkdir -p -m700 /mnt/rabbitmq && chown 100:101 /mnt/rabbitmq" + securityContext: + runAsNonRoot: false + runAsUser: 0 + readOnlyRootFilesystem: true + volumeMounts: + - name: "rabbitmq-data" + mountPath: "/mnt" containers: - name: {{ .Chart.Name }} securityContext: diff --git a/helm/values.yaml b/helm/values.yaml index dae391aaaa..366cea3d46 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -405,7 +405,6 @@ rabbitmq: repository: "ghcr.io/ietf-tools/datatracker-mq" pullPolicy: IfNotPresent tag: "3.12-alpine" - imagePullSecrets: [] nameOverride: "" fullnameOverride: "" @@ -419,9 +418,6 @@ rabbitmq: podAnnotations: {} podLabels: {} - podSecurityContext: {} - # fsGroup: 2000 - replicaCount: 1 resources: {} @@ -436,13 +432,18 @@ rabbitmq: # cpu: 100m # memory: 128Mi - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 + podSecurityContext: + runAsNonRoot: true + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + # rabbitmq image sets up uid/gid 100/101 + runAsUser: 100 + runAsGroup: 101 service: type: ClusterIP @@ -476,6 +477,9 @@ rabbitmq: - name: "rabbitmq-config" configMap: name: "rabbitmq-configmap" + - name: "rabbitmq-tmp" + emptyDir: + sizeLimit: 50Mi # - name: foo # secret: # secretName: mysecret @@ -484,12 +488,12 @@ rabbitmq: # Additional volumeMounts on the output Deployment definition. volumeMounts: - name: "rabbitmq-data" - mountPath: "/var/lib/rabbitmq/mnesia" + mountPath: "/var/lib/rabbitmq" + subPath: "rabbitmq" - name: "rabbitmq-config" mountPath: "/etc/rabbitmq" - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true + - name: "rabbitmq-tmp" + mountPath: "/tmp" tolerations: [] @@ -531,8 +535,6 @@ memcached: podSecurityContext: runAsNonRoot: true - runAsUser: 11211 - runAsGroup: 11211 securityContext: allowPrivilegeEscalation: false @@ -540,6 +542,9 @@ memcached: drop: - ALL readOnlyRootFilesystem: true + # memcached image sets up uid/gid 11211 + runAsUser: 11211 + runAsGroup: 11211 service: type: ClusterIP From c8ee43da95b5e784eeac3f3b2cd1a1d3a45ce445 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 25 Apr 2024 17:24:26 -0300 Subject: [PATCH 43/70] ci: run datatracker pod as non-root user (#7366) * feat: patch_libraries management command * ci: Patch libraries in docker img build * ci: non-root datatracker user * ci: securityContext for datatracker pod --- dev/build/Dockerfile | 6 ++- helm/values.yaml | 45 +++++++------------ .../management/commands/patch_libraries.py | 32 +++++++++++++ 3 files changed, 53 insertions(+), 30 deletions(-) create mode 100644 ietf/utils/management/commands/patch_libraries.py diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 267dbc587c..51d2d3799e 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -3,6 +3,9 @@ LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive +RUN groupadd -g 1000 datatracker && \ + useradd -c "Datatracker User" -u 1000 -g datatracker -m -s /bin/false datatracker + RUN apt-get purge -y imagemagick imagemagick-6-common # Install libreoffice (needed via PPT2PDF_COMMAND) @@ -15,7 +18,8 @@ COPY ./dev/build/start.sh ./start.sh COPY ./dev/build/datatracker-start.sh ./datatracker-start.sh COPY ./dev/build/celery-start.sh ./celery-start.sh -RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt +RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt && \ + ietf/manage.py patch_libraries RUN chmod +x start.sh && \ chmod +x datatracker-start.sh && \ diff --git a/helm/values.yaml b/helm/values.yaml index 366cea3d46..28f9762848 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -67,9 +67,6 @@ datatracker: podAnnotations: {} podLabels: {} - podSecurityContext: {} - # fsGroup: 2000 - #readinessProbe: # httpGet: # # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint @@ -90,13 +87,17 @@ datatracker: # cpu: 100m # memory: 128Mi - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 + podSecurityContext: + runAsNonRoot: true + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 service: type: ClusterIP @@ -132,17 +133,9 @@ datatracker: - name: datatracker-shared-volume persistentVolumeClaim: claimName: "datatracker-shared-volume-claim" -# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume -# - name: cache-volume -# emptyDir: -# sizeLimit: 1Gi -# - name: staging-volume -# emptyDir: -# sizeLimit: 1Gi - # - name: foo - # secret: - # secretName: mysecret - # optional: false + - name: datatracker-tmp + emptyDir: + sizeLimit: "2Gi" # Additional volumeMounts on the output Deployment definition. volumeMounts: @@ -152,14 +145,8 @@ datatracker: readOnly: true - name: datatracker-shared-volume mountPath: /a -# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume -# - name: cache-volume -# mountPath: "/a/cache" -# - name: staging-volume -# mountPath: "/test/staging" - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true + - name: datatracker-tmp + mountPath: /tmp tolerations: [] diff --git a/ietf/utils/management/commands/patch_libraries.py b/ietf/utils/management/commands/patch_libraries.py new file mode 100644 index 0000000000..2ada67a280 --- /dev/null +++ b/ietf/utils/management/commands/patch_libraries.py @@ -0,0 +1,32 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +import django +import os + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from pathlib import Path + +from ietf.utils import patch + + +class Command(BaseCommand): + """Apply IETF patches to libraries""" + requires_system_checks = tuple() + + def handle(self, *args, **options): + library_path = Path(django.__file__).parent.parent + top_dir = Path(settings.BASE_DIR).parent + + # All patches in settings.CHECKS_LIBRARY_PATCHES_TO_APPLY must have a + # relative file path starting from the site-packages dir, e.g. + # 'django/db/models/fields/__init__.py' + for patch_file in settings.CHECKS_LIBRARY_PATCHES_TO_APPLY: + patch_set = patch.fromfile(top_dir / Path(patch_file)) + if not patch_set: + raise CommandError(f"Could not parse patch file '{patch_file}'") + if not patch_set.apply(root=bytes(library_path)): + raise CommandError(f"Could not apply the patch from '{patch_file}'") + if patch_set.already_patched: + self.stdout.write(f"Patch from '{patch_file}' was already applied") + else: + self.stdout.write(f"Applied the patch from '{patch_file}'") From 70c32254a94a444f0c405515b2b5d2af3c6fc34b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 25 Apr 2024 17:44:38 -0300 Subject: [PATCH 44/70] ci: non-root user for scout containers --- helm/templates/deployments/celery.yaml | 4 ++++ helm/templates/deployments/datatracker.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml index d697fcdc9d..5c824aa2d0 100644 --- a/helm/templates/deployments/celery.yaml +++ b/helm/templates/deployments/celery.yaml @@ -37,6 +37,10 @@ spec: - "sh" - "-c" - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + securityContext: + readOnlyRootFilesystem: {{ default true .Values.scoutapm.readOnlyRootFilesystem }} + runAsUser: {{ default 65534 .Values.scoutapm.runAsUser }} # "nobody" user by default + runAsGroup: {{ default 65534 .Values.scoutapm.runAsGroup }} # "nogroup" group by default {{- end }} - name: {{ .Chart.Name }} securityContext: diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml index 85e7556e93..acd54af97a 100644 --- a/helm/templates/deployments/datatracker.yaml +++ b/helm/templates/deployments/datatracker.yaml @@ -37,6 +37,10 @@ spec: - "sh" - "-c" - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + securityContext: + readOnlyRootFilesystem: {{ default true .Values.scoutapm.readOnlyRootFilesystem }} + runAsUser: {{ default 65534 .Values.scoutapm.runAsUser }} # "nobody" user by default + runAsGroup: {{ default 65534 .Values.scoutapm.runAsGroup }} # "nogroup" group by default {{- end }} - name: {{ .Chart.Name }} securityContext: From 4e2b9ce7a6144ef93aeebe48b88b1abdf83bcd09 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 25 Apr 2024 18:23:26 -0300 Subject: [PATCH 45/70] ci: Non-root for celery containers (#7368) --- helm/values.yaml | 90 +++++++++++++++++------------------------------- 1 file changed, 32 insertions(+), 58 deletions(-) diff --git a/helm/values.yaml b/helm/values.yaml index 28f9762848..aa5bbaf30c 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -180,9 +180,6 @@ celery: podAnnotations: {} podLabels: {} - podSecurityContext: {} - # fsGroup: 2000 - replicaCount: 1 resources: {} @@ -197,13 +194,17 @@ celery: # cpu: 100m # memory: 128Mi - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 + podSecurityContext: + runAsNonRoot: true + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 serviceAccount: # Specifies whether a service account should be created @@ -233,17 +234,9 @@ celery: - name: datatracker-shared-volume persistentVolumeClaim: claimName: "datatracker-shared-volume-claim" -# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume -# - name: cache-volume -# emptyDir: -# sizeLimit: 1Gi -# - name: staging-volume -# emptyDir: -# sizeLimit: 1Gi - # - name: foo - # secret: - # secretName: mysecret - # optional: false + - name: celery-tmp + emptyDir: + sizeLimit: "2Gi" # Additional volumeMounts on the output Deployment definition. volumeMounts: @@ -253,14 +246,8 @@ celery: readOnly: true - name: datatracker-shared-volume mountPath: /a -# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume -# - name: cache-volume -# mountPath: "/a/cache" -# - name: staging-volume -# mountPath: "/test/staging" - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true + - name: celery-tmp + mountPath: /tmp tolerations: [] @@ -294,9 +281,6 @@ beat: podAnnotations: {} podLabels: {} - podSecurityContext: {} - # fsGroup: 2000 - replicaCount: 1 resources: {} @@ -311,13 +295,17 @@ beat: # cpu: 100m # memory: 128Mi - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 + podSecurityContext: + runAsNonRoot: true + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 serviceAccount: # Specifies whether a service account should be created @@ -347,17 +335,9 @@ beat: - name: datatracker-shared-volume persistentVolumeClaim: claimName: "datatracker-shared-volume-claim" -# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume -# - name: cache-volume -# emptyDir: -# sizeLimit: 1Gi -# - name: staging-volume -# emptyDir: -# sizeLimit: 1Gi - # - name: foo - # secret: - # secretName: mysecret - # optional: false + - name: beat-tmp + emptyDir: + sizeLimit: "2Gi" # Additional volumeMounts on the output Deployment definition. volumeMounts: @@ -367,14 +347,8 @@ beat: readOnly: true - name: datatracker-shared-volume mountPath: /a -# cache-volume and staging-volume are a hack to create paths for debugging without a /a volume -# - name: cache-volume -# mountPath: "/a/cache" -# - name: staging-volume -# mountPath: "/test/staging" - # - name: foo - # mountPath: "/etc/foo" - # readOnly: true + - name: beat-tmp + mountPath: /tmp tolerations: [] From 8f87573144276980af42a9b391a4f1984748d300 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 25 Apr 2024 20:05:50 -0300 Subject: [PATCH 46/70] ci: Fill in settings_local for docker --- dev/build/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 51d2d3799e..d5578be3df 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -19,7 +19,9 @@ COPY ./dev/build/datatracker-start.sh ./datatracker-start.sh COPY ./dev/build/celery-start.sh ./celery-start.sh RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt && \ - ietf/manage.py patch_libraries + echo '# empty' > ietf/settings_local.py && \ + ietf/manage.py patch_libraries && \ + rm -f ietf/settings_local.py RUN chmod +x start.sh && \ chmod +x datatracker-start.sh && \ From d2623de615b81754fc125ec688e077a06c633bb1 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 26 Apr 2024 17:00:51 -0300 Subject: [PATCH 47/70] feat: config celery logging via Django (#7371) * feat: config celery logger via Django * feat: Disable celery's logging config --- ietf/celeryapp.py | 10 ++++++++-- ietf/settings.py | 14 +++++++++----- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ietf/celeryapp.py b/ietf/celeryapp.py index b36f134636..08089d509e 100644 --- a/ietf/celeryapp.py +++ b/ietf/celeryapp.py @@ -1,14 +1,20 @@ import os import scout_apm.celery -from celery import Celery +import celery from scout_apm.api import Config +# Disable celery's internal logging configuration, we set it up via Django +@celery.signals.setup_logging.connect +def on_setup_logging(**kwargs): + pass + + # Set the default Django settings module for the 'celery' program os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ietf.settings') -app = Celery('ietf') +app = celery.Celery('ietf') # Using a string here means the worker doesn't have to serialize # the configuration object to child processes. diff --git a/ietf/settings.py b/ietf/settings.py index 990d8bf056..9f12e0b4dc 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -252,17 +252,21 @@ def skip_unreadable_post(record): 'level': 'INFO', }, 'django.security': { - 'handlers': ['console', ], + 'handlers': ['console', ], 'level': 'INFO', }, - 'oidc_provider': { - 'handlers': ['debug_console', ], - 'level': 'DEBUG', - }, + 'oidc_provider': { + 'handlers': ['debug_console', ], + 'level': 'DEBUG', + }, 'datatracker': { 'handlers': ['console', ], 'level': 'INFO', }, + 'celery': { + 'handlers': ['console'], + 'level': 'INFO', + } }, # # No logger filters From e35b46eed81e71306a67e34331e566e8ec7cd791 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 26 Apr 2024 17:51:58 -0300 Subject: [PATCH 48/70] ci: fix celery scout env var names (#7373) --- ietf/celeryapp.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ietf/celeryapp.py b/ietf/celeryapp.py index 08089d509e..fda89c30be 100644 --- a/ietf/celeryapp.py +++ b/ietf/celeryapp.py @@ -23,10 +23,13 @@ def on_setup_logging(**kwargs): app.config_from_object('django.conf:settings', namespace='CELERY') # Turn on Scout APM celery instrumentation if configured in the environment -scout_key = os.environ.get("SCOUT_KEY", "") -scout_name = os.environ.get("SCOUT_NAME", "") -scout_core_agent_socket_path = os.environ.get("SCOUT_CORE_AGENT_SOCKET_PATH", "tcp://scoutapm:6590") -if scout_key and scout_name: +scout_key = os.environ.get("DATATRACKER_SCOUT_KEY", None) +if scout_key is not None: + scout_name = os.environ.get("DATATRACKER_SCOUT_NAME", "Datatracker") + scout_core_agent_socket_path = "tcp://{host}:{port}".format( + host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "localhost"), + port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "6590"), + ) Config.set( key=scout_key, name=scout_name, From 3ea70f2ceb345f60053ae7ab05784350adae4940 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 4 May 2024 01:13:19 -0400 Subject: [PATCH 49/70] refactor: helm to kustomize (wip) --- k8s/datatracker.yaml | 190 ++++++++++++++++++++++++++++++ k8s/kustomization.yaml | 10 ++ k8s/memcached.yaml | 14 +++ k8s/rabbitmq.yaml | 183 +++++++++++++++++++++++++++++ k8s/settings_local.py | 259 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 656 insertions(+) create mode 100644 k8s/datatracker.yaml create mode 100644 k8s/kustomization.yaml create mode 100644 k8s/memcached.yaml create mode 100644 k8s/rabbitmq.yaml create mode 100644 k8s/settings_local.py diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml new file mode 100644 index 0000000000..9f59c28c87 --- /dev/null +++ b/k8s/datatracker.yaml @@ -0,0 +1,190 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: deployment +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: datatracker + strategy: + type: Recreate + template: + metadata: + labels: + app: datatracker + spec: + # ------------------------------------------------------- + # Node Affinity + # ------------------------------------------------------- + nodeSelector: + doks.digitalocean.com/node-pool: pool-apollo-hperf + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - mailarchive + topologyKey: "kubernetes.io/hostname" + # ------------------------------------------------------- + securityContext: + runAsNonRoot: true + containers: + # ----------------------------------------------------- + # ScoutAPM Container + # ----------------------------------------------------- + - name: scoutapm + image: "scoutapp/scoutapm:version-1.4.0" + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - "sh" + - "-c" + - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + securityContext: + readOnlyRootFilesystem: true + runAsUser: 65534 # "nobody" user by default + runAsGroup: 65534 # "nogroup" group by default + # ----------------------------------------------------- + # Datatracker Container + # ----------------------------------------------------- + - name: datatracker + image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always + ports: + - containerPort: 80 + name: http + protocol: TCP + volumeMounts: + - name: dt-vol + mountPath: /a + - name: dt-tmp + mountPath: /tmp + - name: dt-cfg + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + envFrom: + - configMapRef: + name: django-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + - name: dt-vol + nfs: + server: 10.108.0.18 + path: "/mnt/datatracker" + readOnly: false + - name: dt-tmp + emptyDir: + sizeLimit: "2Gi" + - name: dt-cfg + configMap: + name: files-cfgmap + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: django-config +data: + # n.b., these are debug values / non-secret secrets + DATATRACKER_SERVER_MODE: "development" # development for staging, production for production + DATATRACKER_ADMINS: |- + Robert Sparks + Ryan Cross + Kesara Rathnayake + Jennifer Richards + Nicolas Giard + DATATRACKER_ALLOWED_HOSTS: ".ietf.org" # newline-separated list also allowed + # DATATRACKER_DATATRACKER_DEBUG: "false" + + # DB access details - needs to be filled in + # DATATRACKER_DBHOST: "db" + # DATATRACKER_DBPORT: "5432" + # DATATRACKER_DBNAME: "datatracker" + # DATATRACKER_DBUSER: "django" # secret + # DATATRACKER_DBPASS: "RkTkDPFnKpko" # secret + + DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret + + # Set this to point testing / staging at the production statics server until we + # sort that out + # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" + + # DATATRACKER_EMAIL_DEBUG: "true" + + # Outgoing email details + # DATATRACKER_EMAIL_HOST: "localhost" # defaults to localhost + # DATATRACKER_EMAIL_PORT: "2025" # defaults to 2025 + + # The value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ=" # secret + + DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" # secret + DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" # secret + DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" # secret + DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" # secret + + # API key configuration + DATATRACKER_API_KEY_TYPE: "ES265" + # secret - value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_API_PUBLIC_KEY_PEM_B64: |- + Ci0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1Ga3dFd1lIS29aSXpqMENBUVlJS + 29aSXpqMERBUWNEUWdBRXFWb2pzYW9mREpTY3VNSk4rdHNodW15Tk01TUUKZ2Fyel + ZQcWtWb3ZtRjZ5RTdJSi9kdjRGY1YrUUtDdEovck9TOGUzNlk4WkFFVll1dWtoZXM + weVoxdz09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo= + # secret - value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- + Ci0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLQpNSUdIQWdFQU1CTUdCeXFHU000O + UFnRUdDQ3FHU000OUF3RUhCRzB3YXdJQkFRUWdvSTZMSmtvcEtxOFhySGk5ClFxR1 + F2RTRBODNURllqcUx6KzhnVUxZZWNzcWhSQU5DQUFTcFdpT3hxaDhNbEp5NHdrMzY + yeUc2Ykkwemt3U0IKcXZOVStxUldpK1lYcklUc2duOTIvZ1Z4WDVBb0swbitzNUx4 + N2ZwanhrQVJWaTY2U0Y2elRKblgKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= + + # DATATRACKER_MEETECHO_API_BASE: "https://meetings.conf.meetecho.com/api/v1/" + DATATRACKER_MEETECHO_CLIENT_ID: "this-is-the-meetecho-client-id" # secret + DATATRACKER_MEETECHO_CLIENT_SECRET: "this-is-the-meetecho-client-secret" # secret + + # DATATRACKER_MATOMO_SITE_ID: "7" # must be present to enable Matomo + # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org" + + CELERY_PASSWORD: "this-is-a-secret" # secret + + DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret + + # use this to override default - one entry per line + # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- + # https://datatracker.staging.ietf.org + + # Scout configuration + DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" + DATATRACKER_SCOUT_NAME: "StagingDatatracker" +--- +apiVersion: v1 +kind: Service +metadata: + name: datatracker +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + app: datatracker \ No newline at end of file diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml new file mode 100644 index 0000000000..61df222ebd --- /dev/null +++ b/k8s/kustomization.yaml @@ -0,0 +1,10 @@ +namespace: datatracker +namePrefix: datatracker- +configMapGenerator: + - name: files-cfgmap + files: + - settings_local.py +resources: + - datatracker.yaml + - memcached.yaml + - rabbitmq.yaml \ No newline at end of file diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml new file mode 100644 index 0000000000..93665c9cf8 --- /dev/null +++ b/k8s/memcached.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: memcached +spec: + type: ClusterIP + ports: + - port: 11211 + targetPort: memcached + protocol: TCP + name: memcached + selector: + app: memcached \ No newline at end of file diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml new file mode 100644 index 0000000000..4c4a472238 --- /dev/null +++ b/k8s/rabbitmq.yaml @@ -0,0 +1,183 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: rabbitmq +spec: + replicas: 1 + revisionHistoryLimit: 2 + serviceName: rabbitmq + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + # ------------------------------------------------------- + # Node Affinity + # ------------------------------------------------------- + nodeSelector: + doks.digitalocean.com/node-pool: pool-apollo-hperf + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - datatracker + topologyKey: "kubernetes.io/hostname" + # ------------------------------------------------------- + securityContext: + runAsNonRoot: true + initContainers: + # ----------------------------------------------------- + # Init RabbitMQ data + # ----------------------------------------------------- + - name: init-rabbitmq + image: busybox:stable + command: + - "sh" + - "-c" + - "mkdir -p -m700 /mnt/rabbitmq && chown 100:101 /mnt/rabbitmq" + securityContext: + runAsNonRoot: false + runAsUser: 0 + readOnlyRootFilesystem: true + volumeMounts: + - name: "rabbitmq-data" + mountPath: "/mnt" + containers: + # ----------------------------------------------------- + # RabbitMQ Container + # ----------------------------------------------------- + - image: "ghcr.io/ietf-tools/datatracker-mq:3.12-alpine" + imagePullPolicy: Always + name: rabbitmq + ports: + - name: amqp + containerPort: 5672 + protocol: TCP + volumeMounts: + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq + subPath: "rabbitmq" + - name: rabbitmq-tmp + mountPath: /tmp + - name: rabbitmq-config + mountPath: "/etc/rabbitmq" + livenessProbe: + exec: + command: ["rabbitmq-diagnostics", "-q", "ping"] + periodSeconds: 30 + timeoutSeconds: 5 + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: ["rabbitmq-diagnostics", "-q", "ping"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + # rabbitmq image sets up uid/gid 100/101 + runAsUser: 100 + runAsGroup: 101 + volumes: + - name: rabbitmq-data + # TODO: TODO TODO + - name: rabbitmq-tmp + emptyDir: + sizeLimit: "50Mi" + - name: rabbitmq-config + configMap: + name: "rabbitmq-configmap" + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: rabbitmq-configmap +data: + definitions.json: |- + { + "permissions": [ + { + "configure": ".*", + "read": ".*", + "user": "datatracker", + "vhost": "dt", + "write": ".*" + } + ], + "users": [ + { + "hashing_algorithm": "rabbit_password_hashing_sha256", + "limits": {}, + "name": "datatracker", + "password_hash": "HJxcItcpXtBN+R/CH7dUelfKBOvdUs3AWo82SBw2yLMSguzb", + "tags": [] + } + ], + "vhosts": [ + { + "limits": [], + "metadata": { + "description": "", + "tags": [] + }, + "name": "dt" + } + ] + } + rabbitmq.conf: |- + # prevent guest from logging in over tcp + loopback_users.guest = true + + # load saved definitions + load_definitions = /etc/rabbitmq/definitions.json + + # Ensure that enough disk is available to flush to disk. To do this, need to limit the + # memory available to the container to something reasonable. See + # https://www.rabbitmq.com/production-checklist.html#monitoring-and-resource-usage + # for recommendations. + + # 1-1.5 times the memory available to the container is adequate for disk limit + disk_free_limit.absolute = 6000MB + + # This should be ~40% of the memory available to the container. Use an + # absolute number because relative will be proprtional to the full machine + # memory. + vm_memory_high_watermark.absolute = 1600MB + + # Logging + log.file = false + log.console = true + log.console.level = info + log.console.formatter = json +--- +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq +spec: + type: ClusterIP + clusterIP: None # headless service + ports: + - port: 5672 + targetPort: amqp + protocol: TCP + name: amqp + selector: + app: rabbitmq \ No newline at end of file diff --git a/k8s/settings_local.py b/k8s/settings_local.py new file mode 100644 index 0000000000..53609afa79 --- /dev/null +++ b/k8s/settings_local.py @@ -0,0 +1,259 @@ +# Copyright The IETF Trust 2007-2024, All Rights Reserved +# -*- coding: utf-8 -*- + +from base64 import b64decode +from email.utils import parseaddr +import json + +from ietf import __release_hash__ +from ietf.settings import * # pyflakes:ignore + + +def _remove_whitespace_and_b64decode(s): + """Helper to strip out whitespace and base64 decode""" + return b64decode("".join(s.split())) + + +def _multiline_to_list(s): + """Helper to split at newlines and conver to list""" + return [item.strip() for item in s.split("\n")] + + +# Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env! +SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development") + +# Secrets +_SECRET_KEY = os.environ.get("DATATRACKER_DJANGO_SECRET_KEY", None) +if _SECRET_KEY is not None: + SECRET_KEY = _SECRET_KEY +else: + raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set") + +_NOMCOM_APP_SECRET_B64 = os.environ.get("DATATRACKER_NOMCOM_APP_SECRET_B64", None) +if _NOMCOM_APP_SECRET_B64 is not None: + NOMCOM_APP_SECRET = _remove_whitespace_and_b64decode(_NOMCOM_APP_SECRET_B64) +else: + raise RuntimeError("DATATRACKER_NOMCOM_APP_SECRET_B64 must be set") + +_IANA_SYNC_PASSWORD = os.environ.get("DATATRACKER_IANA_SYNC_PASSWORD", None) +if _IANA_SYNC_PASSWORD is not None: + IANA_SYNC_PASSWORD = _IANA_SYNC_PASSWORD +else: + raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set") + +_RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD", None) +if _RFC_EDITOR_SYNC_PASSWORD is not None: + RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD") +else: + raise RuntimeError("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD must be set") + +_YOUTUBE_API_KEY = os.environ.get("DATATRACKER_YOUTUBE_API_KEY", None) +if _YOUTUBE_API_KEY is not None: + YOUTUBE_API_KEY = _YOUTUBE_API_KEY +else: + raise RuntimeError("DATATRACKER_YOUTUBE_API_KEY must be set") + +_GITHUB_BACKUP_API_KEY = os.environ.get("DATATRACKER_GITHUB_BACKUP_API_KEY", None) +if _GITHUB_BACKUP_API_KEY is not None: + GITHUB_BACKUP_API_KEY = _GITHUB_BACKUP_API_KEY +else: + raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set") + +_API_KEY_TYPE = os.environ.get("DATATRACKER_API_KEY_TYPE", None) +if _API_KEY_TYPE is not None: + API_KEY_TYPE = _API_KEY_TYPE +else: + raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set") + +_API_PUBLIC_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PUBLIC_KEY_PEM_B64", None) +if _API_PUBLIC_KEY_PEM_B64 is not None: + API_PUBLIC_KEY_PEM = _remove_whitespace_and_b64decode(_API_PUBLIC_KEY_PEM_B64) +else: + raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set") + +_API_PRIVATE_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PRIVATE_KEY_PEM_B64", None) +if _API_PRIVATE_KEY_PEM_B64 is not None: + API_PRIVATE_KEY_PEM = _remove_whitespace_and_b64decode(_API_PRIVATE_KEY_PEM_B64) +else: + raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") + +# Set DEBUG if DATATRACKER_DEBUG env var is the word "true" +DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true" + +# DATATRACKER_ALLOWED_HOSTS env var is a comma-separated list of allowed hosts +_allowed_hosts_str = os.environ.get("DATATRACKER_ALLOWED_HOSTS", None) +if _allowed_hosts_str is not None: + ALLOWED_HOSTS = _multiline_to_list(_allowed_hosts_str) + +DATABASES = { + "default": { + "HOST": os.environ.get("DATATRACKER_DBHOST", "db"), + "PORT": os.environ.get("DATATRACKER_DBPORT", "5432"), + "NAME": os.environ.get("DATATRACKER_DBNAME", "datatracker"), + "ENGINE": "django.db.backends.postgresql", + "USER": os.environ.get("DATATRACKER_DBUSER", "django"), + "PASSWORD": os.environ.get("DATATRACKER_DBPASS", ""), + }, +} + +# DATATRACKER_ADMINS is a newline-delimited list of addresses parseable by email.utils.parseaddr +_admins_str = os.environ.get("DATATRACKER_ADMINS", None) +if _admins_str is not None: + ADMINS = [parseaddr(admin) for admin in _multiline_to_list(_admins_str)] +else: + raise RuntimeError("DATATRACKER_ADMINS must be set") + +USING_DEBUG_EMAIL_SERVER = os.environ.get("DATATRACKER_EMAIL_DEBUG", "false").lower() == "true" +EMAIL_HOST = os.environ.get("DATATRACKER_EMAIL_HOST", "localhost") +EMAIL_PORT = int(os.environ.get("DATATRACKER_EMAIL_PORT", "2025")) + +_celery_password = os.environ.get("CELERY_PASSWORD", None) +if _celery_password is None: + raise RuntimeError("CELERY_PASSWORD must be set") +CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( + host=os.environ.get("RABBITMQ_HOSTNAME", "rabbitmq"), + password=_celery_password, + queue=os.environ.get("RABBITMQ_QUEUE", "dt") +) + +IANA_SYNC_USERNAME = "ietfsync" +IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" +IANA_SYNC_PROTOCOLS_URL = "http://www.iana.org/protocols/" + +RFC_EDITOR_NOTIFICATION_URL = "http://www.rfc-editor.org/parser/parser.php" + +STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://registration.ietf.org/{number}/attendees/?apikey=redacted' + +#FIRST_CUTOFF_DAYS = 12 +#SECOND_CUTOFF_DAYS = 12 +#SUBMISSION_CUTOFF_DAYS = 26 +#SUBMISSION_CORRECTION_DAYS = 57 +MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26 +MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 54 + +HTPASSWD_COMMAND = "/usr/bin/htpasswd2" + +_MEETECHO_CLIENT_ID = os.environ.get("DATATRACKER_MEETECHO_CLIENT_ID", None) +_MEETECHO_CLIENT_SECRET = os.environ.get("DATATRACKER_MEETECHO_CLIENT_SECRET", None) +if _MEETECHO_CLIENT_ID is not None and _MEETECHO_CLIENT_SECRET is not None: + MEETECHO_API_CONFIG = { + "api_base": os.environ.get( + "DATATRACKER_MEETECHO_API_BASE", + "https://meetings.conf.meetecho.com/api/v1/", + ), + "client_id": _MEETECHO_CLIENT_ID, + "client_secret": _MEETECHO_CLIENT_SECRET, + "request_timeout": 3.01, # python-requests doc recommend slightly > a multiple of 3 seconds + } +else: + raise RuntimeError( + "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set" + ) + +_APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None) +if _APP_API_TOKENS_JSON is not None: + APP_API_TOKENS = json.loads(_APP_API_TOKENS_JSON) +else: + APP_API_TOKENS = {} + +EMAIL_COPY_TO = "" + +# Until we teach the datatracker to look beyond cloudflare for this check +IDSUBMIT_MAX_DAILY_SAME_SUBMITTER = 5000 + +# Leave DATATRACKER_MATOMO_SITE_ID unset to disable Matomo reporting +if "DATATRACKER_MATOMO_SITE_ID" in os.environ: + MATOMO_DOMAIN_PATH = os.environ.get("DATATRACKER_MATOMO_DOMAIN_PATH", "analytics.ietf.org") + MATOMO_SITE_ID = os.environ.get("DATATRACKER_MATOMO_SITE_ID") + MATOMO_DISABLE_COOKIES = True + +# Leave DATATRACKER_SCOUT_KEY unset to disable Scout APM agent +_SCOUT_KEY = os.environ.get("DATATRACKER_SCOUT_KEY", None) +if _SCOUT_KEY is not None: + if SERVER_MODE == "production": + PROD_PRE_APPS = ["scout_apm.django", ] + else: + DEV_PRE_APPS = ["scout_apm.django", ] + SCOUT_MONITOR = True + SCOUT_KEY = _SCOUT_KEY + SCOUT_NAME = os.environ.get("DATATRACKER_SCOUT_NAME", "Datatracker") + SCOUT_ERRORS_ENABLED = True + SCOUT_SHUTDOWN_MESSAGE_ENABLED = False + SCOUT_CORE_AGENT_SOCKET_PATH = "tcp://{host}:{port}".format( + host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "localhost"), + port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "6590"), + ) + SCOUT_CORE_AGENT_DOWNLOAD = False + SCOUT_CORE_AGENT_LAUNCH = False + SCOUT_REVISION_SHA = __release_hash__[:7] + +# Path to the email alias lists. Used by ietf.utils.aliases +DRAFT_ALIASES_PATH = "/a/postfix/draft-aliases" +DRAFT_VIRTUAL_PATH = "/a/postfix/draft-virtual" +GROUP_ALIASES_PATH = "/a/postfix/group-aliases" +GROUP_VIRTUAL_PATH = "/a/postfix/group-virtual" + +STATIC_URL = os.environ.get("DATATRACKER_STATIC_URL", None) +if STATIC_URL is None: + from ietf import __version__ + STATIC_URL = f"https://static.ietf.org/dt/{__version__}/" + +# Set these to the same as "production" in settings.py, whether production mode or not +MEDIA_ROOT = "/a/www/www6s/lib/dt/media/" +MEDIA_URL = "https://www.ietf.org/lib/dt/media/" +PHOTOS_DIRNAME = "photo" +PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME + +# Normally only set for debug, but needed until we have a real FS +DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') + +# Binaries that are different in the docker image +DE_GFM_BINARY = "/usr/local/bin/de-gfm" +IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" + +# Duplicating production cache from settings.py and using it whether we're in production mode or not +MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") +MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") +from ietf import __version__ +CACHES = { + "default": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + "VERSION": __version__, + "KEY_PREFIX": "ietf:dt", + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "sessions": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt", + }, + "htmlized": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/htmlized", + "OPTIONS": { + "MAX_ENTRIES": 100000, # 100,000 + }, + }, + "pdfized": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/pdfized", + "OPTIONS": { + "MAX_ENTRIES": 100000, # 100,000 + }, + }, + "slowpages": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/slowpages", + "OPTIONS": { + "MAX_ENTRIES": 5000, + }, + }, +} + +_csrf_trusted_origins_str = os.environ.get("DATATRACKER_CSRF_TRUSTED_ORIGINS") +if _csrf_trusted_origins_str is not None: + CSRF_TRUSTED_ORIGINS = _multiline_to_list(_csrf_trusted_origins_str) From 0a3bb9e3813c7bc53250b57d7f1e90b3d2dfeeb1 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 4 May 2024 15:02:11 -0400 Subject: [PATCH 50/70] refactor: move node affinity to upstream --- k8s/datatracker.yaml | 20 +------------------- k8s/rabbitmq.yaml | 18 ------------------ 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 9f59c28c87..b2a443db30 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: deployment + name: datatracker spec: replicas: 1 revisionHistoryLimit: 2 @@ -15,24 +15,6 @@ spec: labels: app: datatracker spec: - # ------------------------------------------------------- - # Node Affinity - # ------------------------------------------------------- - nodeSelector: - doks.digitalocean.com/node-pool: pool-apollo-hperf - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - mailarchive - topologyKey: "kubernetes.io/hostname" - # ------------------------------------------------------- securityContext: runAsNonRoot: true containers: diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml index 4c4a472238..c5a823ac2a 100644 --- a/k8s/rabbitmq.yaml +++ b/k8s/rabbitmq.yaml @@ -14,24 +14,6 @@ spec: labels: app: rabbitmq spec: - # ------------------------------------------------------- - # Node Affinity - # ------------------------------------------------------- - nodeSelector: - doks.digitalocean.com/node-pool: pool-apollo-hperf - affinity: - podAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - datatracker - topologyKey: "kubernetes.io/hostname" - # ------------------------------------------------------- securityContext: runAsNonRoot: true initContainers: From 24309c2b04e76f598ca99f073a8ceca0c0d4b44c Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 4 May 2024 18:51:18 -0400 Subject: [PATCH 51/70] refactor: move nfs entry upstream --- k8s/datatracker.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index b2a443db30..f63a7004d1 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -64,11 +64,8 @@ spec: runAsUser: 1000 runAsGroup: 1000 volumes: + # To be overriden with the actual shared volume - name: dt-vol - nfs: - server: 10.108.0.18 - path: "/mnt/datatracker" - readOnly: false - name: dt-tmp emptyDir: sizeLimit: "2Gi" From 05bd47cbad4c8cb3e98cd3188c342ec1274615cb Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 4 May 2024 19:38:42 -0400 Subject: [PATCH 52/70] chore: add rabbitmq vol claim template --- k8s/kustomization.yaml | 2 +- k8s/rabbitmq.yaml | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 61df222ebd..021dfe7981 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -1,5 +1,5 @@ namespace: datatracker -namePrefix: datatracker- +namePrefix: dt- configMapGenerator: - name: files-cfgmap files: diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml index c5a823ac2a..4655771213 100644 --- a/k8s/rabbitmq.yaml +++ b/k8s/rabbitmq.yaml @@ -76,7 +76,8 @@ spec: runAsGroup: 101 volumes: - name: rabbitmq-data - # TODO: TODO TODO + persistentVolumeClaim: + claimName: "rabbitmq-data-vol" - name: rabbitmq-tmp emptyDir: sizeLimit: "50Mi" @@ -86,6 +87,16 @@ spec: dnsPolicy: ClusterFirst restartPolicy: Always terminationGracePeriodSeconds: 30 + volumeClaimTemplates: + - metadata: + name: rabbitmq-data-vol + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + # storageClassName: "" --- apiVersion: v1 kind: ConfigMap From d075404fdb78f86f09bb0ad8bec326f0f651a604 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Sat, 4 May 2024 20:15:02 -0400 Subject: [PATCH 53/70] refactor: add beat + celery deployments --- k8s/beat.yaml | 61 +++++++++++++++++++++++++++++++ k8s/celery.yaml | 80 +++++++++++++++++++++++++++++++++++++++++ k8s/datatracker.yaml | 81 ++---------------------------------------- k8s/django-config.yaml | 77 +++++++++++++++++++++++++++++++++++++++ k8s/kustomization.yaml | 3 ++ k8s/memcached.yaml | 39 ++++++++++++++++++++ 6 files changed, 263 insertions(+), 78 deletions(-) create mode 100644 k8s/beat.yaml create mode 100644 k8s/celery.yaml create mode 100644 k8s/django-config.yaml diff --git a/k8s/beat.yaml b/k8s/beat.yaml new file mode 100644 index 0000000000..7099fec76e --- /dev/null +++ b/k8s/beat.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: beat +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: beat + strategy: + type: Recreate + template: + metadata: + labels: + app: beat + spec: + securityContext: + runAsNonRoot: true + containers: + - name: beat + image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + protocol: TCP + volumeMounts: + - name: dt-vol + mountPath: /a + - name: dt-tmp + mountPath: /tmp + - name: dt-cfg + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + env: + - name: "CONTAINER_ROLE" + value: "beat" + envFrom: + - configMapRef: + name: django-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + # To be overriden with the actual shared volume + - name: dt-vol + - name: dt-tmp + emptyDir: + sizeLimit: "2Gi" + - name: dt-cfg + configMap: + name: files-cfgmap + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 \ No newline at end of file diff --git a/k8s/celery.yaml b/k8s/celery.yaml new file mode 100644 index 0000000000..0c6bbf4758 --- /dev/null +++ b/k8s/celery.yaml @@ -0,0 +1,80 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: celery + strategy: + type: Recreate + template: + metadata: + labels: + app: celery + spec: + securityContext: + runAsNonRoot: true + containers: + # ----------------------------------------------------- + # ScoutAPM Container + # ----------------------------------------------------- + - name: scoutapm + image: "scoutapp/scoutapm:version-1.4.0" + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - "sh" + - "-c" + - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + securityContext: + readOnlyRootFilesystem: true + runAsUser: 65534 # "nobody" user by default + runAsGroup: 65534 # "nogroup" group by default + # ----------------------------------------------------- + # Celery Container + # ----------------------------------------------------- + - name: celery + image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + protocol: TCP + volumeMounts: + - name: dt-vol + mountPath: /a + - name: dt-tmp + mountPath: /tmp + - name: dt-cfg + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + env: + - name: "CONTAINER_ROLE" + value: "celery" + envFrom: + - configMapRef: + name: django-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + # To be overriden with the actual shared volume + - name: dt-vol + - name: dt-tmp + emptyDir: + sizeLimit: "2Gi" + - name: dt-cfg + configMap: + name: files-cfgmap + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 \ No newline at end of file diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index f63a7004d1..6a832fb8b7 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -52,6 +52,9 @@ spec: - name: dt-cfg mountPath: /workspace/ietf/settings_local.py subPath: settings_local.py + env: + - name: "CONTAINER_ROLE" + value: "datatracker" envFrom: - configMapRef: name: django-config @@ -77,84 +80,6 @@ spec: terminationGracePeriodSeconds: 30 --- apiVersion: v1 -kind: ConfigMap -metadata: - name: django-config -data: - # n.b., these are debug values / non-secret secrets - DATATRACKER_SERVER_MODE: "development" # development for staging, production for production - DATATRACKER_ADMINS: |- - Robert Sparks - Ryan Cross - Kesara Rathnayake - Jennifer Richards - Nicolas Giard - DATATRACKER_ALLOWED_HOSTS: ".ietf.org" # newline-separated list also allowed - # DATATRACKER_DATATRACKER_DEBUG: "false" - - # DB access details - needs to be filled in - # DATATRACKER_DBHOST: "db" - # DATATRACKER_DBPORT: "5432" - # DATATRACKER_DBNAME: "datatracker" - # DATATRACKER_DBUSER: "django" # secret - # DATATRACKER_DBPASS: "RkTkDPFnKpko" # secret - - DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret - - # Set this to point testing / staging at the production statics server until we - # sort that out - # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" - - # DATATRACKER_EMAIL_DEBUG: "true" - - # Outgoing email details - # DATATRACKER_EMAIL_HOST: "localhost" # defaults to localhost - # DATATRACKER_EMAIL_PORT: "2025" # defaults to 2025 - - # The value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ=" # secret - - DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" # secret - DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" # secret - DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" # secret - DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" # secret - - # API key configuration - DATATRACKER_API_KEY_TYPE: "ES265" - # secret - value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_API_PUBLIC_KEY_PEM_B64: |- - Ci0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1Ga3dFd1lIS29aSXpqMENBUVlJS - 29aSXpqMERBUWNEUWdBRXFWb2pzYW9mREpTY3VNSk4rdHNodW15Tk01TUUKZ2Fyel - ZQcWtWb3ZtRjZ5RTdJSi9kdjRGY1YrUUtDdEovck9TOGUzNlk4WkFFVll1dWtoZXM - weVoxdz09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo= - # secret - value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- - Ci0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLQpNSUdIQWdFQU1CTUdCeXFHU000O - UFnRUdDQ3FHU000OUF3RUhCRzB3YXdJQkFRUWdvSTZMSmtvcEtxOFhySGk5ClFxR1 - F2RTRBODNURllqcUx6KzhnVUxZZWNzcWhSQU5DQUFTcFdpT3hxaDhNbEp5NHdrMzY - yeUc2Ykkwemt3U0IKcXZOVStxUldpK1lYcklUc2duOTIvZ1Z4WDVBb0swbitzNUx4 - N2ZwanhrQVJWaTY2U0Y2elRKblgKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= - - # DATATRACKER_MEETECHO_API_BASE: "https://meetings.conf.meetecho.com/api/v1/" - DATATRACKER_MEETECHO_CLIENT_ID: "this-is-the-meetecho-client-id" # secret - DATATRACKER_MEETECHO_CLIENT_SECRET: "this-is-the-meetecho-client-secret" # secret - - # DATATRACKER_MATOMO_SITE_ID: "7" # must be present to enable Matomo - # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org" - - CELERY_PASSWORD: "this-is-a-secret" # secret - - DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret - - # use this to override default - one entry per line - # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- - # https://datatracker.staging.ietf.org - - # Scout configuration - DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" - DATATRACKER_SCOUT_NAME: "StagingDatatracker" ---- -apiVersion: v1 kind: Service metadata: name: datatracker diff --git a/k8s/django-config.yaml b/k8s/django-config.yaml new file mode 100644 index 0000000000..1e27a32f02 --- /dev/null +++ b/k8s/django-config.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: django-config +data: + # n.b., these are debug values / non-secret secrets + DATATRACKER_SERVER_MODE: "development" # development for staging, production for production + DATATRACKER_ADMINS: |- + Robert Sparks + Ryan Cross + Kesara Rathnayake + Jennifer Richards + Nicolas Giard + DATATRACKER_ALLOWED_HOSTS: ".ietf.org" # newline-separated list also allowed + # DATATRACKER_DATATRACKER_DEBUG: "false" + + # DB access details - needs to be filled in + # DATATRACKER_DBHOST: "db" + # DATATRACKER_DBPORT: "5432" + # DATATRACKER_DBNAME: "datatracker" + # DATATRACKER_DBUSER: "django" # secret + # DATATRACKER_DBPASS: "RkTkDPFnKpko" # secret + + DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret + + # Set this to point testing / staging at the production statics server until we + # sort that out + # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" + + # DATATRACKER_EMAIL_DEBUG: "true" + + # Outgoing email details + # DATATRACKER_EMAIL_HOST: "localhost" # defaults to localhost + # DATATRACKER_EMAIL_PORT: "2025" # defaults to 2025 + + # The value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ=" # secret + + DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" # secret + DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" # secret + DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" # secret + DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" # secret + + # API key configuration + DATATRACKER_API_KEY_TYPE: "ES265" + # secret - value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_API_PUBLIC_KEY_PEM_B64: |- + Ci0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1Ga3dFd1lIS29aSXpqMENBUVlJS + 29aSXpqMERBUWNEUWdBRXFWb2pzYW9mREpTY3VNSk4rdHNodW15Tk01TUUKZ2Fyel + ZQcWtWb3ZtRjZ5RTdJSi9kdjRGY1YrUUtDdEovck9TOGUzNlk4WkFFVll1dWtoZXM + weVoxdz09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo= + # secret - value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- + Ci0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLQpNSUdIQWdFQU1CTUdCeXFHU000O + UFnRUdDQ3FHU000OUF3RUhCRzB3YXdJQkFRUWdvSTZMSmtvcEtxOFhySGk5ClFxR1 + F2RTRBODNURllqcUx6KzhnVUxZZWNzcWhSQU5DQUFTcFdpT3hxaDhNbEp5NHdrMzY + yeUc2Ykkwemt3U0IKcXZOVStxUldpK1lYcklUc2duOTIvZ1Z4WDVBb0swbitzNUx4 + N2ZwanhrQVJWaTY2U0Y2elRKblgKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= + + # DATATRACKER_MEETECHO_API_BASE: "https://meetings.conf.meetecho.com/api/v1/" + DATATRACKER_MEETECHO_CLIENT_ID: "this-is-the-meetecho-client-id" # secret + DATATRACKER_MEETECHO_CLIENT_SECRET: "this-is-the-meetecho-client-secret" # secret + + # DATATRACKER_MATOMO_SITE_ID: "7" # must be present to enable Matomo + # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org" + + CELERY_PASSWORD: "this-is-a-secret" # secret + + DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret + + # use this to override default - one entry per line + # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- + # https://datatracker.staging.ietf.org + + # Scout configuration + DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" + DATATRACKER_SCOUT_NAME: "StagingDatatracker" \ No newline at end of file diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 021dfe7981..3f09ce65a9 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -5,6 +5,9 @@ configMapGenerator: files: - settings_local.py resources: + - beat.yaml + - celery.yaml - datatracker.yaml + - django-config.yaml - memcached.yaml - rabbitmq.yaml \ No newline at end of file diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml index 93665c9cf8..f34876718f 100644 --- a/k8s/memcached.yaml +++ b/k8s/memcached.yaml @@ -1,3 +1,42 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: memcached +spec: + replicas: 1 + revisionHistoryLimit: 2 + serviceName: memcached + selector: + matchLabels: + app: memcached + template: + metadata: + labels: + app: memcached + spec: + securityContext: + runAsNonRoot: true + containers: + - image: "memcached:1.6-alpine" + imagePullPolicy: IfNotPresent + args: ["-m", "1024"] + name: memcached + ports: + - name: memcached + containerPort: 11211 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + # memcached image sets up uid/gid 11211 + runAsUser: 11211 + runAsGroup: 11211 + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 --- apiVersion: v1 kind: Service From 867360e96f15d7d84b80b23579de86be90efc39f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 9 May 2024 15:24:39 -0300 Subject: [PATCH 54/70] ci: k8s fixup (#7401) * ci: remove stray serviceName * ci: volumeClaimTemplate name -> volumeMount, not volume * ci: datatracker listens on containerPort 8000 * ci: services/containers have dt- prefix * chore: adjust indent for k8s yaml * ci: use a secret for CELERY_PASSWORD * fix: touched wrong CELERY_PASSWORD setting * ci: get rid of the celery pw secretGenerator * ci: use DB_PASS instead of DBPASS (etc) for k8s * ci: Fill in django-config.yaml from env vars * ci: add vault-mappings.txt * ci: use $CELERY_PASSWORD in rabbitmq.yaml * ci: moving vault-mappings.txt out of this repo * Revert "ci: Fill in django-config.yaml from env vars" This reverts commit 75cd181deb390d3ab21d6887b091d66c80e1d18e. * Revert "ci: use $CELERY_PASSWORD in rabbitmq.yaml" This reverts commit f251f9920d07c65413f72fd165cc06acd562c2c7. * ci: parameterize db OPTIONS setting --- .editorconfig | 6 + k8s/beat.yaml | 2 +- k8s/celery.yaml | 2 +- k8s/datatracker.yaml | 188 +++++++++++----------- k8s/django-config.yaml | 12 +- k8s/kustomization.yaml | 2 +- k8s/memcached.yaml | 3 +- k8s/rabbitmq.yaml | 351 ++++++++++++++++++++--------------------- k8s/settings_local.py | 17 +- 9 files changed, 294 insertions(+), 289 deletions(-) diff --git a/.editorconfig b/.editorconfig index 35c9168a28..4e5b304b6a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -55,4 +55,10 @@ insert_final_newline = false # --------------------------------------------------------- # Use 2-space indents [helm/**.yaml] +indent_size = 2 + +# Settings for Kubernetes yaml +# --------------------------------------------------------- +# Use 2-space indents +[k8s/**.yaml] indent_size = 2 \ No newline at end of file diff --git a/k8s/beat.yaml b/k8s/beat.yaml index 7099fec76e..3400b21cbf 100644 --- a/k8s/beat.yaml +++ b/k8s/beat.yaml @@ -58,4 +58,4 @@ spec: name: files-cfgmap dnsPolicy: ClusterFirst restartPolicy: Always - terminationGracePeriodSeconds: 30 \ No newline at end of file + terminationGracePeriodSeconds: 30 diff --git a/k8s/celery.yaml b/k8s/celery.yaml index 0c6bbf4758..407f21e66d 100644 --- a/k8s/celery.yaml +++ b/k8s/celery.yaml @@ -77,4 +77,4 @@ spec: name: files-cfgmap dnsPolicy: ClusterFirst restartPolicy: Always - terminationGracePeriodSeconds: 30 \ No newline at end of file + terminationGracePeriodSeconds: 30 diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 6a832fb8b7..7ca92ba99e 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -1,94 +1,94 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: datatracker -spec: - replicas: 1 - revisionHistoryLimit: 2 - selector: - matchLabels: - app: datatracker - strategy: - type: Recreate - template: - metadata: - labels: - app: datatracker - spec: - securityContext: - runAsNonRoot: true - containers: - # ----------------------------------------------------- - # ScoutAPM Container - # ----------------------------------------------------- - - name: scoutapm - image: "scoutapp/scoutapm:version-1.4.0" - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - "sh" - - "-c" - - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" - securityContext: - readOnlyRootFilesystem: true - runAsUser: 65534 # "nobody" user by default - runAsGroup: 65534 # "nogroup" group by default - # ----------------------------------------------------- - # Datatracker Container - # ----------------------------------------------------- - - name: datatracker - image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" - imagePullPolicy: Always - ports: - - containerPort: 80 - name: http - protocol: TCP - volumeMounts: - - name: dt-vol - mountPath: /a - - name: dt-tmp - mountPath: /tmp - - name: dt-cfg - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - env: - - name: "CONTAINER_ROLE" - value: "datatracker" - envFrom: - - configMapRef: - name: django-config - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsUser: 1000 - runAsGroup: 1000 - volumes: - # To be overriden with the actual shared volume - - name: dt-vol - - name: dt-tmp - emptyDir: - sizeLimit: "2Gi" - - name: dt-cfg - configMap: - name: files-cfgmap - dnsPolicy: ClusterFirst - restartPolicy: Always - terminationGracePeriodSeconds: 30 ---- -apiVersion: v1 -kind: Service -metadata: - name: datatracker -spec: - type: ClusterIP - ports: - - port: 80 - targetPort: http - protocol: TCP - name: http - selector: - app: datatracker \ No newline at end of file +apiVersion: apps/v1 +kind: Deployment +metadata: + name: datatracker +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: datatracker + strategy: + type: Recreate + template: + metadata: + labels: + app: datatracker + spec: + securityContext: + runAsNonRoot: true + containers: + # ----------------------------------------------------- + # ScoutAPM Container + # ----------------------------------------------------- + - name: scoutapm + image: "scoutapp/scoutapm:version-1.4.0" + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - "sh" + - "-c" + - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + securityContext: + readOnlyRootFilesystem: true + runAsUser: 65534 # "nobody" user by default + runAsGroup: 65534 # "nogroup" group by default + # ----------------------------------------------------- + # Datatracker Container + # ----------------------------------------------------- + - name: datatracker + image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + protocol: TCP + volumeMounts: + - name: dt-vol + mountPath: /a + - name: dt-tmp + mountPath: /tmp + - name: dt-cfg + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + env: + - name: "CONTAINER_ROLE" + value: "datatracker" + envFrom: + - configMapRef: + name: django-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + # To be overriden with the actual shared volume + - name: dt-vol + - name: dt-tmp + emptyDir: + sizeLimit: "2Gi" + - name: dt-cfg + configMap: + name: files-cfgmap + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + name: datatracker +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + app: datatracker diff --git a/k8s/django-config.yaml b/k8s/django-config.yaml index 1e27a32f02..e6f5b3ca8f 100644 --- a/k8s/django-config.yaml +++ b/k8s/django-config.yaml @@ -15,11 +15,11 @@ data: # DATATRACKER_DATATRACKER_DEBUG: "false" # DB access details - needs to be filled in - # DATATRACKER_DBHOST: "db" - # DATATRACKER_DBPORT: "5432" - # DATATRACKER_DBNAME: "datatracker" - # DATATRACKER_DBUSER: "django" # secret - # DATATRACKER_DBPASS: "RkTkDPFnKpko" # secret + # DATATRACKER_DB_HOST: "db" + # DATATRACKER_DB_PORT: "5432" + # DATATRACKER_DB_NAME: "datatracker" + # DATATRACKER_DB_USER: "django" # secret + # DATATRACKER_DB_PASS: "RkTkDPFnKpko" # secret DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret @@ -74,4 +74,4 @@ data: # Scout configuration DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" - DATATRACKER_SCOUT_NAME: "StagingDatatracker" \ No newline at end of file + DATATRACKER_SCOUT_NAME: "StagingDatatracker" diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 3f09ce65a9..e618bb630d 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -10,4 +10,4 @@ resources: - datatracker.yaml - django-config.yaml - memcached.yaml - - rabbitmq.yaml \ No newline at end of file + - rabbitmq.yaml diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml index f34876718f..56efc235ec 100644 --- a/k8s/memcached.yaml +++ b/k8s/memcached.yaml @@ -5,7 +5,6 @@ metadata: spec: replicas: 1 revisionHistoryLimit: 2 - serviceName: memcached selector: matchLabels: app: memcached @@ -50,4 +49,4 @@ spec: protocol: TCP name: memcached selector: - app: memcached \ No newline at end of file + app: memcached diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml index 4655771213..132ca79ded 100644 --- a/k8s/rabbitmq.yaml +++ b/k8s/rabbitmq.yaml @@ -1,176 +1,175 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: rabbitmq -spec: - replicas: 1 - revisionHistoryLimit: 2 - serviceName: rabbitmq - selector: - matchLabels: - app: rabbitmq - template: - metadata: - labels: - app: rabbitmq - spec: - securityContext: - runAsNonRoot: true - initContainers: - # ----------------------------------------------------- - # Init RabbitMQ data - # ----------------------------------------------------- - - name: init-rabbitmq - image: busybox:stable - command: - - "sh" - - "-c" - - "mkdir -p -m700 /mnt/rabbitmq && chown 100:101 /mnt/rabbitmq" - securityContext: - runAsNonRoot: false - runAsUser: 0 - readOnlyRootFilesystem: true - volumeMounts: - - name: "rabbitmq-data" - mountPath: "/mnt" - containers: - # ----------------------------------------------------- - # RabbitMQ Container - # ----------------------------------------------------- - - image: "ghcr.io/ietf-tools/datatracker-mq:3.12-alpine" - imagePullPolicy: Always - name: rabbitmq - ports: - - name: amqp - containerPort: 5672 - protocol: TCP - volumeMounts: - - name: rabbitmq-data - mountPath: /var/lib/rabbitmq - subPath: "rabbitmq" - - name: rabbitmq-tmp - mountPath: /tmp - - name: rabbitmq-config - mountPath: "/etc/rabbitmq" - livenessProbe: - exec: - command: ["rabbitmq-diagnostics", "-q", "ping"] - periodSeconds: 30 - timeoutSeconds: 5 - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - exec: - command: ["rabbitmq-diagnostics", "-q", "ping"] - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - # rabbitmq image sets up uid/gid 100/101 - runAsUser: 100 - runAsGroup: 101 - volumes: - - name: rabbitmq-data - persistentVolumeClaim: - claimName: "rabbitmq-data-vol" - - name: rabbitmq-tmp - emptyDir: - sizeLimit: "50Mi" - - name: rabbitmq-config - configMap: - name: "rabbitmq-configmap" - dnsPolicy: ClusterFirst - restartPolicy: Always - terminationGracePeriodSeconds: 30 - volumeClaimTemplates: - - metadata: - name: rabbitmq-data-vol - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 8Gi - # storageClassName: "" ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: rabbitmq-configmap -data: - definitions.json: |- - { - "permissions": [ - { - "configure": ".*", - "read": ".*", - "user": "datatracker", - "vhost": "dt", - "write": ".*" - } - ], - "users": [ - { - "hashing_algorithm": "rabbit_password_hashing_sha256", - "limits": {}, - "name": "datatracker", - "password_hash": "HJxcItcpXtBN+R/CH7dUelfKBOvdUs3AWo82SBw2yLMSguzb", - "tags": [] - } - ], - "vhosts": [ - { - "limits": [], - "metadata": { - "description": "", - "tags": [] - }, - "name": "dt" - } - ] - } - rabbitmq.conf: |- - # prevent guest from logging in over tcp - loopback_users.guest = true - - # load saved definitions - load_definitions = /etc/rabbitmq/definitions.json - - # Ensure that enough disk is available to flush to disk. To do this, need to limit the - # memory available to the container to something reasonable. See - # https://www.rabbitmq.com/production-checklist.html#monitoring-and-resource-usage - # for recommendations. - - # 1-1.5 times the memory available to the container is adequate for disk limit - disk_free_limit.absolute = 6000MB - - # This should be ~40% of the memory available to the container. Use an - # absolute number because relative will be proprtional to the full machine - # memory. - vm_memory_high_watermark.absolute = 1600MB - - # Logging - log.file = false - log.console = true - log.console.level = info - log.console.formatter = json ---- -apiVersion: v1 -kind: Service -metadata: - name: rabbitmq -spec: - type: ClusterIP - clusterIP: None # headless service - ports: - - port: 5672 - targetPort: amqp - protocol: TCP - name: amqp - selector: - app: rabbitmq \ No newline at end of file +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: rabbitmq +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + securityContext: + runAsNonRoot: true + initContainers: + # ----------------------------------------------------- + # Init RabbitMQ data + # ----------------------------------------------------- + - name: init-rabbitmq + image: busybox:stable + command: + - "sh" + - "-c" + - "mkdir -p -m700 /mnt/rabbitmq && chown 100:101 /mnt/rabbitmq" + securityContext: + runAsNonRoot: false + runAsUser: 0 + readOnlyRootFilesystem: true + volumeMounts: + - name: "rabbitmq-data" + mountPath: "/mnt" + containers: + # ----------------------------------------------------- + # RabbitMQ Container + # ----------------------------------------------------- + - image: "ghcr.io/ietf-tools/datatracker-mq:3.12-alpine" + imagePullPolicy: Always + name: rabbitmq + ports: + - name: amqp + containerPort: 5672 + protocol: TCP + volumeMounts: + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq + subPath: "rabbitmq" + - name: rabbitmq-tmp + mountPath: /tmp + - name: rabbitmq-config + mountPath: "/etc/rabbitmq" + env: + - name: "CELERY_PASSWORD" + value: "this-is-a-secret" + livenessProbe: + exec: + command: ["rabbitmq-diagnostics", "-q", "ping"] + periodSeconds: 30 + timeoutSeconds: 5 + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: ["rabbitmq-diagnostics", "-q", "ping"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + # rabbitmq image sets up uid/gid 100/101 + runAsUser: 100 + runAsGroup: 101 + volumes: + - name: rabbitmq-tmp + emptyDir: + sizeLimit: "50Mi" + - name: rabbitmq-config + configMap: + name: "rabbitmq-configmap" + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 + volumeClaimTemplates: + - metadata: + name: rabbitmq-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + # storageClassName: "" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: rabbitmq-configmap +data: + definitions.json: |- + { + "permissions": [ + { + "configure": ".*", + "read": ".*", + "user": "datatracker", + "vhost": "dt", + "write": ".*" + } + ], + "users": [ + { + "hashing_algorithm": "rabbit_password_hashing_sha256", + "limits": {}, + "name": "datatracker", + "password_hash": "HJxcItcpXtBN+R/CH7dUelfKBOvdUs3AWo82SBw2yLMSguzb", + "tags": [] + } + ], + "vhosts": [ + { + "limits": [], + "metadata": { + "description": "", + "tags": [] + }, + "name": "dt" + } + ] + } + rabbitmq.conf: |- + # prevent guest from logging in over tcp + loopback_users.guest = true + + # load saved definitions + load_definitions = /etc/rabbitmq/definitions.json + + # Ensure that enough disk is available to flush to disk. To do this, need to limit the + # memory available to the container to something reasonable. See + # https://www.rabbitmq.com/production-checklist.html#monitoring-and-resource-usage + # for recommendations. + + # 1-1.5 times the memory available to the container is adequate for disk limit + disk_free_limit.absolute = 6000MB + + # This should be ~40% of the memory available to the container. Use an + # absolute number because relative will be proprtional to the full machine + # memory. + vm_memory_high_watermark.absolute = 1600MB + + # Logging + log.file = false + log.console = true + log.console.level = info + log.console.formatter = json +--- +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq +spec: + type: ClusterIP + clusterIP: None # headless service + ports: + - port: 5672 + targetPort: amqp + protocol: TCP + name: amqp + selector: + app: rabbitmq diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 53609afa79..00d0cae781 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -87,12 +87,13 @@ def _multiline_to_list(s): DATABASES = { "default": { - "HOST": os.environ.get("DATATRACKER_DBHOST", "db"), - "PORT": os.environ.get("DATATRACKER_DBPORT", "5432"), - "NAME": os.environ.get("DATATRACKER_DBNAME", "datatracker"), + "HOST": os.environ.get("DATATRACKER_DB_HOST", "db"), + "PORT": os.environ.get("DATATRACKER_DB_PORT", "5432"), + "NAME": os.environ.get("DATATRACKER_DB_NAME", "datatracker"), "ENGINE": "django.db.backends.postgresql", - "USER": os.environ.get("DATATRACKER_DBUSER", "django"), - "PASSWORD": os.environ.get("DATATRACKER_DBPASS", ""), + "USER": os.environ.get("DATATRACKER_DB_USER", "django"), + "PASSWORD": os.environ.get("DATATRACKER_DB_PASS", ""), + "OPTIONS": json.loads(os.environ.get("DATATRACKER_DB_OPTS_JSON", "{}")), }, } @@ -111,7 +112,7 @@ def _multiline_to_list(s): if _celery_password is None: raise RuntimeError("CELERY_PASSWORD must be set") CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( - host=os.environ.get("RABBITMQ_HOSTNAME", "rabbitmq"), + host=os.environ.get("RABBITMQ_HOSTNAME", "dt-rabbitmq"), password=_celery_password, queue=os.environ.get("RABBITMQ_QUEUE", "dt") ) @@ -212,8 +213,8 @@ def _multiline_to_list(s): IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" # Duplicating production cache from settings.py and using it whether we're in production mode or not -MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") -MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") +MEMCACHED_HOST = os.environ.get("DT_MEMCACHED_SERVICE_HOST", "127.0.0.1") +MEMCACHED_PORT = os.environ.get("DT_MEMCACHED_SERVICE_PORT", "11211") from ietf import __version__ CACHES = { "default": { From 7e56b2e923a2f1a489ab0717ae7f589fab75e9e2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 10 May 2024 11:17:24 -0300 Subject: [PATCH 55/70] ci: drop helm chart GHA step --- .github/workflows/build.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3ac2441a84..bd65da036c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -248,17 +248,6 @@ jobs: push: true tags: ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }} - - name: Package and Push Chart - env: - VERSION: 2.0.${{ github.run_number }} - run: | - echo "Setting version ${{ env.PKG_VERSION }}..." - sed -i -r -e "s|^version: .*$|version: '${{ env.PKG_VERSION }}'|" -e "s|^appVersion: .*$|appVersion: '${{ env.PKG_VERSION }}'|" helm/Chart.yaml - helm plugin install https://github.com/chartmuseum/helm-push.git - helm repo add chartmuseum https://charts.ietf.org - helm cm-push --version="${{ env.PKG_VERSION }}" --username="${{ secrets.HELM_REPO_USERNAME }}" --password="${{ secrets.HELM_REPO_PASSWORD }}" helm/ chartmuseum - helm repo remove chartmuseum - - name: Update CHANGELOG id: changelog uses: Requarks/changelog-action@v1 From dd46a8af6f0998e0717d141f104cea70fe560577 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 10 May 2024 11:39:43 -0300 Subject: [PATCH 56/70] ci: use ietfa uid/gid for datatracker user (#7407) * ci: use ietfa uid/gid for datatracker user * chore: add comment --- dev/build/Dockerfile | 5 +++-- k8s/datatracker.yaml | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index d5578be3df..3d317f1416 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -3,8 +3,9 @@ LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive -RUN groupadd -g 1000 datatracker && \ - useradd -c "Datatracker User" -u 1000 -g datatracker -m -s /bin/false datatracker +# uid 498 = wwwrun and gid 496 = www on ietfa +RUN groupadd -g 496 datatracker && \ + useradd -c "Datatracker User" -u 498 -g datatracker -m -s /bin/false datatracker RUN apt-get purge -y imagemagick imagemagick-6-common diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 7ca92ba99e..303741daf1 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -64,8 +64,8 @@ spec: drop: - ALL readOnlyRootFilesystem: true - runAsUser: 1000 - runAsGroup: 1000 + runAsUser: 498 # wwwrun uid on ietfa + runAsGroup: 496 # www group on ietfa volumes: # To be overriden with the actual shared volume - name: dt-vol From 3a4939cc0bd00fd4e70d6f795df791eed9ed656a Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 10 May 2024 20:15:36 -0300 Subject: [PATCH 57/70] ci: collect memcached metrics (#7410) * ci: collect memcached metrics * ci: use nobody/nobody for metrics container --- k8s/memcached.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml index 56efc235ec..ba7c663634 100644 --- a/k8s/memcached.yaml +++ b/k8s/memcached.yaml @@ -16,6 +16,21 @@ spec: securityContext: runAsNonRoot: true containers: + - image: "quay.io/prometheus/memcached-exporter:v0.14.3" + imagePullPolicy: IfNotPresent + name: memcached-exporter + ports: + - name: metrics + containerPort: 9150 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 65534 # nobody + runAsGroup: 65534 # nobody - image: "memcached:1.6-alpine" imagePullPolicy: IfNotPresent args: ["-m", "1024"] @@ -41,6 +56,8 @@ apiVersion: v1 kind: Service metadata: name: memcached + annotations: + k8s.grafana.com/scrape: "yes" spec: type: ClusterIP ports: @@ -48,5 +65,9 @@ spec: targetPort: memcached protocol: TCP name: memcached + - port: 9150 + targetPort: metrics + protocol: TCP + name: metrics selector: app: memcached From 8b3d330bff7681330fb0ff2ade8fed19fa8e4ca7 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 10 May 2024 20:42:42 -0300 Subject: [PATCH 58/70] ci: metrics.portName annotation --- k8s/memcached.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml index ba7c663634..56f55d9b71 100644 --- a/k8s/memcached.yaml +++ b/k8s/memcached.yaml @@ -58,6 +58,7 @@ metadata: name: memcached annotations: k8s.grafana.com/scrape: "yes" + k8s.grafana.com/metrics.portName: "metrics" spec: type: ClusterIP ports: From 49a3cdc43c56fed14d4c5f39b8da4e20fe6c7c3b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 10 May 2024 21:00:38 -0300 Subject: [PATCH 59/70] ci: "true" --- k8s/memcached.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml index 56f55d9b71..e94066c9e0 100644 --- a/k8s/memcached.yaml +++ b/k8s/memcached.yaml @@ -57,7 +57,7 @@ kind: Service metadata: name: memcached annotations: - k8s.grafana.com/scrape: "yes" + k8s.grafana.com/scrape: "true" # this is not a bool k8s.grafana.com/metrics.portName: "metrics" spec: type: ClusterIP From c1a7a60eb92c312a523fa03d030bf3de0d33c8af Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 13 May 2024 11:43:32 -0300 Subject: [PATCH 60/70] ci: handle b64-encoded APP_API_TOKENS --- k8s/settings_local.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 00d0cae781..91bfe69e3d 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -151,7 +151,16 @@ def _multiline_to_list(s): "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set" ) -_APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None) +# For APP_API_TOKENS, ccept either base64-encoded JSON or raw JSON, but not both +if "DATATRACKER_APP_API_TOKENS_JSON_B64" in os.environ: + if "DATATRACKER_APP_API_TOKENS_JSON" in os.environ: + raise RuntimeError( + "Only one of DATATRACKER_APP_API_TOKENS_JSON and DATATRACKER_APP_API_TOKENS_JSON_B64 may be set" + ) + _APP_API_TOKENS_JSON = b64decode(os.environ.get("DATATRACKER_APP_API_TOKENS_JSON_B64")) +else: + _APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None) + if _APP_API_TOKENS_JSON is not None: APP_API_TOKENS = json.loads(_APP_API_TOKENS_JSON) else: From 5486345ab0fb111505793e116741c0bd7b37bfb9 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 13 May 2024 11:44:30 -0300 Subject: [PATCH 61/70] ci: remove unnecessary helper b64decode defaults to validate=false, which discards chars outside the base64 alphabet. That includes whitespace. --- k8s/settings_local.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/k8s/settings_local.py b/k8s/settings_local.py index 91bfe69e3d..3c77551a9d 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -9,11 +9,6 @@ from ietf.settings import * # pyflakes:ignore -def _remove_whitespace_and_b64decode(s): - """Helper to strip out whitespace and base64 decode""" - return b64decode("".join(s.split())) - - def _multiline_to_list(s): """Helper to split at newlines and conver to list""" return [item.strip() for item in s.split("\n")] @@ -31,7 +26,7 @@ def _multiline_to_list(s): _NOMCOM_APP_SECRET_B64 = os.environ.get("DATATRACKER_NOMCOM_APP_SECRET_B64", None) if _NOMCOM_APP_SECRET_B64 is not None: - NOMCOM_APP_SECRET = _remove_whitespace_and_b64decode(_NOMCOM_APP_SECRET_B64) + NOMCOM_APP_SECRET = b64decode(_NOMCOM_APP_SECRET_B64) else: raise RuntimeError("DATATRACKER_NOMCOM_APP_SECRET_B64 must be set") @@ -67,13 +62,13 @@ def _multiline_to_list(s): _API_PUBLIC_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PUBLIC_KEY_PEM_B64", None) if _API_PUBLIC_KEY_PEM_B64 is not None: - API_PUBLIC_KEY_PEM = _remove_whitespace_and_b64decode(_API_PUBLIC_KEY_PEM_B64) + API_PUBLIC_KEY_PEM = b64decode(_API_PUBLIC_KEY_PEM_B64) else: raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set") _API_PRIVATE_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PRIVATE_KEY_PEM_B64", None) if _API_PRIVATE_KEY_PEM_B64 is not None: - API_PRIVATE_KEY_PEM = _remove_whitespace_and_b64decode(_API_PRIVATE_KEY_PEM_B64) + API_PRIVATE_KEY_PEM = b64decode(_API_PRIVATE_KEY_PEM_B64) else: raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") From 212a915bd38ae51e2e6a1cd7932afc09fdbd2038 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 13 May 2024 11:55:45 -0300 Subject: [PATCH 62/70] ci: don't set APP_API_TOKENS in django-config --- k8s/django-config.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/k8s/django-config.yaml b/k8s/django-config.yaml index e6f5b3ca8f..9ece6fa9ee 100644 --- a/k8s/django-config.yaml +++ b/k8s/django-config.yaml @@ -66,7 +66,9 @@ data: CELERY_PASSWORD: "this-is-a-secret" # secret - DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret + # Only one of these may be set + # DATATRACKER_APP_API_TOKENS_JSON_B64: "e30K" # secret + # DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret # use this to override default - one entry per line # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- From b744f2b3181386084a8a9aac9423c7d55a8274df Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 13 May 2024 13:15:31 -0300 Subject: [PATCH 63/70] chore: switch back to uid/gid=1000 --- dev/build/Dockerfile | 4 ++-- k8s/datatracker.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 3d317f1416..12f73300dd 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -4,8 +4,8 @@ LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive # uid 498 = wwwrun and gid 496 = www on ietfa -RUN groupadd -g 496 datatracker && \ - useradd -c "Datatracker User" -u 498 -g datatracker -m -s /bin/false datatracker +RUN groupadd -g 1000 datatracker && \ + useradd -c "Datatracker User" -u 1000 -g datatracker -m -s /bin/false datatracker RUN apt-get purge -y imagemagick imagemagick-6-common diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 303741daf1..7ca92ba99e 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -64,8 +64,8 @@ spec: drop: - ALL readOnlyRootFilesystem: true - runAsUser: 498 # wwwrun uid on ietfa - runAsGroup: 496 # www group on ietfa + runAsUser: 1000 + runAsGroup: 1000 volumes: # To be overriden with the actual shared volume - name: dt-vol From 38b0b2c0352ba94932691574552a6bf2ed83da07 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 13 May 2024 17:51:52 -0300 Subject: [PATCH 64/70] feat: get tool versions without VersionInfo model (#7393) (#7403) * chore: remove update_external_command_info call * feat: get tool version without VersionInfo * chore: Remove VersionInfo model * chore: Migration to remove VersionInfo * fix: handle errors better; ignore stderr --- bin/daily | 3 -- bin/dump-to-names-json | 2 +- ietf/name/fixtures/names.json | 44 ------------------- .../commands/generate_name_fixture.py | 3 +- ietf/submit/checkers.py | 24 ++++++---- ietf/submit/tests.py | 4 +- ietf/utils/__init__.py | 30 ++++++++++++- ietf/utils/admin.py | 7 --- .../commands/update_external_command_info.py | 41 ----------------- .../migrations/0002_delete_versioninfo.py | 16 +++++++ ietf/utils/models.py | 9 ---- ietf/utils/resources.py | 20 +-------- 12 files changed, 66 insertions(+), 137 deletions(-) delete mode 100644 ietf/utils/management/commands/update_external_command_info.py create mode 100644 ietf/utils/migrations/0002_delete_versioninfo.py diff --git a/bin/daily b/bin/daily index 8211e1e237..6557a89222 100755 --- a/bin/daily +++ b/bin/daily @@ -24,9 +24,6 @@ $DTDIR/bin/hourly source $DTDIR/env/bin/activate -# Update our information about the current version of some commands we use -$DTDIR/ietf/manage.py update_external_command_info - # Get IANA-registered yang models #YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR') # Hardcode the rsync target to avoid any unwanted deletes: diff --git a/bin/dump-to-names-json b/bin/dump-to-names-json index 9c7dfac07d..20d4e0f951 100644 --- a/bin/dump-to-names-json +++ b/bin/dump-to-names-json @@ -10,7 +10,7 @@ set -x ietf/manage.py dumpdata --indent 1 doc.State doc.BallotType doc.StateType \ - mailtrigger.MailTrigger mailtrigger.Recipient name utils.VersionInfo \ + mailtrigger.MailTrigger mailtrigger.Recipient name \ group.GroupFeatures stats.CountryAlias dbtemplate.DBTemplate \ | jq --sort-keys "sort_by(.model, .pk)" \ | jq '[.[] | select(.model!="dbtemplate.dbtemplate" or .pk==354)]' > ietf/name/fixtures/names.json diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index 35679dcaa1..913c6c987e 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -16789,49 +16789,5 @@ }, "model": "stats.countryalias", "pk": 303 - }, - { - "fields": { - "command": "xym", - "switch": "--version", - "time": "2024-03-21T07:06:23.405Z", - "used": true, - "version": "xym 0.7.0" - }, - "model": "utils.versioninfo", - "pk": 1 - }, - { - "fields": { - "command": "pyang", - "switch": "--version", - "time": "2024-03-21T07:06:23.755Z", - "used": true, - "version": "pyang 2.6.0" - }, - "model": "utils.versioninfo", - "pk": 2 - }, - { - "fields": { - "command": "yanglint", - "switch": "--version", - "time": "2024-03-21T07:06:23.773Z", - "used": true, - "version": "yanglint SO 1.9.2" - }, - "model": "utils.versioninfo", - "pk": 3 - }, - { - "fields": { - "command": "xml2rfc", - "switch": "--version", - "time": "2024-03-21T07:06:24.609Z", - "used": true, - "version": "xml2rfc 3.20.1" - }, - "model": "utils.versioninfo", - "pk": 4 } ] diff --git a/ietf/name/management/commands/generate_name_fixture.py b/ietf/name/management/commands/generate_name_fixture.py index bbf33e600e..ef30e54c73 100644 --- a/ietf/name/management/commands/generate_name_fixture.py +++ b/ietf/name/management/commands/generate_name_fixture.py @@ -77,7 +77,6 @@ def output(seq): from ietf.mailtrigger.models import MailTrigger, Recipient from ietf.meeting.models import BusinessConstraint from ietf.stats.models import CountryAlias - from ietf.utils.models import VersionInfo # Grab all ietf.name.models for n in dir(ietf.name.models): @@ -87,7 +86,7 @@ def output(seq): model_objects[model_name(item)] = list(item.objects.all().order_by('pk')) for m in ( BallotType, State, StateType, GroupFeatures, MailTrigger, Recipient, - CountryAlias, VersionInfo, BusinessConstraint ): + CountryAlias, BusinessConstraint ): model_objects[model_name(m)] = list(m.objects.all().order_by('pk')) for m in ( DBTemplate, ): diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py index 5822f155f5..d29e2a2355 100644 --- a/ietf/submit/checkers.py +++ b/ietf/submit/checkers.py @@ -14,8 +14,8 @@ import debug # pyflakes:ignore +from ietf.utils import tool_version from ietf.utils.log import log, assertion -from ietf.utils.models import VersionInfo from ietf.utils.pipe import pipe from ietf.utils.test_runner import set_coverage_checking @@ -177,8 +177,10 @@ def check_file_txt(self, path): model_list = list(set(model_list)) command = "xym" - cmd_version = VersionInfo.objects.get(command=command).version - message = "%s:\n%s\n\n" % (cmd_version, out.replace('\n\n','\n').strip() if code == 0 else err) + message = "{version}:\n{output}\n\n".format( + version=tool_version[command], + output=out.replace('\n\n', '\n').strip() if code == 0 else err, + ) results.append({ "name": name, @@ -209,7 +211,6 @@ def check_file_txt(self, path): # pyang cmd_template = settings.SUBMIT_PYANG_COMMAND command = [ w for w in cmd_template.split() if not '=' in w ][0] - cmd_version = VersionInfo.objects.get(command=command).version cmd = cmd_template.format(libs=modpath, model=path) venv_path = os.environ.get('VIRTUAL_ENV') or os.path.join(os.getcwd(), 'env') venv_bin = os.path.join(venv_path, 'bin') @@ -238,14 +239,17 @@ def check_file_txt(self, path): except ValueError: pass #passed = passed and code == 0 # For the submission tool. Yang checks always pass - message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if (code == 0 and len(err) == 0) else out+err) + message += "{version}: {template}:\n{output}\n".format( + version=tool_version[command], + template=cmd_template, + output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, + ) # yanglint set_coverage_checking(False) # we can't count the following as it may or may not be run, depending on setup if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY): cmd_template = settings.SUBMIT_YANGLINT_COMMAND command = [ w for w in cmd_template.split() if not '=' in w ][0] - cmd_version = VersionInfo.objects.get(command=command).version cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir, draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, ) @@ -264,7 +268,11 @@ def check_file_txt(self, path): except ValueError: pass #passed = passed and code == 0 # For the submission tool. Yang checks always pass - message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if (code == 0 and len(err) == 0) else out+err) + message += "{version}: {template}:\n{output}\n".format( + version=tool_version[command], + template=cmd_template, + output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, + ) set_coverage_checking(True) else: errors += 1 @@ -293,4 +301,4 @@ def check_file_txt(self, path): items = [ e for res in results for e in res["items"] ] info['items'] = items info['code']['yang'] = model_list - return passed, message, errors, warnings, info \ No newline at end of file + return passed, message, errors, warnings, info diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 08b898c13a..58a47aef8b 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -49,9 +49,9 @@ from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm from ietf.submit.models import Submission, Preapproval, SubmissionExtResource from ietf.submit.tasks import cancel_stale_submissions, process_and_accept_uploaded_submission_task +from ietf.utils import tool_version from ietf.utils.accesstoken import generate_access_token from ietf.utils.mail import outbox, get_payload_text -from ietf.utils.models import VersionInfo from ietf.utils.test_utils import login_testing_unauthorized, TestCase from ietf.utils.timezone import date_today from ietf.utils.draft import PlaintextDraft @@ -1854,7 +1854,7 @@ def test_submit_invalid_yang(self): # m = q('#yang-validation-message').text() for command in ['xym', 'pyang', 'yanglint']: - version = VersionInfo.objects.get(command=command).version + version = tool_version[command] if command != 'yanglint' or (settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY)): self.assertIn(version, m) self.assertIn("draft-yang-testing-invalid-00.txt", m) diff --git a/ietf/utils/__init__.py b/ietf/utils/__init__.py index 7f1df97602..6a7787955b 100644 --- a/ietf/utils/__init__.py +++ b/ietf/utils/__init__.py @@ -1 +1,29 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2024, All Rights Reserved +import subprocess + + +class _ToolVersionManager: + _known = [ + "pyang", + "xml2rfc", + "xym", + "yanglint", + ] + _versions = dict() + + def __getitem__(self, item): + if item not in self._known: + return "Unknown" + elif item not in self._versions: + try: + self._versions[item] = subprocess.run( + [item, "--version"], + capture_output=True, + check=True, + ).stdout.decode().strip() + except subprocess.CalledProcessError: + return "Unknown" + return self._versions[item] + + +tool_version = _ToolVersionManager() diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py index fa1ebb7081..6c1c8726e1 100644 --- a/ietf/utils/admin.py +++ b/ietf/utils/admin.py @@ -5,8 +5,6 @@ from django.contrib import admin from django.utils.encoding import force_str -from ietf.utils.models import VersionInfo - def name(obj): if hasattr(obj, 'abbrev'): return obj.abbrev() @@ -58,8 +56,3 @@ class DumpInfoAdmin(admin.ModelAdmin): list_display = ['date', 'host', 'tz'] list_filter = ['date'] admin.site.register(DumpInfo, DumpInfoAdmin) - -class VersionInfoAdmin(admin.ModelAdmin): - list_display = ['command', 'switch', 'version', 'time', ] -admin.site.register(VersionInfo, VersionInfoAdmin) - diff --git a/ietf/utils/management/commands/update_external_command_info.py b/ietf/utils/management/commands/update_external_command_info.py deleted file mode 100644 index e9e24f000d..0000000000 --- a/ietf/utils/management/commands/update_external_command_info.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright The IETF Trust 2017-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -import sys - -from textwrap import dedent - -from django.core.management.base import BaseCommand - -import debug # pyflakes:ignore - -from ietf.utils.models import VersionInfo -from ietf.utils.pipe import pipe - -class Command(BaseCommand): - """ - Update the version information for external commands used by the datatracker. - - Iterates through the entries in the VersionInfo table, runs the relevant - command, and updates the version string with the result. - - """ - - help = dedent(__doc__).strip() - - def handle(self, *filenames, **options): - for c in VersionInfo.objects.filter(used=True): - cmd = "%s %s" % (c.command, c.switch) - code, out, err = pipe(cmd) - out = out.decode('utf-8') - err = err.decode('utf-8') - if code != 0: - sys.stderr.write("Command '%s' returned %s: \n%s\n%s\n" % (cmd, code, out, err)) - else: - c.version = (out.strip()+'\n'+err.strip()).strip() - if options.get('verbosity', 1) > 1: - sys.stdout.write( - "Command: %s\n" - " Version: %s\n" % (cmd, c.version)) - c.save() diff --git a/ietf/utils/migrations/0002_delete_versioninfo.py b/ietf/utils/migrations/0002_delete_versioninfo.py new file mode 100644 index 0000000000..2835bb017b --- /dev/null +++ b/ietf/utils/migrations/0002_delete_versioninfo.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.11 on 2024-05-03 21:03 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("utils", "0001_initial"), + ] + + operations = [ + migrations.DeleteModel( + name="VersionInfo", + ), + ] diff --git a/ietf/utils/models.py b/ietf/utils/models.py index 0915537fd8..21af5766e9 100644 --- a/ietf/utils/models.py +++ b/ietf/utils/models.py @@ -9,15 +9,6 @@ class DumpInfo(models.Model): host = models.CharField(max_length=128) tz = models.CharField(max_length=32, default='UTC') -class VersionInfo(models.Model): - time = models.DateTimeField(auto_now=True) - command = models.CharField(max_length=32) - switch = models.CharField(max_length=16) - version = models.CharField(max_length=64) - used = models.BooleanField(default=True) - class Meta: - verbose_name_plural = 'VersionInfo' - class ForeignKey(models.ForeignKey): "A local ForeignKey proxy which provides the on_delete value required under Django 2.0." def __init__(self, to, on_delete=models.CASCADE, **kwargs): diff --git a/ietf/utils/resources.py b/ietf/utils/resources.py index 6d61c5e2ed..1252cfef14 100644 --- a/ietf/utils/resources.py +++ b/ietf/utils/resources.py @@ -12,7 +12,7 @@ from django.contrib.contenttypes.models import ContentType from ietf import api -from ietf.utils.models import DumpInfo, VersionInfo +from ietf.utils.models import DumpInfo class UserResource(ModelResource): @@ -43,21 +43,3 @@ class Meta: "host": ALL, } api.utils.register(DumpInfoResource()) - - -class VersionInfoResource(ModelResource): - class Meta: - queryset = VersionInfo.objects.all() - serializer = api.Serializer() - cache = SimpleCache() - #resource_name = 'versioninfo' - ordering = ['id', ] - filtering = { - "id": ALL, - "time": ALL, - "command": ALL, - "switch": ALL, - "version": ALL, - "used": ALL, - } -api.utils.register(VersionInfoResource()) From 635ca28ec1412f095dac09077719071cdf406dfb Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Mon, 13 May 2024 21:40:13 -0400 Subject: [PATCH 65/70] chore: remove helm stuff --- .github/workflows/build.yml | 2 +- helm/.helmignore | 23 - helm/Chart.yaml | 23 - helm/settings_local.py | 259 ------- helm/templates/_helpers.tpl | 179 ----- helm/templates/configmap.yaml | 69 -- helm/templates/deployments/beat.yaml | 75 -- helm/templates/deployments/celery.yaml | 90 --- helm/templates/deployments/datatracker.yaml | 90 --- helm/templates/deployments/memcached.yaml | 74 -- helm/templates/hpa.yaml | 32 - helm/templates/ingress.yaml | 61 -- helm/templates/persistentvolumeclaims.yaml | 43 -- helm/templates/rbac/beat-serviceaccount.yaml | 12 - .../templates/rbac/celery-serviceaccount.yaml | 12 - .../rbac/datatracker-serviceaccount.yaml | 12 - .../rbac/memcached-serviceaccount.yaml | 12 - .../rbac/rabbitmq-serviceaccount.yaml | 12 - helm/templates/services/datatracker.yaml | 21 - helm/templates/services/memcached.yaml | 21 - helm/templates/services/rabbitmq.yaml | 22 - helm/templates/statefulsets/rabbitmq.yaml | 87 --- helm/values.yaml | 645 ------------------ 23 files changed, 1 insertion(+), 1875 deletions(-) delete mode 100644 helm/.helmignore delete mode 100644 helm/Chart.yaml delete mode 100644 helm/settings_local.py delete mode 100644 helm/templates/_helpers.tpl delete mode 100644 helm/templates/configmap.yaml delete mode 100644 helm/templates/deployments/beat.yaml delete mode 100644 helm/templates/deployments/celery.yaml delete mode 100644 helm/templates/deployments/datatracker.yaml delete mode 100644 helm/templates/deployments/memcached.yaml delete mode 100644 helm/templates/hpa.yaml delete mode 100644 helm/templates/ingress.yaml delete mode 100644 helm/templates/persistentvolumeclaims.yaml delete mode 100644 helm/templates/rbac/beat-serviceaccount.yaml delete mode 100644 helm/templates/rbac/celery-serviceaccount.yaml delete mode 100644 helm/templates/rbac/datatracker-serviceaccount.yaml delete mode 100644 helm/templates/rbac/memcached-serviceaccount.yaml delete mode 100644 helm/templates/rbac/rabbitmq-serviceaccount.yaml delete mode 100644 helm/templates/services/datatracker.yaml delete mode 100644 helm/templates/services/memcached.yaml delete mode 100644 helm/templates/services/rabbitmq.yaml delete mode 100644 helm/templates/statefulsets/rabbitmq.yaml delete mode 100644 helm/values.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd65da036c..4181beef4a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -220,7 +220,7 @@ jobs: .devcontainer .github .vscode - helm + k8s playwright svn-history docker-compose.yml diff --git a/helm/.helmignore b/helm/.helmignore deleted file mode 100644 index 691fa13d6a..0000000000 --- a/helm/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ \ No newline at end of file diff --git a/helm/Chart.yaml b/helm/Chart.yaml deleted file mode 100644 index 844087fa69..0000000000 --- a/helm/Chart.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v2 -name: datatracker -description: The day-to-day front-end to the IETF database for people who work on IETF standards. -home: https://datatracker.ietf.org -sources: - - https://github.com/ietf-tools/datatracker -maintainers: - - name: IETF Tools Team - email: tools-discuss@ietf.org - url: https://github.com/ietf-tools - -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.0.0" \ No newline at end of file diff --git a/helm/settings_local.py b/helm/settings_local.py deleted file mode 100644 index 53609afa79..0000000000 --- a/helm/settings_local.py +++ /dev/null @@ -1,259 +0,0 @@ -# Copyright The IETF Trust 2007-2024, All Rights Reserved -# -*- coding: utf-8 -*- - -from base64 import b64decode -from email.utils import parseaddr -import json - -from ietf import __release_hash__ -from ietf.settings import * # pyflakes:ignore - - -def _remove_whitespace_and_b64decode(s): - """Helper to strip out whitespace and base64 decode""" - return b64decode("".join(s.split())) - - -def _multiline_to_list(s): - """Helper to split at newlines and conver to list""" - return [item.strip() for item in s.split("\n")] - - -# Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env! -SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development") - -# Secrets -_SECRET_KEY = os.environ.get("DATATRACKER_DJANGO_SECRET_KEY", None) -if _SECRET_KEY is not None: - SECRET_KEY = _SECRET_KEY -else: - raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set") - -_NOMCOM_APP_SECRET_B64 = os.environ.get("DATATRACKER_NOMCOM_APP_SECRET_B64", None) -if _NOMCOM_APP_SECRET_B64 is not None: - NOMCOM_APP_SECRET = _remove_whitespace_and_b64decode(_NOMCOM_APP_SECRET_B64) -else: - raise RuntimeError("DATATRACKER_NOMCOM_APP_SECRET_B64 must be set") - -_IANA_SYNC_PASSWORD = os.environ.get("DATATRACKER_IANA_SYNC_PASSWORD", None) -if _IANA_SYNC_PASSWORD is not None: - IANA_SYNC_PASSWORD = _IANA_SYNC_PASSWORD -else: - raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set") - -_RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD", None) -if _RFC_EDITOR_SYNC_PASSWORD is not None: - RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD") -else: - raise RuntimeError("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD must be set") - -_YOUTUBE_API_KEY = os.environ.get("DATATRACKER_YOUTUBE_API_KEY", None) -if _YOUTUBE_API_KEY is not None: - YOUTUBE_API_KEY = _YOUTUBE_API_KEY -else: - raise RuntimeError("DATATRACKER_YOUTUBE_API_KEY must be set") - -_GITHUB_BACKUP_API_KEY = os.environ.get("DATATRACKER_GITHUB_BACKUP_API_KEY", None) -if _GITHUB_BACKUP_API_KEY is not None: - GITHUB_BACKUP_API_KEY = _GITHUB_BACKUP_API_KEY -else: - raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set") - -_API_KEY_TYPE = os.environ.get("DATATRACKER_API_KEY_TYPE", None) -if _API_KEY_TYPE is not None: - API_KEY_TYPE = _API_KEY_TYPE -else: - raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set") - -_API_PUBLIC_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PUBLIC_KEY_PEM_B64", None) -if _API_PUBLIC_KEY_PEM_B64 is not None: - API_PUBLIC_KEY_PEM = _remove_whitespace_and_b64decode(_API_PUBLIC_KEY_PEM_B64) -else: - raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set") - -_API_PRIVATE_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PRIVATE_KEY_PEM_B64", None) -if _API_PRIVATE_KEY_PEM_B64 is not None: - API_PRIVATE_KEY_PEM = _remove_whitespace_and_b64decode(_API_PRIVATE_KEY_PEM_B64) -else: - raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") - -# Set DEBUG if DATATRACKER_DEBUG env var is the word "true" -DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true" - -# DATATRACKER_ALLOWED_HOSTS env var is a comma-separated list of allowed hosts -_allowed_hosts_str = os.environ.get("DATATRACKER_ALLOWED_HOSTS", None) -if _allowed_hosts_str is not None: - ALLOWED_HOSTS = _multiline_to_list(_allowed_hosts_str) - -DATABASES = { - "default": { - "HOST": os.environ.get("DATATRACKER_DBHOST", "db"), - "PORT": os.environ.get("DATATRACKER_DBPORT", "5432"), - "NAME": os.environ.get("DATATRACKER_DBNAME", "datatracker"), - "ENGINE": "django.db.backends.postgresql", - "USER": os.environ.get("DATATRACKER_DBUSER", "django"), - "PASSWORD": os.environ.get("DATATRACKER_DBPASS", ""), - }, -} - -# DATATRACKER_ADMINS is a newline-delimited list of addresses parseable by email.utils.parseaddr -_admins_str = os.environ.get("DATATRACKER_ADMINS", None) -if _admins_str is not None: - ADMINS = [parseaddr(admin) for admin in _multiline_to_list(_admins_str)] -else: - raise RuntimeError("DATATRACKER_ADMINS must be set") - -USING_DEBUG_EMAIL_SERVER = os.environ.get("DATATRACKER_EMAIL_DEBUG", "false").lower() == "true" -EMAIL_HOST = os.environ.get("DATATRACKER_EMAIL_HOST", "localhost") -EMAIL_PORT = int(os.environ.get("DATATRACKER_EMAIL_PORT", "2025")) - -_celery_password = os.environ.get("CELERY_PASSWORD", None) -if _celery_password is None: - raise RuntimeError("CELERY_PASSWORD must be set") -CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( - host=os.environ.get("RABBITMQ_HOSTNAME", "rabbitmq"), - password=_celery_password, - queue=os.environ.get("RABBITMQ_QUEUE", "dt") -) - -IANA_SYNC_USERNAME = "ietfsync" -IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" -IANA_SYNC_PROTOCOLS_URL = "http://www.iana.org/protocols/" - -RFC_EDITOR_NOTIFICATION_URL = "http://www.rfc-editor.org/parser/parser.php" - -STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://registration.ietf.org/{number}/attendees/?apikey=redacted' - -#FIRST_CUTOFF_DAYS = 12 -#SECOND_CUTOFF_DAYS = 12 -#SUBMISSION_CUTOFF_DAYS = 26 -#SUBMISSION_CORRECTION_DAYS = 57 -MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26 -MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 54 - -HTPASSWD_COMMAND = "/usr/bin/htpasswd2" - -_MEETECHO_CLIENT_ID = os.environ.get("DATATRACKER_MEETECHO_CLIENT_ID", None) -_MEETECHO_CLIENT_SECRET = os.environ.get("DATATRACKER_MEETECHO_CLIENT_SECRET", None) -if _MEETECHO_CLIENT_ID is not None and _MEETECHO_CLIENT_SECRET is not None: - MEETECHO_API_CONFIG = { - "api_base": os.environ.get( - "DATATRACKER_MEETECHO_API_BASE", - "https://meetings.conf.meetecho.com/api/v1/", - ), - "client_id": _MEETECHO_CLIENT_ID, - "client_secret": _MEETECHO_CLIENT_SECRET, - "request_timeout": 3.01, # python-requests doc recommend slightly > a multiple of 3 seconds - } -else: - raise RuntimeError( - "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set" - ) - -_APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None) -if _APP_API_TOKENS_JSON is not None: - APP_API_TOKENS = json.loads(_APP_API_TOKENS_JSON) -else: - APP_API_TOKENS = {} - -EMAIL_COPY_TO = "" - -# Until we teach the datatracker to look beyond cloudflare for this check -IDSUBMIT_MAX_DAILY_SAME_SUBMITTER = 5000 - -# Leave DATATRACKER_MATOMO_SITE_ID unset to disable Matomo reporting -if "DATATRACKER_MATOMO_SITE_ID" in os.environ: - MATOMO_DOMAIN_PATH = os.environ.get("DATATRACKER_MATOMO_DOMAIN_PATH", "analytics.ietf.org") - MATOMO_SITE_ID = os.environ.get("DATATRACKER_MATOMO_SITE_ID") - MATOMO_DISABLE_COOKIES = True - -# Leave DATATRACKER_SCOUT_KEY unset to disable Scout APM agent -_SCOUT_KEY = os.environ.get("DATATRACKER_SCOUT_KEY", None) -if _SCOUT_KEY is not None: - if SERVER_MODE == "production": - PROD_PRE_APPS = ["scout_apm.django", ] - else: - DEV_PRE_APPS = ["scout_apm.django", ] - SCOUT_MONITOR = True - SCOUT_KEY = _SCOUT_KEY - SCOUT_NAME = os.environ.get("DATATRACKER_SCOUT_NAME", "Datatracker") - SCOUT_ERRORS_ENABLED = True - SCOUT_SHUTDOWN_MESSAGE_ENABLED = False - SCOUT_CORE_AGENT_SOCKET_PATH = "tcp://{host}:{port}".format( - host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "localhost"), - port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "6590"), - ) - SCOUT_CORE_AGENT_DOWNLOAD = False - SCOUT_CORE_AGENT_LAUNCH = False - SCOUT_REVISION_SHA = __release_hash__[:7] - -# Path to the email alias lists. Used by ietf.utils.aliases -DRAFT_ALIASES_PATH = "/a/postfix/draft-aliases" -DRAFT_VIRTUAL_PATH = "/a/postfix/draft-virtual" -GROUP_ALIASES_PATH = "/a/postfix/group-aliases" -GROUP_VIRTUAL_PATH = "/a/postfix/group-virtual" - -STATIC_URL = os.environ.get("DATATRACKER_STATIC_URL", None) -if STATIC_URL is None: - from ietf import __version__ - STATIC_URL = f"https://static.ietf.org/dt/{__version__}/" - -# Set these to the same as "production" in settings.py, whether production mode or not -MEDIA_ROOT = "/a/www/www6s/lib/dt/media/" -MEDIA_URL = "https://www.ietf.org/lib/dt/media/" -PHOTOS_DIRNAME = "photo" -PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME - -# Normally only set for debug, but needed until we have a real FS -DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') - -# Binaries that are different in the docker image -DE_GFM_BINARY = "/usr/local/bin/de-gfm" -IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" - -# Duplicating production cache from settings.py and using it whether we're in production mode or not -MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") -MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") -from ietf import __version__ -CACHES = { - "default": { - "BACKEND": "ietf.utils.cache.LenientMemcacheCache", - "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", - "VERSION": __version__, - "KEY_PREFIX": "ietf:dt", - "KEY_FUNCTION": lambda key, key_prefix, version: ( - f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" - ), - }, - "sessions": { - "BACKEND": "ietf.utils.cache.LenientMemcacheCache", - "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", - # No release-specific VERSION setting. - "KEY_PREFIX": "ietf:dt", - }, - "htmlized": { - "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "/a/cache/datatracker/htmlized", - "OPTIONS": { - "MAX_ENTRIES": 100000, # 100,000 - }, - }, - "pdfized": { - "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "/a/cache/datatracker/pdfized", - "OPTIONS": { - "MAX_ENTRIES": 100000, # 100,000 - }, - }, - "slowpages": { - "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", - "LOCATION": "/a/cache/datatracker/slowpages", - "OPTIONS": { - "MAX_ENTRIES": 5000, - }, - }, -} - -_csrf_trusted_origins_str = os.environ.get("DATATRACKER_CSRF_TRUSTED_ORIGINS") -if _csrf_trusted_origins_str is not None: - CSRF_TRUSTED_ORIGINS = _multiline_to_list(_csrf_trusted_origins_str) diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl deleted file mode 100644 index 503b3f1bb8..0000000000 --- a/helm/templates/_helpers.tpl +++ /dev/null @@ -1,179 +0,0 @@ -{{/* - Expand the name of the chart. - */}} -{{- define "datatracker.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "datatracker.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create a fully qualified datatracker name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.datatracker.fullname" -}} -{{- if .Values.datatracker.fullnameOverride -}} -{{- .Values.datatracker.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.datatracker.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified celery name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.celery.fullname" -}} -{{- if .Values.celery.fullnameOverride -}} -{{- .Values.celery.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.celery.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.celery.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified celery name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.beat.fullname" -}} -{{- if .Values.beat.fullnameOverride -}} -{{- .Values.beat.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.beat.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.beat.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified rabbitmq name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.rabbitmq.fullname" -}} -{{- if .Values.rabbitmq.fullnameOverride -}} -{{- .Values.rabbitmq.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.rabbitmq.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create a fully qualified memcached name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -*/}} -{{- define "datatracker.memcached.fullname" -}} -{{- if .Values.memcached.fullnameOverride -}} -{{- .Values.memcached.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- printf "%s-%s" .Release.Name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s-%s" .Release.Name $name .Values.memcached.name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "datatracker.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "datatracker.commonLabels" -}} -helm.sh/chart: {{ include "datatracker.chart" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -app.kubernetes.io/instance: {{ .Release.Name }} -app.kubernetes.io/part-of: {{ include "datatracker.name" . | default "datatracker" }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "datatracker.selectorLabels" -}} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "datatracker.serviceAccountName.datatracker" -}} -{{- if .Values.serviceAccounts.datatracker.create -}} - {{ default (include "datatracker.datatracker.fullname" .) .Values.serviceAccounts.datatracker.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.datatracker.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.celery" -}} -{{- if .Values.serviceAccounts.celery.create -}} - {{ default (include "datatracker.celery.fullname" .) .Values.serviceAccounts.celery.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.celery.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.beat" -}} -{{- if .Values.serviceAccounts.beat.create -}} - {{ default (include "datatracker.beat.fullname" .) .Values.serviceAccounts.beat.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.beat.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.rabbitmq" -}} -{{- if .Values.serviceAccounts.rabbitmq.create -}} - {{ default (include "datatracker.rabbitmq.fullname" .) .Values.serviceAccounts.rabbitmq.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.rabbitmq.name }} -{{- end -}} -{{- end }} - -{{- define "datatracker.serviceAccountName.memcached" -}} -{{- if .Values.serviceAccounts.memcached.create -}} - {{ default (include "datatracker.memcached.fullname" .) .Values.serviceAccounts.memcached.name }} -{{- else -}} - {{ default "default" .Values.serviceAccounts.memcached.name }} -{{- end -}} -{{- end }} diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml deleted file mode 100644 index 06cda4a04b..0000000000 --- a/helm/templates/configmap.yaml +++ /dev/null @@ -1,69 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: django-configmap -data: - settings_local.py: |- - {{- .Files.Get "settings_local.py" | nindent 4 }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: rabbitmq-configmap -data: - definitions.json: |- - { - "permissions": [ - { - "configure": ".*", - "read": ".*", - "user": "datatracker", - "vhost": "dt", - "write": ".*" - } - ], - "users": [ - { - "hashing_algorithm": "rabbit_password_hashing_sha256", - "limits": {}, - "name": "datatracker", - "password_hash": "HJxcItcpXtBN+R/CH7dUelfKBOvdUs3AWo82SBw2yLMSguzb", - "tags": [] - } - ], - "vhosts": [ - { - "limits": [], - "metadata": { - "description": "", - "tags": [] - }, - "name": "dt" - } - ] - } - rabbitmq.conf: |- - # prevent guest from logging in over tcp - loopback_users.guest = true - - # load saved definitions - load_definitions = /etc/rabbitmq/definitions.json - - # Ensure that enough disk is available to flush to disk. To do this, need to limit the - # memory available to the container to something reasonable. See - # https://www.rabbitmq.com/production-checklist.html#monitoring-and-resource-usage - # for recommendations. - - # 1-1.5 times the memory available to the container is adequate for disk limit - disk_free_limit.absolute = 6000MB - - # This should be ~40% of the memory available to the container. Use an - # absolute number because relative will be proprtional to the full machine - # memory. - vm_memory_high_watermark.absolute = 1600MB - - # Logging - log.file = false - log.console = true - log.console.level = info - log.console.formatter = json diff --git a/helm/templates/deployments/beat.yaml b/helm/templates/deployments/beat.yaml deleted file mode 100644 index 83347481cd..0000000000 --- a/helm/templates/deployments/beat.yaml +++ /dev/null @@ -1,75 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.beat.fullname" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - app.kubernetes.io/name: {{ .Values.beat.name }} -spec: - {{- $podValues := .Values.beat }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - app.kubernetes.io/name: {{ $podValues.name }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - app.kubernetes.io/name: {{ $podValues.name }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.beat" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ default $.Values.datatracker.image.repository $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" - imagePullPolicy: {{ default "IfNotPresent" (default $.Values.datatracker.image.imagePullPolicy $podValues.image.imagePullPolicy) }} - env: - - name: "CONTAINER_ROLE" - value: "beat" - {{- if .Values.env }} - {{- range $key, $val := .Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/templates/deployments/celery.yaml b/helm/templates/deployments/celery.yaml deleted file mode 100644 index 5c824aa2d0..0000000000 --- a/helm/templates/deployments/celery.yaml +++ /dev/null @@ -1,90 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.celery.fullname" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - app.kubernetes.io/name: {{ .Values.celery.name }} -spec: - {{- $podValues := .Values.celery }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - app.kubernetes.io/name: {{ $podValues.name }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - app.kubernetes.io/name: {{ $podValues.name }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.celery" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - {{- if .Values.scoutapm }} - - name: "scoutapm" - image: "{{ .Values.scoutapm.image.repository }}:{{ default "latest" .Values.scoutapm.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.scoutapm.image.imagePullPolicy }} - livenessProbe: - exec: - command: - - "sh" - - "-c" - - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" - securityContext: - readOnlyRootFilesystem: {{ default true .Values.scoutapm.readOnlyRootFilesystem }} - runAsUser: {{ default 65534 .Values.scoutapm.runAsUser }} # "nobody" user by default - runAsGroup: {{ default 65534 .Values.scoutapm.runAsGroup }} # "nogroup" group by default - {{- end }} - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ default $.Values.datatracker.image.repository $podValues.image.repository }}:{{ default .Chart.AppVersion (default $.Values.datatracker.image.tag $podValues.image.tag) }}" - imagePullPolicy: {{ default "IfNotPresent" (default $.Values.datatracker.image.imagePullPolicy $podValues.image.imagePullPolicy) }} - env: - - name: "CONTAINER_ROLE" - value: "celery" - {{- if .Values.env }} - {{- range $key, $val := .Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/templates/deployments/datatracker.yaml b/helm/templates/deployments/datatracker.yaml deleted file mode 100644 index acd54af97a..0000000000 --- a/helm/templates/deployments/datatracker.yaml +++ /dev/null @@ -1,90 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.datatracker.fullname" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - app.kubernetes.io/name: {{ .Values.datatracker.name }} -spec: - {{- $podValues := .Values.datatracker }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - app.kubernetes.io/name: {{ $podValues.name }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - app.kubernetes.io/name: {{ $podValues.name }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.datatracker" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - {{- if .Values.scoutapm }} - - name: "scoutapm" - image: "{{ .Values.scoutapm.image.repository }}:{{ default "latest" .Values.scoutapm.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.scoutapm.image.imagePullPolicy }} - livenessProbe: - exec: - command: - - "sh" - - "-c" - - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" - securityContext: - readOnlyRootFilesystem: {{ default true .Values.scoutapm.readOnlyRootFilesystem }} - runAsUser: {{ default 65534 .Values.scoutapm.runAsUser }} # "nobody" user by default - runAsGroup: {{ default 65534 .Values.scoutapm.runAsGroup }} # "nogroup" group by default - {{- end }} - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default .Chart.AppVersion $podValues.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} - env: - - name: "CONTAINER_ROLE" - value: "datatracker" - {{- if $.Values.env }} - {{- range $key, $val := $.Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/templates/deployments/memcached.yaml b/helm/templates/deployments/memcached.yaml deleted file mode 100644 index 7bf21c1d79..0000000000 --- a/helm/templates/deployments/memcached.yaml +++ /dev/null @@ -1,74 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.memcached.fullname" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - app.kubernetes.io/name: {{ .Values.memcached.name }} -spec: - {{- $podValues := .Values.memcached }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - app.kubernetes.io/name: {{ $podValues.name }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - app.kubernetes.io/name: {{ $podValues.name }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.memcached" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" - args: ["-m", "$(MEMCACHED_MEM_LIMIT)"] - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.imagePullPolicy }} - env: - {{- if .Values.env }} - {{- range $key, $val := .Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: memcached - containerPort: 11211 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/templates/hpa.yaml b/helm/templates/hpa.yaml deleted file mode 100644 index 646000bdcb..0000000000 --- a/helm/templates/hpa.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "datatracker.fullname" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "datatracker.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml deleted file mode 100644 index 3b1a9febc5..0000000000 --- a/helm/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.datatracker.ingress.enabled -}} -{{- $fullName := include "datatracker.fullname" . -}} -{{- $svcPort := .Values.datatracker.service.port -}} -{{- if and .Values.datatracker.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.datatracker.ingress.annotations "kubernetes.io/ingress.class" .Values.datatracker.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.datatracker.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.datatracker.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.datatracker.ingress.className }} - {{- end }} - {{- if .Values.datatracker.ingress.tls }} - tls: - {{- range .Values.datatracker.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.datatracker.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/helm/templates/persistentvolumeclaims.yaml b/helm/templates/persistentvolumeclaims.yaml deleted file mode 100644 index 412de9d02d..0000000000 --- a/helm/templates/persistentvolumeclaims.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: "datatracker-shared-volume-claim" - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - app.kubernetes.io/name: {{ .Values.datatracker.name }} -spec: - {{- with .Values.persistentVolumes.datatrackerSharedVolume }} - storageClassName: {{ .storageClassName | quote }} - {{- if .volumeName }} - volumeName: {{ .volumeName | quote }} - {{- end }} - accessModes: - {{- range .accessModes }} - - {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ .storage | quote }} - {{- end }} ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: "rabbitmq-data-claim" - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - app.kubernetes.io/name: {{ .Values.rabbitmq.name }} -spec: - {{- with .Values.persistentVolumes.rabbitmqDataVolume }} - storageClassName: {{ .storageClassName | quote }} - {{- if .volumeName }} - volumeName: {{ .volumeName | quote }} - {{- end }} - accessModes: - {{- range .accessModes }} - - {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ .storage | quote }} - {{- end }} diff --git a/helm/templates/rbac/beat-serviceaccount.yaml b/helm/templates/rbac/beat-serviceaccount.yaml deleted file mode 100644 index 9834013756..0000000000 --- a/helm/templates/rbac/beat-serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccounts.beat.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.beat" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.serviceAccounts.beat.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} diff --git a/helm/templates/rbac/celery-serviceaccount.yaml b/helm/templates/rbac/celery-serviceaccount.yaml deleted file mode 100644 index e27673e385..0000000000 --- a/helm/templates/rbac/celery-serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccounts.celery.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.celery" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.serviceAccounts.celery.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} diff --git a/helm/templates/rbac/datatracker-serviceaccount.yaml b/helm/templates/rbac/datatracker-serviceaccount.yaml deleted file mode 100644 index d64f1523da..0000000000 --- a/helm/templates/rbac/datatracker-serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccounts.datatracker.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.datatracker" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.serviceAccounts.datatracker.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} diff --git a/helm/templates/rbac/memcached-serviceaccount.yaml b/helm/templates/rbac/memcached-serviceaccount.yaml deleted file mode 100644 index cc8d1a1212..0000000000 --- a/helm/templates/rbac/memcached-serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccounts.memcached.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.memcached" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.serviceAccounts.memcached.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} diff --git a/helm/templates/rbac/rabbitmq-serviceaccount.yaml b/helm/templates/rbac/rabbitmq-serviceaccount.yaml deleted file mode 100644 index 58c89b2dcf..0000000000 --- a/helm/templates/rbac/rabbitmq-serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccounts.rabbitmq.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName.rabbitmq" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.serviceAccounts.rabbitmq.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} diff --git a/helm/templates/services/datatracker.yaml b/helm/templates/services/datatracker.yaml deleted file mode 100644 index 201f5a3b4b..0000000000 --- a/helm/templates/services/datatracker.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{include "datatracker.fullname" .}} - labels: {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.datatracker.service.annotations }} - annotations: - {{- range $key, $value := . }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} -spec: - type: {{.Values.datatracker.service.type}} - ports: - - port: {{ default "80" .Values.datatracker.service.port}} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "datatracker.selectorLabels" . | nindent 4}} - app.kubernetes.io/name: {{ .Values.datatracker.name }} diff --git a/helm/templates/services/memcached.yaml b/helm/templates/services/memcached.yaml deleted file mode 100644 index 572c382b5b..0000000000 --- a/helm/templates/services/memcached.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: memcached - labels: {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.memcached.service.annotations }} - annotations: - {{- range $key, $value := . }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} -spec: - type: {{.Values.memcached.service.type}} - ports: - - port: {{ default "11211" .Values.memcached.service.port}} - targetPort: memcached - protocol: TCP - name: memcached - selector: - {{- include "datatracker.selectorLabels" . | nindent 4}} - app.kubernetes.io/name: {{ .Values.memcached.name }} diff --git a/helm/templates/services/rabbitmq.yaml b/helm/templates/services/rabbitmq.yaml deleted file mode 100644 index e67cb31694..0000000000 --- a/helm/templates/services/rabbitmq.yaml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: rabbitmq - labels: {{- include "datatracker.commonLabels" . | nindent 4 }} - {{- with .Values.rabbitmq.service.annotations }} - annotations: - {{- range $key, $value := . }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} -spec: - type: {{.Values.rabbitmq.service.type}} - clusterIP: None # headless service - ports: - - port: {{ default "5672" .Values.rabbitmq.service.port}} - targetPort: amqp - protocol: TCP - name: amqp - selector: - {{- include "datatracker.selectorLabels" . | nindent 4}} - app.kubernetes.io/name: {{ .Values.rabbitmq.name }} diff --git a/helm/templates/statefulsets/rabbitmq.yaml b/helm/templates/statefulsets/rabbitmq.yaml deleted file mode 100644 index 6cb2cc8367..0000000000 --- a/helm/templates/statefulsets/rabbitmq.yaml +++ /dev/null @@ -1,87 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ include "datatracker.rabbitmq.fullname" . }} - labels: - {{- include "datatracker.commonLabels" . | nindent 4 }} - app.kubernetes.io/name: {{ .Values.rabbitmq.name }} -spec: - {{- $podValues := .Values.rabbitmq }} - replicas: {{ $podValues.replicaCount }} - revisionHistoryLimit: {{ $podValues.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - app.kubernetes.io/name: {{ $podValues.name }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - app.kubernetes.io/name: {{ $podValues.name }} - spec: - {{- with $podValues.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName.rabbitmq" . }} - securityContext: - {{- toYaml $podValues.podSecurityContext | nindent 8 }} - initContainers: - - name: init-rabbitmq - image: busybox:stable - command: - - "sh" - - "-c" - - "mkdir -p -m700 /mnt/rabbitmq && chown 100:101 /mnt/rabbitmq" - securityContext: - runAsNonRoot: false - runAsUser: 0 - readOnlyRootFilesystem: true - volumeMounts: - - name: "rabbitmq-data" - mountPath: "/mnt" - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml $podValues.securityContext | nindent 12 }} - image: "{{ $podValues.image.repository }}:{{ default "latest" $podValues.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" $podValues.image.pullPolicy }} - env: - {{- if .Values.env }} - {{- range $key, $val := .Values.env }} - - name: {{ $key | quote }} - value: {{ $val | quote }} - {{- end }} - {{- end }} - {{- with $podValues.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: amqp - containerPort: 5672 - protocol: TCP - livenessProbe: - {{- toYaml $podValues.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml $podValues.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml $podValues.startupProbe | nindent 12 }} - resources: - {{- toYaml $podValues.resources | nindent 12 }} - {{- with $podValues.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $podValues.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/values.yaml b/helm/values.yaml deleted file mode 100644 index aa5bbaf30c..0000000000 --- a/helm/values.yaml +++ /dev/null @@ -1,645 +0,0 @@ -# Default values for datatracker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -## Define serviceAccount names for components. Defaults to component's fully qualified name. -## -serviceAccounts: - datatracker: - create: true - name: datatracker - annotations: {} - celery: - create: true - name: celery - annotations: {} - beat: - create: true - name: beat - annotations: {} - rabbitmq: - create: true - name: rabbitmq - annotations: {} - memcached: - create: true - name: memcached - annotations: {} - -# ------------------------------------------------------------- -# DATATRACKER -# ------------------------------------------------------------- - -datatracker: - name: datatracker - image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: datatracker.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -# livenessProbe: -# httpGet: -# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint -# path: /submit/tool-instructions/ -# port: http - - podAnnotations: {} - podLabels: {} - -#readinessProbe: -# httpGet: -# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint -# path: /submit/tool-instructions/ -# port: http - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - podSecurityContext: - runAsNonRoot: true - - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsUser: 1000 - runAsGroup: 1000 - - service: - type: ClusterIP - port: 80 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -# startupProbe: -# initialDelaySeconds: 15 -# periodSeconds: 5 -# timeoutSeconds: 5 -# successThreshold: 1 -# failureThreshold: 60 -# httpGet: -# # /submit/tool-instructions/ just happens to be cheap until we get a real health endpoint -# path: /submit/tool-instructions/ -# port: http - - # Additional volumes on the output Deployment definition. - volumes: - - name: settings-local-volume - configMap: - name: django-configmap - - name: datatracker-shared-volume - persistentVolumeClaim: - claimName: "datatracker-shared-volume-claim" - - name: datatracker-tmp - emptyDir: - sizeLimit: "2Gi" - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: settings-local-volume - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - readOnly: true - - name: datatracker-shared-volume - mountPath: /a - - name: datatracker-tmp - mountPath: /tmp - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# CELERY -# ------------------------------------------------------------- - -celery: - name: celery - image: {} - # defaults to datatracker settings if not specified separately - #repository: "ghcr.io/ietf-tools/datatracker" - #pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - livenessProbe: - exec: - command: ["celery", "-A", "ietf", "inspect", "ping"] - periodSeconds: 30 - timeoutSeconds: 5 - - podAnnotations: {} - podLabels: {} - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - podSecurityContext: - runAsNonRoot: true - - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsUser: 1000 - runAsGroup: 1000 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - exec: - command: ["celery", "-A", "ietf", "inspect", "ping"] - - # Additional volumes on the output Deployment definition. - volumes: - - name: settings-local-volume - configMap: - name: django-configmap - - name: datatracker-shared-volume - persistentVolumeClaim: - claimName: "datatracker-shared-volume-claim" - - name: celery-tmp - emptyDir: - sizeLimit: "2Gi" - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: settings-local-volume - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - readOnly: true - - name: datatracker-shared-volume - mountPath: /a - - name: celery-tmp - mountPath: /tmp - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# BEAT -# ------------------------------------------------------------- - -beat: - name: beat - image: {} - # defaults to datatracker settings if not specified separately - # repository: "ghcr.io/ietf-tools/datatracker" - # pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - livenessProbe: - exec: - command: ["celery", "-A", "ietf", "inspect", "ping"] - periodSeconds: 30 - timeoutSeconds: 5 - - podAnnotations: {} - podLabels: {} - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - podSecurityContext: - runAsNonRoot: true - - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsUser: 1000 - runAsGroup: 1000 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - exec: - command: ["celery", "-A", "ietf", "inspect", "ping"] - - # Additional volumes on the output Deployment definition. - volumes: - - name: settings-local-volume - configMap: - name: django-configmap - - name: datatracker-shared-volume - persistentVolumeClaim: - claimName: "datatracker-shared-volume-claim" - - name: beat-tmp - emptyDir: - sizeLimit: "2Gi" - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: settings-local-volume - mountPath: /workspace/ietf/settings_local.py - subPath: settings_local.py - readOnly: true - - name: datatracker-shared-volume - mountPath: /a - - name: beat-tmp - mountPath: /tmp - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# RABBITMQ -# ------------------------------------------------------------- - -rabbitmq: - name: "rabbitmq" - image: - repository: "ghcr.io/ietf-tools/datatracker-mq" - pullPolicy: IfNotPresent - tag: "3.12-alpine" - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - livenessProbe: - exec: - command: ["rabbitmq-diagnostics", "-q", "ping"] - periodSeconds: 30 - timeoutSeconds: 5 - - podAnnotations: {} - podLabels: {} - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - podSecurityContext: - runAsNonRoot: true - - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - # rabbitmq image sets up uid/gid 100/101 - runAsUser: 100 - runAsGroup: 101 - - service: - type: ClusterIP - port: 5672 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - exec: - command: ["rabbitmq-diagnostics", "-q", "ping"] - - # Additional volumes on the output Deployment definition. - volumes: - - name: "rabbitmq-data" - persistentVolumeClaim: - claimName: "rabbitmq-data-claim" - - name: "rabbitmq-config" - configMap: - name: "rabbitmq-configmap" - - name: "rabbitmq-tmp" - emptyDir: - sizeLimit: 50Mi - # - name: foo - # secret: - # secretName: mysecret - # optional: false - - # Additional volumeMounts on the output Deployment definition. - volumeMounts: - - name: "rabbitmq-data" - mountPath: "/var/lib/rabbitmq" - subPath: "rabbitmq" - - name: "rabbitmq-config" - mountPath: "/etc/rabbitmq" - - name: "rabbitmq-tmp" - mountPath: "/tmp" - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# MEMCACHED -# ------------------------------------------------------------- - -memcached: - name: memcached - image: - repository: "memcached" - pullPolicy: IfNotPresent - tag: "1.6-alpine" - - imagePullSecrets: [] - nameOverride: "" - fullnameOverride: "" - - podAnnotations: {} - podLabels: {} - - replicaCount: 1 - - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - podSecurityContext: - runAsNonRoot: true - - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - # memcached image sets up uid/gid 11211 - runAsUser: 11211 - runAsGroup: 11211 - - service: - type: ClusterIP - port: 11211 - - serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - tolerations: [] - - nodeSelector: {} - - affinity: {} - -# ------------------------------------------------------------- -# SCOUT APM SETTINGS -# ------------------------------------------------------------- -# Set this to enable a Scout APM Core Agent sidecar -scoutapm: - image: - repository: "scoutapp/scoutapm" - tag: "version-1.4.0" - -# ------------------------------------------------------------- -# PERSISTENT VOLUMES -# ------------------------------------------------------------- - -persistentVolumes: - datatrackerSharedVolume: - # Note: This is the /a shared volume - volumeName: "datatracker-shared-volume" - accessModes: - - "ReadWriteMany" # or ReadWriteOnce and force datatracker/celery/beat to a single node - storageClassName: "" # Empty string means do not use default storage class - storage: "600Gi" # actual PersistentVolume must be at least this big or the PVC will not bind - - rabbitmqDataVolume: - volumeName: "rabbitmq-data-volume" - accessModes: - - "ReadWriteOnce" - storageClassName: "" # Empty string means do not use default storage class - storage: "8Gi" # actual PersistentVolume must be at least this big or the PVC will not bind - -# ------------------------------------------------------------- -# COMMON -# ------------------------------------------------------------- - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -env: - # n.b., these are debug values / non-secret secrets - DATATRACKER_SERVER_MODE: "development" # development for staging, production for production - DATATRACKER_ADMINS: |- - Robert Sparks - Ryan Cross - Kesara Rathnayake - Jennifer Richards - Nicolas Giard - DATATRACKER_ALLOWED_HOSTS: ".ietf.org" # newline-separated list also allowed - # DATATRACKER_DATATRACKER_DEBUG: "false" - - # DB access details - needs to be filled in - # DATATRACKER_DBHOST: "db" - # DATATRACKER_DBPORT: "5432" - # DATATRACKER_DBNAME: "datatracker" - # DATATRACKER_DBUSER: "django" # secret - # DATATRACKER_DBPASS: "RkTkDPFnKpko" # secret - - DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret - - # Set this to point testing / staging at the production statics server until we - # sort that out - # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" - - # DATATRACKER_EMAIL_DEBUG: "true" - - # Outgoing email details - # DATATRACKER_EMAIL_HOST: "localhost" # defaults to localhost - # DATATRACKER_EMAIL_PORT: "2025" # defaults to 2025 - - # The value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ=" # secret - - DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" # secret - DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" # secret - DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" # secret - DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" # secret - - # API key configuration - DATATRACKER_API_KEY_TYPE: "ES265" - # secret - value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_API_PUBLIC_KEY_PEM_B64: |- - Ci0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1Ga3dFd1lIS29aSXpqMENBUVlJS - 29aSXpqMERBUWNEUWdBRXFWb2pzYW9mREpTY3VNSk4rdHNodW15Tk01TUUKZ2Fyel - ZQcWtWb3ZtRjZ5RTdJSi9kdjRGY1YrUUtDdEovck9TOGUzNlk4WkFFVll1dWtoZXM - weVoxdz09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo= - # secret - value here is the default from settings.py (i.e., not actually secret) - DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- - Ci0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLQpNSUdIQWdFQU1CTUdCeXFHU000O - UFnRUdDQ3FHU000OUF3RUhCRzB3YXdJQkFRUWdvSTZMSmtvcEtxOFhySGk5ClFxR1 - F2RTRBODNURllqcUx6KzhnVUxZZWNzcWhSQU5DQUFTcFdpT3hxaDhNbEp5NHdrMzY - yeUc2Ykkwemt3U0IKcXZOVStxUldpK1lYcklUc2duOTIvZ1Z4WDVBb0swbitzNUx4 - N2ZwanhrQVJWaTY2U0Y2elRKblgKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= - - # DATATRACKER_MEETECHO_API_BASE: "https://meetings.conf.meetecho.com/api/v1/" - DATATRACKER_MEETECHO_CLIENT_ID: "this-is-the-meetecho-client-id" # secret - DATATRACKER_MEETECHO_CLIENT_SECRET: "this-is-the-meetecho-client-secret" # secret - - # DATATRACKER_MATOMO_SITE_ID: "7" # must be present to enable Matomo - # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org" - - CELERY_PASSWORD: "this-is-a-secret" # secret - - DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret - - # use this to override default - one entry per line - # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- - # https://datatracker.staging.ietf.org - - # Scout configuration - DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" - DATATRACKER_SCOUT_NAME: "StagingDatatracker" - - MEMCACHED_MEM_LIMIT: "1024" From 6cb0fa5ed40629c51c8a619773957ff6163d88a9 Mon Sep 17 00:00:00 2001 From: Nicolas Giard Date: Mon, 13 May 2024 21:42:21 -0400 Subject: [PATCH 66/70] chore: Remove helm config from .editorconfig --- .editorconfig | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.editorconfig b/.editorconfig index 4e5b304b6a..7e5ce6236a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -51,14 +51,8 @@ indent_size = 2 [ietf/**.html] insert_final_newline = false -# Settings for Helm chart yaml -# --------------------------------------------------------- -# Use 2-space indents -[helm/**.yaml] -indent_size = 2 - # Settings for Kubernetes yaml # --------------------------------------------------------- # Use 2-space indents [k8s/**.yaml] -indent_size = 2 \ No newline at end of file +indent_size = 2 From 06b99fa64b1f4bff532794c1e8007b314a97a4f2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 15 May 2024 16:13:47 -0300 Subject: [PATCH 67/70] chore: remove unused import --- ietf/utils/management/commands/patch_libraries.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ietf/utils/management/commands/patch_libraries.py b/ietf/utils/management/commands/patch_libraries.py index 2ada67a280..d9ae11097b 100644 --- a/ietf/utils/management/commands/patch_libraries.py +++ b/ietf/utils/management/commands/patch_libraries.py @@ -1,6 +1,5 @@ # Copyright The IETF Trust 2024, All Rights Reserved import django -import os from django.conf import settings from django.core.management.base import BaseCommand, CommandError From 1c3825bdec1079de1b0f32cc0174f71857a0298b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 15 May 2024 18:25:59 -0300 Subject: [PATCH 68/70] test: ensure logs only go to console during tests --- ietf/settings_test.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 3f69f0ae38..962628514a 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -60,3 +60,36 @@ def __getitem__(self, item): TEMPLATES[0]['OPTIONS']['context_processors'] = [ p for p in TEMPLATES[0]['OPTIONS']['context_processors'] if not p in DEV_TEMPLATE_CONTEXT_PROCESSORS ] # pyflakes:ignore REQUEST_PROFILE_STORE_ANONYMOUS_SESSIONS = False + +# Override loggers with a safer set in case things go to the log during testing. Specifically, +# make sure there are no syslog loggers that might send things to a real syslog. +LOGGING["loggers"] = { + 'django': { + 'handlers': ['debug_console'], + 'level': 'INFO', + }, + 'django.request': { + 'handlers': ['debug_console'], + 'level': 'ERROR', + }, + 'django.server': { + 'handlers': ['django.server'], + 'level': 'INFO', + }, + 'django.security': { + 'handlers': ['debug_console', ], + 'level': 'INFO', + }, + 'oidc_provider': { + 'handlers': ['debug_console', ], + 'level': 'DEBUG', + }, + 'datatracker': { + 'handlers': ['debug_console'], + 'level': 'INFO', + }, + 'celery': { + 'handlers': ['debug_console'], + 'level': 'INFO', + }, +} From 0d68646fdceb762f8ca1e1250629a7d5f2ff6750 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 15 May 2024 20:47:00 -0300 Subject: [PATCH 69/70] chore: suppress pyflakes check --- ietf/settings_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 962628514a..024512a8db 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -63,7 +63,7 @@ def __getitem__(self, item): # Override loggers with a safer set in case things go to the log during testing. Specifically, # make sure there are no syslog loggers that might send things to a real syslog. -LOGGING["loggers"] = { +LOGGING["loggers"] = { # pyflakes:ignore 'django': { 'handlers': ['debug_console'], 'level': 'INFO', From d9c6ae7b7e5581edc2342fe9f6cc511fba14e1fd Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 16 May 2024 12:55:23 -0300 Subject: [PATCH 70/70] chore: remove syslog handler from settings.py (#7427) System-level integration like this really needs to be in settings_local. This was causing problems when running in the dev environment. --- ietf/settings.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/ietf/settings.py b/ietf/settings.py index e3aa351ed0..91c1ef205d 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -252,7 +252,7 @@ def skip_unreadable_post(record): 'level': 'INFO', }, 'django.security': { - 'handlers': ['debug_console', ], + 'handlers': ['debug_console', ], 'level': 'INFO', }, 'oidc_provider': { @@ -260,11 +260,11 @@ def skip_unreadable_post(record): 'level': 'DEBUG', }, 'datatracker': { - 'handlers': ['syslog'], + 'handlers': ['debug_console'], 'level': 'INFO', }, 'celery': { - 'handlers': ['syslog'], + 'handlers': ['debug_console'], 'level': 'INFO', }, }, @@ -289,13 +289,6 @@ def skip_unreadable_post(record): 'class': 'logging.StreamHandler', 'formatter': 'django.server', }, - 'syslog': { - 'level': 'DEBUG', - 'class': 'logging.handlers.SysLogHandler', - 'facility': 'user', - 'formatter': 'plain', - 'address': '/dev/log', - }, 'mail_admins': { 'level': 'ERROR', 'filters': [