From 511e714028ce49a939a2170a50f77ccf5ec6eb44 Mon Sep 17 00:00:00 2001
From: iamsudip <sudip.maji@traceable.ai>
Date: Wed, 29 Nov 2023 21:01:10 +0530
Subject: [PATCH] add support for s3 as destination for backups (#37)

* helm: add s3 backup support

* add default configurationOverrides for kafkaStore

* don't bump version

* bump

---------

Co-authored-by: Ravi Singal <62086374+ravisingal@users.noreply.github.com>
---
 README.md                   |  2 +-
 helm/templates/cronjob.yaml | 20 +++++++++++++++++---
 helm/values.yaml            | 13 ++++++++++++-
 3 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index dc39044..936421b 100644
--- a/README.md
+++ b/README.md
@@ -27,7 +27,7 @@ This chart will do the following:
 * Optionally add an [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) resource.
 * Optionally start a JMX Exporter container inside schema registry pods.
 * Optionally create a Prometheus ServiceMonitor for each enabled jmx exporter container.
-* Optionally add a cronjob to take backup the schema registry topic and save it in [Google Cloud Storage](https://cloud.google.com/storage).
+* Optionally add a cronjob to take backup the schema registry topic and save it in [Google Cloud Storage](https://cloud.google.com/storage) or [AWS S3](https://aws.amazon.com/pm/serv-s3/)
 
 
 ## Installing the Chart
diff --git a/helm/templates/cronjob.yaml b/helm/templates/cronjob.yaml
index 84ae324..d3db956 100644
--- a/helm/templates/cronjob.yaml
+++ b/helm/templates/cronjob.yaml
@@ -45,23 +45,36 @@ spec:
                 - |
                   timestamp=$(date +%Y-%m-%d-%H-%M-%S)
                   month=${timestamp:0:7}
-                  BACKUP_LOCATION=gs://$BUCKET/schema-registry/$CLUSTER_NAME/$month
                   unset JMX_PORT KAFKA_OPTS KAFKA_HEAP_OPTS KAFKA_LOG4J_OPTS
                   /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server $KAFKA_BOOTSTRAP_SERVERS --topic $KAFKA_TOPIC --from-beginning --property print.key=true --timeout-ms 60000 1> schemas-${timestamp}.log || exit 2
                   tar cvfz schemas-${timestamp}.tar.gz schemas-$timestamp.log || exit 2
+                  {{- if .Values.backup.gcloud }}
+                  BACKUP_LOCATION=gs://$BUCKET/schema-registry/$CLUSTER_NAME/$month
                   gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS || exit 2
                   gsutil cp schemas-${timestamp}.tar.gz $BACKUP_LOCATION/schemas-${timestamp}.tar.gz || exit 2
+                  {{- end }}
+                  {{- if .Values.backup.aws }}
+                  BACKUP_LOCATION=s3://$S3_BUCKET/${CLUSTER_NAME}-backups/schema-registry/$month
+                  aws s3 cp schemas-${timestamp}.tar.gz $BACKUP_LOCATION/schemas-${timestamp}.tar.gz || exit 2
+                  {{- end }}
               env:
                 - name: CLUSTER_NAME
                   value: {{ .Values.backup.cluster }}
+                {{- if .Values.backup.gcloud }}
                 - name: BUCKET
                   value: {{ .Values.backup.gcloud.bucket }}
+                - name: GOOGLE_APPLICATION_CREDENTIALS
+                  value: "/accounts/key.json"
+                {{- end }}
+                {{- if .Values.backup.aws }}
+                - name: S3_BUCKET
+                  value: {{ .Values.backup.aws.bucket }}
+                {{- end }}
                 - name: KAFKA_BOOTSTRAP_SERVERS
                   value: {{ template "schema-registry.kafka.bootstrapServers" . }}
                 - name: KAFKA_TOPIC
                   value: {{ .Values.backup.topic }}
-                - name: GOOGLE_APPLICATION_CREDENTIALS
-                  value: "/accounts/key.json"
+              {{- if .Values.backup.gcloud }}
               volumeMounts:
                 - name: gcs-iam-secret
                   mountPath: "/accounts"
@@ -69,6 +82,7 @@ spec:
             - name: gcs-iam-secret
               secret:
                 secretName: {{ .Values.backup.gcloud.secretName }}
+              {{- end }}
           {{- if .Values.backup.imagePullSecrets }}
           imagePullSecrets:
             {{- toYaml .Values.imagePullSecrets | nindent 12 }}
diff --git a/helm/values.yaml b/helm/values.yaml
index dd4099f..b6e2b6c 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -12,7 +12,9 @@ image:
 
 imagePullSecrets: []
 
-configurationOverrides: {}
+configurationOverrides:
+  kafkastore.timeout.ms: 2000
+
 customEnv: {}
 schemaRegistryOpts: {}
 overrideGroupId: ""
@@ -143,6 +145,14 @@ backup:
   imagePullSecrets: []
   cluster: "test"
   gcloud: {}
+  # GCS Bucket Configuration
+  # gcloud:
+  #   bucket: bucketName
+  #   secretName: gcs-bucket-secret
+  aws: {}
+  # AWS S3 Bucket Configuration
+  # aws:
+  #   bucket: bucketName/backups
   affinity: {}
   nodeSelector: {}
   securityContext: {}
@@ -155,3 +165,4 @@ servicemonitor:
   interval: 15s
   secure: false
   tlsConfig: {}
+