Support for separate Kafka properties (#336)

* Support for separate Kafka properties Default behavior preserved. In addition to that it is supported to set or override Kafka parameters for producer and consumer individually.
2gis · Dec 5, 2023 · de605a1 · de605a1
1 parent 431528f
commit de605a1
Show file tree

Hide file tree

Showing 8 changed files with 148 additions and 42 deletions.
diff --git a/charts/navi-async-matrix/README.md b/charts/navi-async-matrix/README.md
@@ -26,6 +26,7 @@ See the [documentation](https://docs.2gis.com/en/on-premise/navigation/distance-
 | --------------------- | --------------------------------------------------------------------------------------- | ----- |
 | `dgctlDockerRegistry` | Docker Registry endpoint where On-Premise services' images reside. Format: `host:port`. | `""`  |
 
+
 ### Common settings
 
 | Name                            | Description                                                                                                                 | Value  |
@@ -47,6 +48,7 @@ See the [documentation](https://docs.2gis.com/en/on-premise/navigation/distance-
 | `terminationGracePeriodSeconds` | Kubernetes [termination grace period](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/).           | `60`   |
 | `prometheusEnabled`             | If Prometheus scrape is enabled.                                                                                            | `true` |
 
+
 ### Deployment settings
 
 | Name               | Description | Value                               |
@@ -55,6 +57,7 @@ See the [documentation](https://docs.2gis.com/en/on-premise/navigation/distance-
 | `image.tag`        | Tag         | `1.6.2`                             |
 | `image.pullPolicy` | Pull Policy | `IfNotPresent`                      |
 
+
 ### Service account settings
 
 | Name                         | Description                                                                                                             | Value   |
@@ -63,12 +66,23 @@ See the [documentation](https://docs.2gis.com/en/on-premise/navigation/distance-
 | `serviceAccount.annotations` | Annotations to add to the service account.                                                                              | `{}`    |
 | `serviceAccount.name`        | The name of the service account to use. If not set and create is true, a name is generated using the fullname template. | `""`    |
 
+
+### RBAC parameters
+
+| Name               | Description                                     | Value   |
+| ------------------ | ----------------------------------------------- | ------- |
+| `rbac.create`      | Whether to create and use RBAC resources or not | `false` |
+| `rbac.annotations` | Role and RoleBinding annotations                | `{}`    |
+| `rbac.labels`      | Role and RoleBinding additional labels          | `{}`    |
+
+
 ### Strategy settings
 
 | Name                  | Description                                                          | Value           |
 | --------------------- | -------------------------------------------------------------------- | --------------- |
 | `updateStrategy.type` | Type of Kubernetes deployment. Can be `Recreate` or `RollingUpdate`. | `RollingUpdate` |
 
+
 ### Limits
 
 | Name                        | Description       | Value |
@@ -78,6 +92,7 @@ See the [documentation](https://docs.2gis.com/en/on-premise/navigation/distance-
 | `resources.limits.cpu`      | A CPU limit.      |       |
 | `resources.limits.memory`   | A memory limit.   |       |
 
+
 ### Service settings
 
 | Name                  | Description                                                                                                                    | Value       |
@@ -88,6 +103,7 @@ See the [documentation](https://docs.2gis.com/en/on-premise/navigation/distance-
 | `service.annotations` | Kubernetes [service annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/).              | `{}`        |
 | `service.labels`      | Kubernetes [service labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/).                        | `{}`        |
 
+
 ### Kubernetes [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) settings
 
 | Name                                 | Description                               | Value                           |
@@ -99,6 +115,7 @@ See the [documentation](https://docs.2gis.com/en/on-premise/navigation/distance-
 | `ingress.hosts[0].paths[0].pathType` | Type of the path for the Ingress service. | `Prefix`                        |
 | `ingress.tls`                        | TLS configuration                         | `[]`                            |
 
+
 ### Kubernetes [Pod Disruption Budget](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/#pod-disruption-budgets) settings
 
 | Name                 | Description                                          | Value  |
@@ -107,6 +124,7 @@ See the [documentation](https://docs.2gis.com/en/on-premise/navigation/distance-
 | `pdb.minAvailable`   | How many pods must be available after the eviction.  | `""`   |
 | `pdb.maxUnavailable` | How many pods can be unavailable after the eviction. | `1`    |
 
+
 ### Kubernetes [Horizontal Pod Autoscaling](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) settings
 
 | Name                                    | Description                                                                                                                                                          | Value   |
@@ -118,6 +136,7 @@ See the [documentation](https://docs.2gis.com/en/on-premise/navigation/distance-
 | `hpa.targetMemoryUtilizationPercentage` | Target average memory utilization (represented as a percentage of requested memory) over all the pods; if not specified the default autoscaling policy will be used. | `""`    |
 | `hpa.behavior`                          | HPA Behavior                                                                                                                                                         | `{}`    |
 
+
 ### Kubernetes [Vertical Pod Autoscaling](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/README.md) settings
 
 | Name                    | Description                                                                                                  | Value   |
@@ -129,6 +148,7 @@ See the [documentation](https://docs.2gis.com/en/on-premise/navigation/distance-
 | `vpa.maxAllowed.cpu`    | Upper limit for the number of CPUs to which the autoscaler can scale up.                                     | `2000m` |
 | `vpa.maxAllowed.memory` | Upper limit for the RAM size to which the autoscaler can scale up.                                           | `2Gi`   |
 
+
 ### Distance Matrix Async API settings
 
 | Name                    | Description                                                                                                                   | Value  |
@@ -140,6 +160,7 @@ See the [documentation](https://docs.2gis.com/en/on-premise/navigation/distance-
 | `dm.citiesUrl`          | URL of the information about cities provided by the Navi-Castle service, ex: http://navi-castle.svc/cities.conf. **Required** | `""`   |
 | `dm.citiesUpdatePeriod` | Period (in seconds) between requesting data from `citiesUrl`.                                                                 | `3600` |
 
+
 ### Database settings
 
 | Name              | Description                                 | Value         |
@@ -156,20 +177,28 @@ See the [documentation](https://docs.2gis.com/en/on-premise/navigation/distance-
 | `db.tls.key`      | Key of postgresql server.                   | `""`          |
 | `db.tls.mode`     | Level of protection.                        | `verify-full` |
 
+
 ### Kafka settings
 
-| Name                              | Description                                                                                                               | Value               |
-| --------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | ------------------- |
-| `kafka.groupId`                   | Distance Matrix Async API group identifier.                                                                               | `navi_async_matrix` |
-| `kafka.statusTopic`               | Name of the topic for sending new tasks to.                                                                               | `status_topic`      |
-| `kafka.cancelTopic`               | Name of the topic for canceling or receiving information about finished tasks.                                            | `cancel_topic`      |
-| `kafka.properties`                | Properties as supported by kafka-python. Refer to inline comments for details.                                            |                     |
-| `kafka.sensitiveProperties`       | As kafka.properties, but kept in Secrets. Refer to inlines comments for details.                                          | `{}`                |
-| `kafka.fileProperties`            | As kafka.properties, but kept in a file, which passed to application as a filename. Refer to inline comments for details. | `{}`                |
-| `kafka.taskTopicRules`            | **Information about the topics that Distance Matrix Async API will use to send the requests.**                            |                     |
-| `kafka.taskTopicRules[].topic`    | Name of the topic.                                                                                                        |                     |
-| `kafka.taskTopicRules[].default`  | If this topic is used for projects by default.                                                                            |                     |
-| `kafka.taskTopicRules[].projects` | List of projects to use this topic for, e.g., `['moscow']`.                                                               |                     |
+| Name                                          | Description                                                                                                               | Value               |
+| --------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | ------------------- |
+| `kafka.groupId`                               | Distance Matrix Async API group identifier.                                                                               | `navi_async_matrix` |
+| `kafka.statusTopic`                           | Name of the topic for sending new tasks to.                                                                               | `status_topic`      |
+| `kafka.cancelTopic`                           | Name of the topic for canceling or receiving information about finished tasks.                                            | `cancel_topic`      |
+| `kafka.properties`                            | Properties as supported by kafka-python. Refer to inline comments for details.                                            |                     |
+| `kafka.sensitiveProperties`                   | As kafka.properties, but kept in Secrets. Refer to inlines comments for details.                                          | `{}`                |
+| `kafka.fileProperties`                        | As kafka.properties, but kept in a file, which passed to application as a filename. Refer to inline comments for details. | `{}`                |
+| `kafka.consumerOverrides.properties`          | Consumer specific properties as simple key-value pairs.                                                                   | `{}`                |
+| `kafka.consumerOverrides.sensitiveProperties` | Consumer specific properties mounted as secrets.                                                                          | `{}`                |
+| `kafka.consumerOverrides.fileProperties`      | Consumer specific properties mounted as regular files.                                                                    | `{}`                |
+| `kafka.producerOverrides.properties`          | Consumer specific properties as simple key-value pairs.                                                                   | `{}`                |
+| `kafka.producerOverrides.sensitiveProperties` | Consumer specific properties mounted as secrets.                                                                          | `{}`                |
+| `kafka.producerOverrides.fileProperties`      | Consumer specific properties mounted as regular files.                                                                    | `{}`                |
+| `kafka.taskTopicRules`                        | **Information about the topics that Distance Matrix Async API will use to send the requests.**                            |                     |
+| `kafka.taskTopicRules[].topic`                | Name of the topic.                                                                                                        |                     |
+| `kafka.taskTopicRules[].default`              | If this topic is used for projects by default.                                                                            |                     |
+| `kafka.taskTopicRules[].projects`             | List of projects to use this topic for, e.g., `['moscow']`.                                                               |                     |
+
 
 ### S3-compatible storage settings
 
@@ -181,6 +210,7 @@ See the [documentation](https://docs.2gis.com/en/on-premise/navigation/distance-
 | `s3.secretKey`    | S3 secret key for accessing the bucket. **Required**              | `""`  |
 | `s3.publicNetloc` | Announce proxy URL for S3 results instead of s3.url if not empty. | `nil` |
 
+
 ### API keys service
 
 | Name         | Description                                                                 | Value |

diff --git a/charts/navi-async-matrix/templates/_helpers.tpl b/charts/navi-async-matrix/templates/_helpers.tpl
@@ -67,20 +67,26 @@ Create the name of the service account to use
        Context:
          .kafka.properties
          .kafka.fileProperties
+         .overrides.properties
+         .overrides.fileProperties
          .mountpoint
 
        File properties values (file contents) replaced with keys (file names).
        File names prepended with the supposed directory from .mountpoint.
+       overrides.* are merged with kafka.* accordingly.
+
        Returns {"ret": that-merged-dict}.
        Folding result in "ret" needed for marshalling.
      */ -}}
 {{- define "navi-async-matrix.kafkaProperties" -}}
   {{- $ctx := . -}}
   {{- $kafkaProperties := dict -}}
-  {{- range $key, $_ := $ctx.kafka.fileProperties -}}
+  {{- $fileProperties := deepCopy $ctx.kafka.fileProperties | mustMerge (($ctx.overrides).fileProperties | default dict) -}}
+  {{- range $key, $_ := $fileProperties -}}
     {{- $_ := set $kafkaProperties $key (printf "%s/%s" $ctx.mountpoint $key) -}}
   {{- end -}}
-  {{- $kafkaProperties = mustMerge $kafkaProperties $ctx.kafka.properties -}}
+  {{- $regularProperties := deepCopy $ctx.kafka.properties | mustMerge (($ctx.overrides).properties | default dict) -}}
+  {{- $kafkaProperties = deepCopy $regularProperties | mustMerge $kafkaProperties -}}
   {{- dict "ret" $kafkaProperties | toYaml }}
 {{- end }}
 
@@ -90,6 +96,9 @@ Create the name of the service account to use
          .kafka.properties
          .kafka.fileProperties
          .kafka.sensitiveProperties
+         .overrides.properties
+         .overrides.fileProperties
+         .overrides.sensitiveProperties
          .mountpoint
          .secretname
          .prefix
@@ -113,6 +122,8 @@ Create the name of the service account to use
            }
          }
        where secretKeyRef.name is from .secretname
+       overrides.* are merged with kafka.* accordingly.
+
        Resulting object folded in {"ret":...} for marshalling.
      */ -}}
 {{- define "navi-async-matrix.kafkaPropertiesEnv" -}}
@@ -125,7 +136,8 @@ Create the name of the service account to use
           "value" $val
         ) -}}
   {{- end -}}
-  {{- range $prop, $val := $ctx.kafka.sensitiveProperties -}}
+  {{- $sensitiveProperties := deepCopy $ctx.kafka.sensitiveProperties | mustMerge (($ctx.overrides).sensitiveProperties | default dict) -}}
+  {{- range $prop, $val := $sensitiveProperties -}}
     {{- $env = append $env (dict
           "name" (print $ctx.prefix ($prop | upper | replace "." "_"))
           "valueFrom" (dict

diff --git a/charts/navi-async-matrix/templates/configmap.yaml b/charts/navi-async-matrix/templates/configmap.yaml
@@ -18,9 +18,9 @@ data:
         {{- if .Values.kafka.fileProperties }}
         "volumes": [
           {
-            "name": {{ printf "%s-kafka" (include "navi-async-matrix.fullname" .) | mustToJson }},
+            "name": {{ printf "%s-producer" (include "navi-async-matrix.fullname" .) | mustToJson }},
             "secret": {
-              "secretName": {{ printf "%s-kafka" (include "navi-async-matrix.fullname" .) | mustToJson }}
+              "secretName": {{ printf "%s-producer" (include "navi-async-matrix.fullname" .) | mustToJson }}
             }
           }
         ],
@@ -31,11 +31,11 @@ data:
             "name": null,
             "image": "{{ .Values.dgctlDockerRegistry }}/{{ .Values.dm.archiver.image.repository }}:{{ .Values.dm.archiver.image.tag }}",
             "resources": {{ .Values.dm.archiver.resources | mustToJson }},
-            {{- if .Values.kafka.fileProperties }}
+            {{- if .Values.kafka.fileProperties | or (.Values.kafka.producerOverrides).fileProperties }}
             "volumeMounts": [
               {
-                "name": {{ printf "%s-kafka" (include "navi-async-matrix.fullname" .) | mustToJson }},
-                "mountPath": "/app/secret/"
+                "name": {{ printf "%s-producer" (include "navi-async-matrix.fullname" .) | mustToJson }},
+                "mountPath": "/app/secret/producer"
               }
             ],
             {{- end }}
@@ -50,11 +50,12 @@ data:
               },
             {{- $kafkaPropertiesEnv := include "navi-async-matrix.kafkaPropertiesEnv" (dict
                   "kafka" .Values.kafka
+                  "overrides" .Values.kafka.producerOverrides
                   "secretname" (print
                       (include "navi-async-matrix.fullname" .)
-                      "-kafka"
+                      "-producer"
                     )
-                  "mountpoint" "/app/secret"
+                  "mountpoint" "/app/secret/producer"
                   "prefix" "KAFKA_PRODUCER_SETTING__CONFIG__"
                 ) | fromYaml -}}
             {{- include "navi-async-matrix.partialListToJson" $kafkaPropertiesEnv | nindent 14 -}}

diff --git a/charts/navi-async-matrix/templates/role.yaml b/charts/navi-async-matrix/templates/role.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.dm.archiver.enabled }}
+{{- if and .Values.rbac.create .Values.dm.archiver.enabled }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
@@ -31,4 +31,4 @@ rules:
   - patch
   - update
   - watch
-{{- end }}{{- /* .Values.dm.archiver.enabled */}}
+{{- end }}{{- /* .Values.rbac.create .Values.dm.archiver.enabled */}}
diff --git a/charts/navi-async-matrix/templates/rolebinding.yaml b/charts/navi-async-matrix/templates/rolebinding.yaml
@@ -1,4 +1,4 @@
-{{- if .Values.dm.archiver.enabled }}
+{{- if and .Values.rbac.create .Values.dm.archiver.enabled }}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
@@ -19,4 +19,4 @@ roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
   name: {{ include "navi-async-matrix.fullname" . }}
-{{- end }}{{- /* .Values.dm.archiver.enabled */}}
+{{- end }}{{- /* .Values.rbac.create .Values.dm.archiver.enabled */}}