chore: Update Kafka blueprint (awslabs#373)

streaming/kafka/examples/kafka-producers-consumers.yaml

@@ -0,0 +1,110 @@
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: java-kafka-producer
+  name: java-kafka-producer
+  namespace: kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: java-kafka-producer
+  template:
+    metadata:
+      labels:
+        app: java-kafka-producer
+    spec:
+      containers:
+        - name: java-kafka-producer
+          image: quay.io/strimzi-examples/java-kafka-producer:latest
+          env:
+            - name: STRIMZI_TOPIC
+              value: my-topic
+            - name: STRIMZI_DELAY_MS
+              value: "1000"
+            - name: STRIMZI_LOG_LEVEL
+              value: "INFO"
+            - name: STRIMZI_MESSAGE_COUNT
+              value: "1000000"
+            - name: KAFKA_BOOTSTRAP_SERVERS
+              value: cluster-kafka-bootstrap:9092
+            - name: KAFKA_KEY_SERIALIZER
+              value: "org.apache.kafka.common.serialization.StringSerializer"
+            - name: KAFKA_VALUE_SERIALIZER
+              value: "org.apache.kafka.common.serialization.StringSerializer"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: java-kafka-streams
+  name: java-kafka-streams
+  namespace: kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: java-kafka-streams
+  template:
+    metadata:
+      labels:
+        app: java-kafka-streams
+    spec:
+      containers:
+        - name: java-kafka-streams
+          image: quay.io/strimzi-examples/java-kafka-streams:latest
+          env:
+            - name: STRIMZI_SOURCE_TOPIC
+              value: my-topic
+            - name: STRIMZI_TARGET_TOPIC
+              value: my-topic-reversed
+            - name: STRIMZI_LOG_LEVEL
+              value: "INFO"
+            - name: KAFKA_BOOTSTRAP_SERVERS
+              value: cluster-kafka-bootstrap:9092
+            - name: KAFKA_APPLICATION_ID
+              value: java-kafka-streams
+            - name: KAFKA_DEFAULT_COMMIT_INTERVAL_MS
+              value: "5000"
+            - name: KAFKA_DEFAULT_KEY_SERDE
+              value: "org.apache.kafka.common.serialization.Serdes$StringSerde"
+            - name: KAFKA_DEFAULT_VALUE_SERDE
+              value: "org.apache.kafka.common.serialization.Serdes$StringSerde"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: java-kafka-consumer
+  name: java-kafka-consumer
+  namespace: kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: java-kafka-consumer
+  template:
+    metadata:
+      labels:
+        app: java-kafka-consumer
+    spec:
+      containers:
+        - name: java-kafka-consumer
+          image: quay.io/strimzi-examples/java-kafka-consumer:latest
+          env:
+            - name: STRIMZI_TOPIC
+              value: my-topic-reversed
+            - name: STRIMZI_LOG_LEVEL
+              value: "INFO"
+            - name: STRIMZI_MESSAGE_COUNT
+              value: "1000000"
+            - name: KAFKA_BOOTSTRAP_SERVERS
+              value: cluster-kafka-bootstrap:9092
+            - name: KAFKA_GROUP_ID
+              value: java-kafka-consumer
+            - name: KAFKA_KEY_DESERIALIZER
+              value: "org.apache.kafka.common.serialization.StringDeserializer"
+            - name: KAFKA_VALUE_DESERIALIZER
+              value: "org.apache.kafka.common.serialization.StringDeserializer"

streaming/kafka/examples/kafka-topics.yaml

@@ -1,10 +1,21 @@
-apiVersion: kafka.strimzi.io/v1beta1
+apiVersion: kafka.strimzi.io/v1beta2
 kind: KafkaTopic
 metadata:
-  name: test-topic
+  name: my-topic
   namespace: kafka
   labels:
     strimzi.io/cluster: cluster
 spec:
-  partitions: 3
   replicas: 3
+  partitions: 12
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: my-topic-reversed
+  namespace: kafka
+  labels:
+    strimzi.io/cluster: cluster
+spec:
+  replicas: 3
+  partitions: 12

streaming/kafka/helm-values/strimzi-kafka-values.yaml

@@ -10,7 +10,7 @@ watchAnyNamespace: true
 
 defaultImageRegistry: quay.io
 defaultImageRepository: strimzi
-defaultImageTag: 0.35.0
+defaultImageTag: 0.38.0
 
 nodeSelector:
   kubernetes.io/os: ${operating_system}

streaming/kafka/install.sh

@@ -12,6 +12,9 @@ targets=(
   "module.eks_data_addons"
 )
 
+# Initialize Terraform
+terraform init -upgrade
+
 # Apply modules in sequence
 for target in "${targets[@]}"
 do

streaming/kafka/kafka-manifests/kafka-cluster.yaml

@@ -4,10 +4,9 @@ metadata:
   name: cluster
   namespace: kafka
 spec:
-  cruiseControl: {}
   kafka:
-    version: 3.4.0
-    replicas: 3
+    version: 3.6.0
+    replicas: 4
     listeners:
       - name: plain
         port: 9092
@@ -17,13 +16,15 @@ spec:
         port: 9093
         type: internal
         tls: true
+        authentication:
+          type: tls
     config:
       offsets.topic.replication.factor: 3
       transaction.state.log.replication.factor: 3
       transaction.state.log.min.isr: 2
       default.replication.factor: 3
       min.insync.replicas: 2
-      inter.broker.protocol.version: "3.4"
+      inter.broker.protocol.version: "3.6"
     resources:
       requests:
         memory: 58Gi
@@ -37,11 +38,11 @@ spec:
     storage:
       type: jbod
       volumes:
-      - id: 0
-        type: persistent-claim
-        size: 1000Gi
-        class: gp3
-        deleteClaim: false
+        - id: 0
+          type: persistent-claim
+          size: 1000Gi
+          class: gp3
+          deleteClaim: false
     template:
       pod:
         tolerations:
@@ -54,10 +55,10 @@ spec:
             requiredDuringSchedulingIgnoredDuringExecution:
               nodeSelectorTerms:
                 - matchExpressions:
-                  - key: NodeGroupType
-                    operator: In
-                    values:
-                    - kafka
+                    - key: NodeGroupType
+                      operator: In
+                      values:
+                        - kafka
     metricsConfig:
       type: jmxPrometheusExporter
       valueFrom:
@@ -66,7 +67,6 @@ spec:
           key: kafka-metrics-config.yml
     rack:
       topologyKey: topology.kubernetes.io/zone
-  kafkaExporter: {}
   zookeeper:
     replicas: 3
     storage:
@@ -83,6 +83,10 @@ spec:
   entityOperator:
     topicOperator: {}
     userOperator: {}
+  cruiseControl: {}
+  kafkaExporter:
+    topicRegex: ".*"
+    groupRegex: ".*"
 
 ---
 kind: ConfigMap
@@ -248,3 +252,10 @@ data:
       labels:
         replicaId: "$2"
         memberType: "$3"
+  cruise-control-config.yml: |
+    # See https://github.com/prometheus/jmx_exporter for more info about JMX Prometheus Exporter metrics
+    lowercaseOutputName: true
+    rules:
+    - pattern: kafka.cruisecontrol<name=(.+)><>(\w+)
+      name: kafka_cruisecontrol_$1_$2
+      type: GAUGE

streaming/kafka/kafka-manifests/kafka-rebalance.yaml

@@ -0,0 +1,22 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaRebalance
+metadata:
+  name: my-rebalance
+  namespace: kafka
+  labels:
+    strimzi.io/cluster: cluster
+spec:
+  goals:
+    - RackAwareGoal
+    - ReplicaCapacityGoal
+    - DiskCapacityGoal
+    - NetworkInboundCapacityGoal
+    - NetworkOutboundCapacityGoal
+    - CpuCapacityGoal
+    - ReplicaDistributionGoal
+    - DiskUsageDistributionGoal
+    - NetworkInboundUsageDistributionGoal
+    - NetworkOutboundUsageDistributionGoal
+    - TopicReplicaDistributionGoal
+    - LeaderReplicaDistributionGoal
+    - LeaderBytesInDistributionGoal

streaming/kafka/main.tf

@@ -75,7 +75,7 @@ module "eks" {
       name        = "core-node-group"
       description = "EKS managed node group example launch template"
 
-      min_size     = 1
+      min_size     = 3
       max_size     = 9
       desired_size = 3
 
@@ -86,7 +86,7 @@ module "eks" {
         xvda = {
           device_name = "/dev/xvda"
           ebs = {
-            volume_size = 100
+            volume_size = 1000
             volume_type = "gp3"
           }
         }
@@ -99,6 +99,7 @@ module "eks" {
         Name = "core-node-grp"
       }
     }
+
     kafka_node_group = {
       name        = "kafka-node-group"
       description = "EKS managed node group example launch template"
@@ -108,13 +109,14 @@ module "eks" {
       desired_size = 5
 
       instance_types = ["r6i.2xlarge"]
-      ebs_optimized  = true
+
+      ebs_optimized = true
       # This is the root filesystem Not used by the brokers
       block_device_mappings = {
         xvda = {
           device_name = "/dev/xvda"
           ebs = {
-            volume_size = 100
+            volume_size = 1000
             volume_type = "gp3"
           }
         }