chore: support parameters api for pulsar

addons-cluster/pulsar/templates/cluster.yaml

@@ -10,7 +10,7 @@ metadata:
   {{- end }}
 spec:
   terminationPolicy: {{ $.Values.terminationPolicy }}
-  {{- $topologyEnable :=  eq .Values.mode "" | default false }}
+  {{- $topologyEnable :=  ne .Values.mode "" | default false }}
   {{- if $topologyEnable}}
   clusterDef: pulsar
   topology: {{ .Values.mode }}

addons/pulsar/config/2.11.2/bookies-config.tpl

@@ -75,4 +75,6 @@ zkLedgersRootPath=/ledgers
     {{- $autoRecoveryDaemonEnabled = "false" }}
   {{- end }}
 {{- end }}
-autoRecoveryDaemonEnabled={{ $autoRecoveryDaemonEnabled }}
+autoRecoveryDaemonEnabled={{ $autoRecoveryDaemonEnabled }}
+
+zkServers={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/2.11.2/bookies-recovery-config.tpl

@@ -0,0 +1,9 @@
+httpServerEnabled=true
+httpServerPort=8000
+prometheusStatsHttpPort=8000
+useHostNameAsBookieID=true
+# how long to wait, in seconds, before starting autorecovery of a lost bookie.
+# TODO: set to 0 after opsRequest for rollingUpdate supports hooks
+lostBookieRecoveryDelay=300
+
+zkServers={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/2.11.2/broker-config.tpl

@@ -57,3 +57,6 @@ brokerDeleteInactiveTopicsEnabled=false
 # configurations in KoP because of KIP-679.
 kafkaTransactionCoordinatorEnabled=true
 brokerDeduplicationEnabled=true
+
+zookeeperServers={{ .ZOOKEEPER_SERVERS }}
+configurationStoreServers={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/2.11.2/proxy-config.tpl

@@ -1,2 +1,4 @@
 statusFilePath=/pulsar/status
-maxMessageSize=5242880
+maxMessageSize=5242880
+
+metadataStoreUrl={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/2.11.2/zookeeper-config.tpl

@@ -0,0 +1,2 @@
+dataDir=/pulsar/data/zookeeper
+serverCnxnFactory=org.apache.zookeeper.server.NIOServerCnxnFactory

addons/pulsar/config/3.0.2/bookies-config.tpl

@@ -78,3 +78,5 @@ zkLedgersRootPath=/ledgers
   {{- end }}
 {{- end }}
 autoRecoveryDaemonEnabled={{ $autoRecoveryDaemonEnabled }}
+
+zkServers={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/3.0.2/bookies-recovery-config.tpl

@@ -0,0 +1,9 @@
+httpServerEnabled=true
+httpServerPort=8000
+prometheusStatsHttpPort=8000
+useHostNameAsBookieID=true
+# how long to wait, in seconds, before starting autorecovery of a lost bookie.
+# TODO: set to 0 after opsRequest for rollingUpdate supports hooks
+lostBookieRecoveryDelay=300
+
+zkServers={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/3.0.2/broker-config.tpl

@@ -57,3 +57,6 @@ brokerDeleteInactiveTopicsEnabled=false
 # configurations in KoP because of KIP-679.
 kafkaTransactionCoordinatorEnabled=true
 brokerDeduplicationEnabled=true
+
+zookeeperServers={{ .ZOOKEEPER_SERVERS }}
+configurationStoreServers={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/3.0.2/proxy-config.tpl

@@ -1,2 +1,4 @@
 statusFilePath=/pulsar/status
-maxMessageSize=5242880
+maxMessageSize=5242880
+
+metadataStoreUrl={{ .ZOOKEEPER_SERVERS }}

addons/pulsar/config/3.0.2/zookeeper-config.tpl

@@ -0,0 +1,2 @@
+dataDir=/pulsar/data/zookeeper
+serverCnxnFactory=org.apache.zookeeper.server.NIOServerCnxnFactory

addons/pulsar/config/pulsar-env-optimize.tpl

@@ -0,0 +1,9 @@
+## for proxy or broker
+export PULSAR_EXTRA_OPTS="-Dpulsar.allocator.exit_on_oom=true -Dio.netty.recycler.maxCapacity.default=1000 -Dio.netty.recycler.linkCapacity=1024 -Dnetworkaddress.cache.ttl=60 -XX:ActiveProcessorCount=1 -XshowSettings:vm -Ddepth=64"
+export PULSAR_GC="-XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+DoEscapeAnalysis -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB"
+{{- $maxDirectMemorySize := "" }}
+{{- $phyMemory := getContainerMemory ( index $.podSpec.containers 0 ) }}
+{{- if gt $phyMemory 0 }}
+  {{- $maxDirectMemorySize = printf "-XX:MaxDirectMemorySize=%dm" (mul (div $phyMemory ( mul 1024 1024 10)) 6) }}
+{{- end }}
+export PULSAR_MEM="-XX:MinRAMPercentage=30 -XX:MaxRAMPercentage=30 {{ $maxDirectMemorySize }}"

addons/pulsar/config/pulsar-env.tpl

@@ -0,0 +1,8 @@
+export PULSAR_GC="-XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=4 -XX:ConcGCThreads=4 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -XshowSettings:vm -Ddepth=64"
+{{- $maxDirectMemorySize := "" }}
+{{- $phyMemory := getContainerMemory ( index $.podSpec.containers 0 ) }}
+{{- if gt $phyMemory 0 }}
+  {{- $phyMemoryMB := div $phyMemory ( mul 1024 1024 ) }}
+  {{- $maxDirectMemorySize = printf "-XX:MaxDirectMemorySize=%dm" (div ( mul $phyMemoryMB 3 ) 4 ) }}
+{{- end }}
+export PULSAR_MEM="-XX:MinRAMPercentage=25 -XX:MaxRAMPercentage=50 {{ $maxDirectMemorySize }}"

addons/pulsar/config/zookeeper-env.tpl

@@ -1,5 +1,2 @@
-PULSAR_GC: -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -Dcom.sun.management.jmxremote -Djute.maxbuffer=10485760 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+DoEscapeAnalysis -XX:+DisableExplicitGC -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -XX:ActiveProcessorCount=1
-PULSAR_MEM: -XX:MinRAMPercentage=40 -XX:MaxRAMPercentage=60
-PULSAR_PREFIX_serverCnxnFactory: org.apache.zookeeper.server.NIOServerCnxnFactory
-dataDir: /pulsar/data/zookeeper
-serverCnxnFactory: org.apache.zookeeper.server.NIOServerCnxnFactory
+export PULSAR_GC="-XX:+UseG1GC -XX:MaxGCPauseMillis=10 -Dcom.sun.management.jmxremote -Djute.maxbuffer=10485760 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+DoEscapeAnalysis -XX:+DisableExplicitGC -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -XX:ActiveProcessorCount=1"
+export PULSAR_MEM="-XX:MinRAMPercentage=40 -XX:MaxRAMPercentage=60"

addons/pulsar/scripts-ut-spec/init_bookies_spec.sh

@@ -89,7 +89,7 @@ Describe "Pulsar Init Bookies Bash Script Tests"
 
   Describe "init_bookies()"
     It "initializes bookies"
-      export zkServers="zookeeper.example.com:2181"
+      export ZOOKEEPER_SERVERS="zookeeper.example.com:2181"
 
       wait_for_zookeeper() {
         echo "wait_for_zookeeper called"
@@ -114,11 +114,11 @@ Describe "Pulsar Init Bookies Bash Script Tests"
       The output should include "init_bookkeeper_cluster called"
     End
 
-    It "exits with status 1 when zkServers environment variable is not set"
-      unset zkServers
+    It "exits with status 1 when ZOOKEEPER_SERVERS environment variable is not set"
+      unset ZOOKEEPER_SERVERS
 
       When run init_bookies
-      The output should include "Error: zkServers environment variable is not set, Please set the zkServers environment variable and try again."
+      The output should include "Error: ZOOKEEPER_SERVERS environment variable is not set, Please set the ZOOKEEPER_SERVERS environment variable and try again."
       The status should be failure
     End
   End

addons/pulsar/scripts/check-bookies.sh

@@ -23,6 +23,24 @@ wait_for_bookkeeper() {
   echo "Bookkeeper started successfully"
 }
 
+merge_bookkeeper_config() {
+  local src_config="conf/bookkeeper.conf"
+  local dest_config="/opt/pulsar/conf/bookkeeper.conf"
+
+  echo "Merging Pulsar configuration files:"
+  echo "  - Source: $src_config"
+  echo "  - Destination: $dest_config"
+  python3 /kb-scripts/merge_pulsar_config.py "$src_config" "$dest_config"
+}
+
+load_env_file() {
+  local pulsar_env_config="/opt/pulsar/conf/pulsar.env"
+
+  if [ -f "${pulsar_env_config}" ];then
+     source ${pulsar_env_config}
+  fi
+}
+
 set_tcp_keepalive() {
   local keepalive_time=1
   local keepalive_intvl=11
@@ -46,6 +64,8 @@ set_tcp_keepalive() {
 ${__SOURCED__:+false} : || return 0
 
 # main
+load_env_file
+merge_bookkeeper_config
 apply_config_from_env
 wait_for_bookkeeper
 set_tcp_keepalive

addons/pulsar/scripts/init-bookies.sh

@@ -46,15 +46,24 @@ init_bookkeeper_cluster() {
   fi
 }
 
+load_env_file() {
+  local pulsar_env_config="/opt/pulsar/conf/pulsar.env"
+
+  if [ -f "${pulsar_env_config}" ];then
+     source ${pulsar_env_config}
+  fi
+}
+
 init_bookies() {
-  if [[ -z "$zkServers" ]]; then
+  if [[ -z "$ZOOKEEPER_SERVERS" ]]; then
     echo "Error: zkServers environment variable is not set, Please set the zkServers environment variable and try again."
     exit 1
   fi
 
-  wait_for_zookeeper "$zkServers"
+  wait_for_zookeeper "$ZOOKEEPER_SERVERS"
   merge_bookkeeper_config
   apply_config_from_env
+  load_env_file
   init_bookkeeper_cluster
 }