[Feature][K8S deploy] add helm chart (#8291)

.github/workflows/publish-helm-chart.yaml

@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: publish-helm-chart
+
+on:
+  push:
+    tags:
+      - '*'
+    paths-ignore:
+      - 'docs/**'
+      - '**/*.md'
+
+env:
+  DOCKER_USERNAME: ${{ secrets.DOCKERHUB_USER }}
+  DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
+  DOCKER_REGISTRY: docker.io
+  HUB: registry-1.docker.io/apache
+
+jobs:
+  build:
+    if: github.repository == 'apache/seatunnel'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.DOCKER_REGISTRY }}
+          username: ${{ env.DOCKER_USERNAME }}
+          password: ${{ env.DOCKER_PASSWORD }}
+      - name: Publish Helm Chart
+        working-directory: deploy/kubernetes
+        run: |
+          helm dep up seatunnel
+          helm package seatunnel
+          helm push seatunnel-helm-*.tgz oci://${{ env.HUB }}

deploy/kubernetes/seatunnel/Chart.yaml

@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+apiVersion: v2
+name: seatunnel-helm
+description: SeaTunnel is a next-generation, high-performance, distributed data integration tool, capable of synchronizing vast amounts of data daily. It's trusted by numerous companies for its efficiency and stability.
+home: https://seatunnel.apache.org
+icon: https://seatunnel.apache.org/image/logo.png
+keywords:
+  - seatunnel
+  - integration
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+version: 2.3.9
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application.
+appVersion: 2.3.9

deploy/kubernetes/seatunnel/conf/hazelcast-client.yaml

@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hazelcast-client:
+  cluster-name: {{ include "seatunnel.fullname" . }}
+  properties:
+    hazelcast.logging.type: log4j2
+  connection-strategy:
+    connection-retry:
+      cluster-connect-timeout-millis: 3000
+  network:
+    cluster-members:
+      - {{ include "seatunnel.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local:5801

deploy/kubernetes/seatunnel/conf/hazelcast-master.yaml

@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hazelcast:
+  cluster-name: {{ include "seatunnel.fullname" . }}
+  network:
+    rest-api:
+      enabled: true
+      endpoint-groups:
+        CLUSTER_WRITE:
+          enabled: true
+        DATA:
+          enabled: true
+    join:
+      kubernetes:
+        enabled: true
+        service-dns: {{ include "seatunnel.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local
+        service-port: 5801
+    port:
+      auto-increment: false
+      port: 5801
+  properties:
+    hazelcast.invocation.max.retry.count: 20
+    hazelcast.tcp.join.port.try.count: 30
+    hazelcast.logging.type: log4j2
+    hazelcast.operation.generic.thread.count: 50
+    hazelcast.heartbeat.failuredetector.type: phi-accrual
+    hazelcast.heartbeat.interval.seconds: 2
+    hazelcast.max.no.heartbeat.seconds: 180
+    hazelcast.heartbeat.phiaccrual.failuredetector.threshold: 10
+    hazelcast.heartbeat.phiaccrual.failuredetector.sample.size: 200
+    hazelcast.heartbeat.phiaccrual.failuredetector.min.std.dev.millis: 100

deploy/kubernetes/seatunnel/conf/hazelcast-worker.yaml

@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hazelcast:
+  cluster-name: {{ include "seatunnel.fullname" . }}
+  network:
+    rest-api:
+      enabled: true
+      endpoint-groups:
+        CLUSTER_WRITE:
+          enabled: true
+        DATA:
+          enabled: true
+    join:
+      kubernetes:
+        enabled: true
+        service-dns: {{ include "seatunnel.fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local
+        service-port: 5801
+    port:
+      auto-increment: false
+      port: 5801
+  properties:
+    hazelcast.invocation.max.retry.count: 20
+    hazelcast.tcp.join.port.try.count: 30
+    hazelcast.logging.type: log4j2
+    hazelcast.operation.generic.thread.count: 50
+    hazelcast.heartbeat.failuredetector.type: phi-accrual
+    hazelcast.heartbeat.interval.seconds: 2
+    hazelcast.max.no.heartbeat.seconds: 180
+    hazelcast.heartbeat.phiaccrual.failuredetector.threshold: 10
+    hazelcast.heartbeat.phiaccrual.failuredetector.sample.size: 200
+    hazelcast.heartbeat.phiaccrual.failuredetector.min.std.dev.millis: 100
+  member-attributes:
+    rule:
+      type: string
+      value: worker

deploy/kubernetes/seatunnel/conf/log4j2.properties

@@ -0,0 +1,103 @@
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# The minimum amount of time, in seconds, that must elapse before the file configuration is checked for changes.
+monitorInterval = 60
+
+property.file_path = ${sys:seatunnel.logs.path:-/tmp/seatunnel/logs}
+property.file_name = ${sys:seatunnel.logs.file_name:-seatunnel}
+property.file_split_size = 100MB
+property.file_count = 100
+property.file_ttl = 7d
+
+rootLogger.level = INFO
+
+logger.zeta.name=org.apache.seatunnel.engine
+logger.zeta.level=INFO
+
+logger.debezium.name=io.debezium.connector
+logger.debezium.level=WARN
+
+############################ log output to console #############################
+rootLogger.appenderRef.consoleStdout.ref = consoleStdoutAppender
+rootLogger.appenderRef.consoleStderr.ref = consoleStderrAppender
+############################ log output to console #############################
+############################ log output to file    #############################
+#rootLogger.appenderRef.file.ref = fileAppender
+############################ log output to file    #############################
+
+appender.consoleStdout.name = consoleStdoutAppender
+appender.consoleStdout.type = CONSOLE
+appender.consoleStdout.target = SYSTEM_OUT
+appender.consoleStdout.layout.type = PatternLayout
+appender.consoleStdout.layout.pattern = [%X{ST-JID}] %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n
+appender.consoleStdout.filter.acceptLtWarn.type = ThresholdFilter
+appender.consoleStdout.filter.acceptLtWarn.level = WARN
+appender.consoleStdout.filter.acceptLtWarn.onMatch = DENY
+appender.consoleStdout.filter.acceptLtWarn.onMismatch = ACCEPT
+
+appender.consoleStderr.name = consoleStderrAppender
+appender.consoleStderr.type = CONSOLE
+appender.consoleStderr.target = SYSTEM_ERR
+appender.consoleStderr.layout.type = PatternLayout
+appender.consoleStderr.layout.pattern = [%X{ST-JID}] %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n
+appender.consoleStderr.filter.acceptGteWarn.type = ThresholdFilter
+appender.consoleStderr.filter.acceptGteWarn.level = WARN
+appender.consoleStderr.filter.acceptGteWarn.onMatch = ACCEPT
+appender.consoleStderr.filter.acceptGteWarn.onMismatch = DENY
+
+appender.routing.name = routingAppender
+appender.routing.type = Routing
+appender.routing.purge.type = IdlePurgePolicy
+appender.routing.purge.timeToLive = 60
+appender.routing.route.type = Routes
+appender.routing.route.pattern = $${ctx:ST-JID}
+appender.routing.route.system.type = Route
+appender.routing.route.system.key = $${ctx:ST-JID}
+appender.routing.route.system.ref = fileAppender
+appender.routing.route.job.type = Route
+appender.routing.route.job.appender.type = File
+appender.routing.route.job.appender.name = job-${ctx:ST-JID}
+appender.routing.route.job.appender.fileName = ${file_path}/job-${ctx:ST-JID}.log
+appender.routing.route.job.appender.layout.type = PatternLayout
+appender.routing.route.job.appender.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n
+
+appender.file.name = fileAppender
+appender.file.type = RollingFile
+appender.file.fileName = ${file_path}/${file_name}.log
+appender.file.filePattern = ${file_path}/${file_name}.log.%d{yyyy-MM-dd}-%i
+appender.file.append = true
+appender.file.layout.type = PatternLayout
+appender.file.layout.pattern = [%X{ST-JID}] %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%-30.30c{1.}] [%t] - %m%n
+appender.file.policies.type = Policies
+appender.file.policies.time.type = TimeBasedTriggeringPolicy
+appender.file.policies.time.modulate = true
+appender.file.policies.size.type = SizeBasedTriggeringPolicy
+appender.file.policies.size.size = ${file_split_size}
+appender.file.strategy.type = DefaultRolloverStrategy
+appender.file.strategy.fileIndex = nomax
+appender.file.strategy.action.type = Delete
+appender.file.strategy.action.basepath = ${file_path}
+appender.file.strategy.action.maxDepth = 1
+appender.file.strategy.action.condition.type = IfFileName
+appender.file.strategy.action.condition.glob = ${file_name}.log*
+appender.file.strategy.action.condition.nested_condition.type = IfAny
+appender.file.strategy.action.condition.nested_condition.lastModify.type = IfLastModified
+appender.file.strategy.action.condition.nested_condition.lastModify.age = ${file_ttl}
+appender.file.strategy.action.condition.nested_condition.fileCount.type = IfAccumulatedFileCount
+appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}

deploy/kubernetes/seatunnel/conf/seatunnel.yaml

@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+seatunnel:
+  engine:
+    history-job-expire-minutes: 1440
+    backup-count: 1
+    queue-type: blockingqueue
+    print-execution-info-interval: 60
+    print-job-metrics-info-interval: 60
+    classloader-cache-mode: true
+    slot-service:
+      dynamic-slot: true
+    http:
+      enable-http: true
+      port: 8080
+      enable-dynamic-port: false
+      port-range: 100
+    checkpoint:
+      interval: 300000
+      timeout: 10000
+      storage:
+        type: hdfs
+        max-retained: 3
+        plugin-config:
+          namespace: /tmp/seatunnel/checkpoint_snapshot/
+          storage.type: hdfs
+          fs.defaultFS: file:///tmp/
+    telemetry:
+      metric:
+        enabled: true