From 1dbc215451345575a974970e30ab1c5c2bf69235 Mon Sep 17 00:00:00 2001 From: Dominic Evans Date: Fri, 18 Aug 2023 12:01:15 +0100 Subject: [PATCH] feat(fvt): test wider range of kafkas Experiment with whether we can successfully run against a wider range of kafka broker versions. Assuming this is successful, we could run a larger FV against the main branch, whilst keeping the PR builds to a smaller subset of versions using GitHub actions re-usable workflows https://docs.github.com/en/actions/using-workflows/reusing-workflows#using-a-matrix-strategy-with-a-reusable-workflow Signed-off-by: Dominic Evans --- .github/workflows/fvt.yml | 20 ++++++++++++++- docker-compose.yml | 54 +++++++++++++++++++++------------------ 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/.github/workflows/fvt.yml b/.github/workflows/fvt.yml index 5b8ed844be..06be3fb6aa 100644 --- a/.github/workflows/fvt.yml +++ b/.github/workflows/fvt.yml @@ -15,11 +15,29 @@ jobs: fail-fast: false matrix: go-version: [1.21.x] - kafka-version: [3.3.2, 3.4.1, 3.5.1] + kafka-version: [1.0.2, 2.0.1, 2.2.2, 2.6.2, 2.8.2, 3.0.2, 3.3.2, 3.5.1] + include: + - kafka-version: 1.0.2 + scala-version: 2.11 + - kafka-version: 2.0.1 + scala-version: 2.12 + - kafka-version: 2.2.2 + scala-version: 2.12 + - kafka-version: 2.6.2 + scala-version: 2.12 + - kafka-version: 2.8.2 + scala-version: 2.12 + - kafka-version: 3.0.2 + scala-version: 2.12 + - kafka-version: 3.3.2 + scala-version: 2.13 + - kafka-version: 3.5.1 + scala-version: 2.13 env: DEBUG: true GOFLAGS: -trimpath KAFKA_VERSION: ${{ matrix.kafka-version }} + SCALA_VERSION: ${{ matrix.scala-version }} steps: - uses: actions/checkout@v3 - name: Setup Docker diff --git a/docker-compose.yml b/docker-compose.yml index 2f852e4222..2b1c099ca0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ version: '3.9' services: zookeeper-1: + hostname: 'zookeeper-1' image: 'docker.io/library/zookeeper:3.6.3' restart: always environment: @@ -12,6 +13,7 @@ services: ZOO_MAX_CLIENT_CNXNS: '0' ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok' zookeeper-2: + hostname: 'zookeeper-2' image: 'docker.io/library/zookeeper:3.6.3' restart: always environment: @@ -23,6 +25,7 @@ services: ZOO_MAX_CLIENT_CNXNS: '0' ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok' zookeeper-3: + hostname: 'zookeeper-3' image: 'docker.io/library/zookeeper:3.6.3' restart: always environment: @@ -34,6 +37,7 @@ services: ZOO_MAX_CLIENT_CNXNS: '0' ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok' kafka-1: + hostname: 'kafka-1' image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . @@ -45,12 +49,9 @@ services: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh', + '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-1:9091', - '--broker', - '1', - '--describe', ] interval: 15s timeout: 15s @@ -66,7 +67,7 @@ services: KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091' - KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091' + KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://127.0.0.1:29091' KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL' KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT' KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2' @@ -78,7 +79,10 @@ services: KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' + KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions" kafka-2: + hostname: 'kafka-2' image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . @@ -90,12 +94,9 @@ services: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh', + '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-2:9091', - '--broker', - '2', - '--describe', ] interval: 15s timeout: 15s @@ -111,7 +112,7 @@ services: KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092' - KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092' + KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://127.0.0.1:29092' KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL' KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT' KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2' @@ -123,7 +124,10 @@ services: KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' + KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions" kafka-3: + hostname: 'kafka-3' image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . @@ -135,12 +139,9 @@ services: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh', + '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-3:9091', - '--broker', - '3', - '--describe', ] interval: 15s timeout: 15s @@ -156,7 +157,7 @@ services: KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093' - KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093' + KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://127.0.0.1:29093' KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL' KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT' KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2' @@ -168,7 +169,10 @@ services: KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' + KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions" kafka-4: + hostname: 'kafka-4' image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . @@ -180,12 +184,9 @@ services: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh', + '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-4:9091', - '--broker', - '4', - '--describe', ] interval: 15s timeout: 15s @@ -201,7 +202,7 @@ services: KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094' - KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094' + KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://127.0.0.1:29094' KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL' KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT' KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2' @@ -213,7 +214,10 @@ services: KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' + KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions" kafka-5: + hostname: 'kafka-5' image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}' build: context: . @@ -225,12 +229,9 @@ services: test: [ 'CMD', - '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh', + '/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh', '--bootstrap-server', 'kafka-5:9091', - '--broker', - '5', - '--describe', ] interval: 15s timeout: 15s @@ -246,7 +247,7 @@ services: KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1} KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095' - KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095' + KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://127.0.0.1:29095' KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL' KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT' KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2' @@ -258,7 +259,10 @@ services: KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true' KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false' + KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions" toxiproxy: + hostname: 'toxiproxy' image: 'ghcr.io/shopify/toxiproxy:2.4.0' healthcheck: test: ['CMD', '/toxiproxy-cli', 'l']