Skip to content

Commit

Permalink
feat(fvt): test wider range of kafkas
Browse files Browse the repository at this point in the history 
Experiment with whether we can successfully run against a wider range of
kafka broker versions. Assuming this is successful, we could run a
larger FV against the main branch, whilst keeping the PR builds to a
smaller subset of versions using GitHub actions re-usable workflows
https://docs.github.com/en/actions/using-workflows/reusing-workflows#using-a-matrix-strategy-with-a-reusable-workflow

Signed-off-by: Dominic Evans <[email protected]>
  • Loading branch information
@dnwe
dnwe committed Aug 18, 2023
1 parent 8681621 commit 1dbc215
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 26 deletions.
20 changes: 19 additions & 1 deletion .github/workflows/fvt.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,29 @@ jobs:
fail-fast: false
matrix:
go-version: [1.21.x]
kafka-version: [3.3.2, 3.4.1, 3.5.1]
kafka-version: [1.0.2, 2.0.1, 2.2.2, 2.6.2, 2.8.2, 3.0.2, 3.3.2, 3.5.1]
include:
- kafka-version: 1.0.2
scala-version: 2.11
- kafka-version: 2.0.1
scala-version: 2.12
- kafka-version: 2.2.2
scala-version: 2.12
- kafka-version: 2.6.2
scala-version: 2.12
- kafka-version: 2.8.2
scala-version: 2.12
- kafka-version: 3.0.2
scala-version: 2.12
- kafka-version: 3.3.2
scala-version: 2.13
- kafka-version: 3.5.1
scala-version: 2.13
env:
DEBUG: true
GOFLAGS: -trimpath
KAFKA_VERSION: ${{ matrix.kafka-version }}
SCALA_VERSION: ${{ matrix.scala-version }}
steps:
- uses: actions/checkout@v3
- name: Setup Docker
Expand Down
54 changes: 29 additions & 25 deletions docker-compose.yml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
version: '3.9'
services:
zookeeper-1:
hostname: 'zookeeper-1'
image: 'docker.io/library/zookeeper:3.6.3'
restart: always
environment:
Expand All @@ -12,6 +13,7 @@ services:
ZOO_MAX_CLIENT_CNXNS: '0'
ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
zookeeper-2:
hostname: 'zookeeper-2'
image: 'docker.io/library/zookeeper:3.6.3'
restart: always
environment:
Expand All @@ -23,6 +25,7 @@ services:
ZOO_MAX_CLIENT_CNXNS: '0'
ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
zookeeper-3:
hostname: 'zookeeper-3'
image: 'docker.io/library/zookeeper:3.6.3'
restart: always
environment:
Expand All @@ -34,6 +37,7 @@ services:
ZOO_MAX_CLIENT_CNXNS: '0'
ZOO_4LW_COMMANDS_WHITELIST: 'mntr,conf,ruok'
kafka-1:
hostname: 'kafka-1'
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
build:
context: .
Expand All @@ -45,12 +49,9 @@ services:
test:
[
'CMD',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-1:9091',
'--broker',
'1',
'--describe',
]
interval: 15s
timeout: 15s
Expand All @@ -66,7 +67,7 @@ services:
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://127.0.0.1:29091'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
Expand All @@ -78,7 +79,10 @@ services:
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
kafka-2:
hostname: 'kafka-2'
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
build:
context: .
Expand All @@ -90,12 +94,9 @@ services:
test:
[
'CMD',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-2:9091',
'--broker',
'2',
'--describe',
]
interval: 15s
timeout: 15s
Expand All @@ -111,7 +112,7 @@ services:
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://127.0.0.1:29092'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
Expand All @@ -123,7 +124,10 @@ services:
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
kafka-3:
hostname: 'kafka-3'
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
build:
context: .
Expand All @@ -135,12 +139,9 @@ services:
test:
[
'CMD',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-3:9091',
'--broker',
'3',
'--describe',
]
interval: 15s
timeout: 15s
Expand All @@ -156,7 +157,7 @@ services:
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://127.0.0.1:29093'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
Expand All @@ -168,7 +169,10 @@ services:
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
kafka-4:
hostname: 'kafka-4'
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
build:
context: .
Expand All @@ -180,12 +184,9 @@ services:
test:
[
'CMD',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-4:9091',
'--broker',
'4',
'--describe',
]
interval: 15s
timeout: 15s
Expand All @@ -201,7 +202,7 @@ services:
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://127.0.0.1:29094'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
Expand All @@ -213,7 +214,10 @@ services:
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
kafka-5:
hostname: 'kafka-5'
image: 'sarama/fv-kafka-${KAFKA_VERSION:-3.5.1}'
build:
context: .
Expand All @@ -225,12 +229,9 @@ services:
test:
[
'CMD',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-configs.sh',
'/opt/kafka-${KAFKA_VERSION:-3.5.1}/bin/kafka-broker-api-versions.sh',
'--bootstrap-server',
'kafka-5:9091',
'--broker',
'5',
'--describe',
]
interval: 15s
timeout: 15s
Expand All @@ -246,7 +247,7 @@ services:
KAFKA_VERSION: ${KAFKA_VERSION:-3.5.1}
KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181'
KAFKA_CFG_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095'
KAFKA_CFG_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://127.0.0.1:29095'
KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL'
KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT'
KAFKA_CFG_DEFAULT_REPLICATION_FACTOR: '2'
Expand All @@ -258,7 +259,10 @@ services:
KAFKA_CFG_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector'
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'false'
KAFKA_CFG_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_JVM_PERFORMANCE_OPTS: "-XX:+IgnoreUnrecognizedVMOptions"
toxiproxy:
hostname: 'toxiproxy'
image: 'ghcr.io/shopify/toxiproxy:2.4.0'
healthcheck:
test: ['CMD', '/toxiproxy-cli', 'l']
Expand Down

0 comments on commit 1dbc215

Please sign in to comment.