From 479c75f6a7573faf650a4071e84be205f72b4c37 Mon Sep 17 00:00:00 2001 From: Ravi Singal <62086374+ravisingal@users.noreply.github.com> Date: Tue, 1 Oct 2024 09:05:54 +0530 Subject: [PATCH] chore: remove kafka-zookeeper image (#52) --- .github/workflows/docker-tests.yml | 44 --------- .../kafka-zookeeper/docker-compose.yml | 11 --- .github/workflows/pr-build.yml | 9 +- README.md | 14 +-- kafka-zookeeper/Dockerfile | 40 --------- kafka-zookeeper/build.gradle.kts | 9 -- kafka-zookeeper/docker-bin/docker-healthcheck | 31 ------- .../docker-bin/start-kafka-zookeeper | 26 ------ kafka-zookeeper/install.sh | 89 ------------------- settings.gradle.kts | 1 - 10 files changed, 2 insertions(+), 272 deletions(-) delete mode 100644 .github/workflows/docker-tests.yml delete mode 100644 .github/workflows/kafka-zookeeper/docker-compose.yml delete mode 100644 kafka-zookeeper/Dockerfile delete mode 100644 kafka-zookeeper/build.gradle.kts delete mode 100755 kafka-zookeeper/docker-bin/docker-healthcheck delete mode 100755 kafka-zookeeper/docker-bin/start-kafka-zookeeper delete mode 100755 kafka-zookeeper/install.sh diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml deleted file mode 100644 index 699eb27..0000000 --- a/.github/workflows/docker-tests.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Continuous Build Docker -on: - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request_target: - branches: - - main - # workflow_dispatch will let us manually trigger the workflow from GitHub actions dashboard. - # See https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/manually-running-a-workflow - workflow_dispatch: - -jobs: - build-on-linux: - runs-on: ubuntu-latest - name: Build and run Docker images - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - with: - ref: ${{github.event.pull_request.head.ref}} - repository: ${{github.event.pull_request.head.repo.full_name}} - fetch-depth: 0 - - - name: Build with Gradle - uses: hypertrace/github-actions/gradle@main - with: - args: dockerBuildImages - - - name: Verify kafka-zookeeper image - working-directory: ./.github/workflows/kafka-zookeeper - # Below tests a docker-compose.yml service named 'sut' with a valid HEALTHCHECK instruction: - run: | - docker-compose up -d - while status="$(docker inspect --format="{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}" "$(docker-compose ps -q sut)")"; do - case $status in - starting) sleep 1;; - healthy) exit 0;; - unhealthy) exit 1;; - esac - done - exit 1 diff --git a/.github/workflows/kafka-zookeeper/docker-compose.yml b/.github/workflows/kafka-zookeeper/docker-compose.yml deleted file mode 100644 index 71476bd..0000000 --- a/.github/workflows/kafka-zookeeper/docker-compose.yml +++ /dev/null @@ -1,11 +0,0 @@ -# uses 2.4 so we can use condition: service_healthy -version: "2.4" -services: - sut: - image: hypertrace/kafka-zookeeper:main - container_name: sut - networks: - default: - aliases: - - kafka - - zookeeper diff --git a/.github/workflows/pr-build.yml b/.github/workflows/pr-build.yml index 9f358c4..6163e6f 100644 --- a/.github/workflows/pr-build.yml +++ b/.github/workflows/pr-build.yml @@ -22,7 +22,7 @@ jobs: - name: Build with Gradle uses: hypertrace/github-actions/gradle@main with: - args: build dockerBuildImages + args: dockerBuildImages - name: Run Trivy vulnerability scanner for kafka image uses: hypertrace/github-actions/trivy-image-scan@main @@ -31,13 +31,6 @@ jobs: output-mode: github category: kafka - - name: Run Trivy vulnerability scanner for kafka-zookeeper image - uses: hypertrace/github-actions/trivy-image-scan@main - with: - image: hypertrace/kafka-zookeeper - output-mode: github - category: kafka-zookeeper - validate-helm-charts: runs-on: ubuntu-20.04 steps: diff --git a/README.md b/README.md index 41ce0ca..a7ac8b8 100644 --- a/README.md +++ b/README.md @@ -22,25 +22,13 @@ To build kafka image locally, run: ``` `Note:` -- docker-compose uses `kafka-zookeeper` image so you have to build it from that folder in case you are working on that one. - To read more about installing and configuring helm chart refer [BUILD.md](/BUILD.md). ## Testing -You can test the image you built after modification by running docker-compose or helm setup. - -### docker-compose -Change the tag for `kafka-zookeeper` from `:main` to `:test` in [docker-compose file](https://github.com/hypertrace/hypertrace/blob/main/docker/docker-compose.yml) like this. - -```yaml - kafka-zookeeper: - image: hypertrace/kafka-zookeeper:test - container_name: kafka-zookeeper - ... -``` +You can test the image you built after modification by running docker-compose or helm setup. and then run `docker-compose up` to test the setup. ## Docker Image Source: - [DockerHub > kafka](https://hub.docker.com/r/hypertrace/kafka) -- [DockerHub > kafka-zookeeper](https://hub.docker.com/r/hypertrace/kafka-zookeeper) diff --git a/kafka-zookeeper/Dockerfile b/kafka-zookeeper/Dockerfile deleted file mode 100644 index 858c0af..0000000 --- a/kafka-zookeeper/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -# This Dockerfile and supporting content derived originally from -# https://github.com/openzipkin/zipkin/blob/master/docker/collector/kafka/ - -# Using the same image as we use in CircleCI to avoid transfer costs -FROM cimg/openjdk:14.0.2 AS install - -# Use latest stable release here. Scala 2.13+ supports JRE 14 -ENV KAFKA_VERSION=3.7.1 SCALA_VERSION=2.13 - -USER root -WORKDIR /install -COPY install.sh /tmp/ -RUN /tmp/install.sh && rm /tmp/install.sh - -# Share the same base image to reduce layers used in testing -FROM hypertrace/java:11 -LABEL MAINTAINER Hypertrace "https://www.hypertrace.org/" - -# Add HEALTHCHECK and ENTRYPOINT scripts into the default search path -COPY docker-bin/* /usr/local/bin/ - -# All content including binaries and logs write under WORKDIR -WORKDIR /opt/kafka -ARG USER=kafka - -# Ensure the process doesn't run as root -RUN adduser -g '' -h ${PWD} -D ${USER} -USER ${USER} - -# Copy binaries and config we installed earlier -COPY --from=install --chown=${USER} /install . - -# ${KAFKA_ADVERTISED_HOST_NAME}:19092 is for connections from the Docker host -ENV KAFKA_ADVERTISED_HOST_NAME=localhost -EXPOSE 2181 9092 19092 - -# We use start period of 30s to avoid marking the container unhealthy on slow or contended CI hosts -HEALTHCHECK --interval=1s --start-period=30s --timeout=5s CMD ["docker-healthcheck"] - -ENTRYPOINT ["start-kafka-zookeeper"] diff --git a/kafka-zookeeper/build.gradle.kts b/kafka-zookeeper/build.gradle.kts deleted file mode 100644 index 57fc7e9..0000000 --- a/kafka-zookeeper/build.gradle.kts +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id("org.hypertrace.docker-publish-plugin") -} - -hypertraceDocker { - defaultImage { - imageName.set("kafka-zookeeper") - } -} diff --git a/kafka-zookeeper/docker-bin/docker-healthcheck b/kafka-zookeeper/docker-bin/docker-healthcheck deleted file mode 100755 index 871acaa..0000000 --- a/kafka-zookeeper/docker-bin/docker-healthcheck +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -# HEALTHCHECK for use in `docker ps` or `docker-compose ps`. -# It can also be used as a readiness probe in k8s - -# Fail on unset variables, but don't quit on rc!=0, so we can log what happened -set -u +e - -IP="$(hostname -i || echo '127.0.0.1')" - -# kafka-topics.sh invokes a Java command which can overrun CPU quota, so we -# check the services at TCP layer instead. -ZK_HEALTH=$(echo ruok | nc ${IP} 2181) -ZK_RC=$? -if [ "$ZK_RC" == "0" ]; then - if [ "$ZK_HEALTH" == "imok" ]; then - nc -z ${IP} 9092 > /dev/null 2>&1 - KAFKA_RC=$? - if [ "$KAFKA_RC" == "0" ]; then - exit 0 - fi - - echo Kafka port check failed with code: $KAFKA_RC - exit 1 - fi - - echo ZooKeeper not healthy: $ZK_HEALTH - exit 1 -fi - -echo ZooKeeper port check failed with code: $ZK_RC -exit 1 diff --git a/kafka-zookeeper/docker-bin/start-kafka-zookeeper b/kafka-zookeeper/docker-bin/start-kafka-zookeeper deleted file mode 100755 index 3b77697..0000000 --- a/kafka-zookeeper/docker-bin/start-kafka-zookeeper +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# ENTRYPOINT script that starts ZooKeeper and then Kafka -# -# This intentionally locates config using the current working directory, in order to consolidate -# Dockerfile instructions to WORKDIR -set -eu - -# Apply one-time deferred configuration that relies on ENV variables -# -# Internal docker producers and consumers use the normal hostname:9092, and outside docker the advertised host on port 19092 -ADVERTISED_LISTENERS="advertised.listeners=PLAINTEXT://${HOSTNAME}:9092,PLAINTEXT_HOST://${KAFKA_ADVERTISED_HOST_NAME}:19092" -KAFKA_CONFIG=./config/server.properties -grep -qF -- "$ADVERTISED_LISTENERS" $KAFKA_CONFIG || echo "$ADVERTISED_LISTENERS" >> $KAFKA_CONFIG - -echo Starting ZooKeeper -bin/kafka-run-class.sh \ - -Dlog4j.configuration=file:./config/log4j.properties \ - org.apache.zookeeper.server.quorum.QuorumPeerMain ./config/zookeeper.properties & - -# Wait for ZooKeeper to be ok -until echo ruok | nc 127.0.0.1 2181 > /dev/null; do sleep 1; done - -echo Starting Kafka -exec bin/kafka-run-class.sh -name kafkaServer \ - -Dlog4j.configuration=file:./config/log4j.properties \ - kafka.Kafka ./config/server.properties diff --git a/kafka-zookeeper/install.sh b/kafka-zookeeper/install.sh deleted file mode 100755 index e9cd990..0000000 --- a/kafka-zookeeper/install.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/sh -# install script used only in building the docker image, but not at runtime. -# This uses relative path so that you can change the home dir without editing this file. -# This also trims dependencies to only those used at runtime. -set -eux - -echo "*** Installing Kafka and dependencies" - -# Download scripts and config for Kafka and ZooKeeper, but not for Connect -wget -qO- https://archive.apache.org/dist/kafka/$KAFKA_VERSION/kafka_$SCALA_VERSION-$KAFKA_VERSION.tgz | tar xz \ - --wildcards --strip=1 --exclude=connect* */bin/zookeeper-* */bin/kafka-* */config - -# Remove bash as our images don't have it, and it isn't required -sed -i 's~#!/bin/bash~#!/bin/sh~g' bin/*sh -# Remove bash syntax relating to irrelevant CYGWIN (prevents console errors) -sed -i -e 's/$(uname -a) =~ "CYGWIN"/0/g' -e 's/.*\(\( CYGWIN \)\).*//g' bin/* -# Remove bash shopt commands (kafka-start-class uses nullglob, but it is non-critical to avoid it) -sed -i 's/.*shopt.*//g' bin/*sh - -# Hush logging for both Kafka and ZooKeeper -sed -i 's~INFO~WARN~g' config/log4j.properties - -# Make sure you use relative paths in references like this, so that installation -# is decoupled from runtime -mkdir -p data/kafka data/zookeeper logs - -# Set explicit, basic configuration -cat > config/zookeeper.properties <<-'EOF' -dataDir=./data/zookeeper -clientPort=2181 -maxClientCnxns=0 -admin.enableServer=false -# allow ruok command for testing ZK health -4lw.commands.whitelist=srvr,ruok -EOF - -cat > config/server.properties <<-'EOF' -broker.id=0 -zookeeper.connect=127.0.0.1:2181 -replica.socket.timeout.ms=1500 -# log.dirs is about Kafka's data not Log4J -log.dirs=./data/kafka -auto.create.topics.enable=true -offsets.topic.replication.factor=1 -listeners=PLAINTEXT://0.0.0.0:9092,PLAINTEXT_HOST://0.0.0.0:19092 -listener.security.protocol.map=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT -EOF - -# Dist includes large dependencies needed by streams and connect: retain only broker and ZK. -# We can do this because broker is independent from both kafka-streams and connect modules. -# See KAFKA-10380 -# -# TODO: MDEP-723 if addressed can remove the pom.xml here -cat > pom.xml <<-'EOF' - - 4.0.0 - - org.hypertrace.kafka - get-kafka - 0.1.0-SNAPSHOT - pom - - - - org.apache.kafka - kafka_${SCALA_VERSION} - ${KAFKA_VERSION} - - - org.slf4j - slf4j-log4j12 - 1.7.36 - - - org.apache.zookeeper - zookeeper - 3.8.4 - - - org.apache.zookeeper - zookeeper-jute - 3.8.4 - - - -EOF -mvn -q --batch-mode dependency:copy-dependencies -DoutputDirectory=libs && rm pom.xml - -echo "*** Image build complete" diff --git a/settings.gradle.kts b/settings.gradle.kts index f42619b..2eee358 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -10,5 +10,4 @@ plugins { id("org.hypertrace.version-settings") version "0.2.0" } -include(":kafka-zookeeper") include(":docker")