From d0ad5568fbe5f67116d489013eb751da8dfab0a6 Mon Sep 17 00:00:00 2001 From: Wurstmeister Date: Tue, 14 Feb 2017 18:23:44 +0100 Subject: [PATCH 1/2] merged master --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 11c01b2e..0d726241 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM anapsix/alpine-java -ARG kafka_version=0.10.1.1 +ARG kafka_version=0.10.0.1 ARG scala_version=2.11 MAINTAINER wurstmeister From e37bc385266c4d43a32ccb433dd9f013760130ab Mon Sep 17 00:00:00 2001 From: hmusavi Date: Fri, 21 Apr 2017 08:09:33 -0400 Subject: [PATCH 2/2] Introduced DOCKER_HOST_IP environment variable to avoid hard-coding the IP address the files. Also added two shell scripts to make it easier to start producers and consumers inside kafka container. --- Dockerfile | 8 +++++++- README.md | 2 +- broker-list.sh | 2 +- docker-compose-single-broker.yml | 2 +- docker-compose.yml | 2 +- start-console-consumer.sh | 9 +++++++++ start-console-producer.sh | 9 +++++++++ start-kafka-shell.sh | 2 +- 8 files changed, 30 insertions(+), 6 deletions(-) create mode 100644 start-console-consumer.sh create mode 100644 start-console-producer.sh diff --git a/Dockerfile b/Dockerfile index 4d8579d3..a854cfdd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,9 +18,15 @@ ENV PATH ${PATH}:${KAFKA_HOME}/bin ADD start-kafka.sh /usr/bin/start-kafka.sh ADD broker-list.sh /usr/bin/broker-list.sh ADD create-topics.sh /usr/bin/create-topics.sh +ADD start-console-consumer.sh /usr/bin/start-console-consumer.sh +ADD start-console-producer.sh /usr/bin/start-console-producer.sh + # The scripts need to have executable permission RUN chmod a+x /usr/bin/start-kafka.sh && \ chmod a+x /usr/bin/broker-list.sh && \ - chmod a+x /usr/bin/create-topics.sh + chmod a+x /usr/bin/create-topics.sh && \ + chmod a+x /usr/bin/start-console-consumer.sh && \ + chmod a+x /usr/bin/start-console-producer.sh + # Use "exec" form so that it runs as PID 1 (useful for graceful shutdown) CMD ["start-kafka.sh"] diff --git a/README.md b/README.md index bf9c2931..3b25e267 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ The image is available directly from [Docker Hub](https://hub.docker.com/r/wurst ##Pre-Requisites - install docker-compose [https://docs.docker.com/compose/install/](https://docs.docker.com/compose/install/) -- modify the ```KAFKA_ADVERTISED_HOST_NAME``` in ```docker-compose.yml``` to match your docker host IP (Note: Do not use localhost or 127.0.0.1 as the host ip if you want to run multiple brokers.) +- set an environment variable ```DOCKER_HOST_IP``` to match your docker host IP (Note: Do not use localhost or 127.0.0.1 as the host ip if you want to run multiple brokers.) - if you want to customise any Kafka parameters, simply add them as environment variables in ```docker-compose.yml```, e.g. in order to increase the ```message.max.bytes``` parameter set the environment to ```KAFKA_MESSAGE_MAX_BYTES: 2000000```. To turn off automatic topic creation set ```KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false'``` ##Usage diff --git a/broker-list.sh b/broker-list.sh index 7f046393..191d9350 100755 --- a/broker-list.sh +++ b/broker-list.sh @@ -1,5 +1,5 @@ #!/bin/bash CONTAINERS=$(docker ps | grep 9092 | awk '{print $1}') -BROKERS=$(for CONTAINER in $CONTAINERS; do docker port $CONTAINER 9092 | sed -e "s/0.0.0.0:/$HOST_IP:/g"; done) +BROKERS=$(for CONTAINER in $CONTAINERS; do docker port $CONTAINER 9092 | sed -e "s/0.0.0.0:/$DOCKER_HOST_IP:/g"; done) echo $BROKERS | sed -e 's/ /,/g' diff --git a/docker-compose-single-broker.yml b/docker-compose-single-broker.yml index 4d8e9f51..467e9e7c 100644 --- a/docker-compose-single-broker.yml +++ b/docker-compose-single-broker.yml @@ -9,7 +9,7 @@ services: ports: - "9092:9092" environment: - KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100 + KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP} KAFKA_CREATE_TOPICS: "test:1:1" KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 04b82c37..1ff824ce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ services: ports: - "9092" environment: - KAFKA_ADVERTISED_HOST_NAME: 192.168.99.100 + KAFKA_ADVERTISED_HOST_NAME: ${DOCKER_HOST_IP} KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 volumes: - /var/run/docker.sock:/var/run/docker.sock diff --git a/start-console-consumer.sh b/start-console-consumer.sh new file mode 100644 index 00000000..1d144cca --- /dev/null +++ b/start-console-consumer.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Use this script in a kafka shell instance to create a simple producer that reads from stdin and published to the given topic. + +if [[ -z "$1" ]]; then + echo 'Usage:' $0 '' + exit 1; +fi + +$KAFKA_HOME/bin/kafka-console-consumer.sh --topic=$1 --bootstrap-server=`broker-list.sh` diff --git a/start-console-producer.sh b/start-console-producer.sh new file mode 100644 index 00000000..86d1a216 --- /dev/null +++ b/start-console-producer.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Use this script in a kafka shell instance to create a simple producer that reads from stdin and published to the given topic. + +if [[ -z "$1" ]]; then + echo 'Usage:' $0 '' + exit 1; +fi + +$KAFKA_HOME/bin/kafka-console-producer.sh --topic=$1 --broker-list=`broker-list.sh` \ No newline at end of file diff --git a/start-kafka-shell.sh b/start-kafka-shell.sh index 62663e49..d6c9f595 100755 --- a/start-kafka-shell.sh +++ b/start-kafka-shell.sh @@ -1,2 +1,2 @@ #!/bin/bash -docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e HOST_IP=$1 -e ZK=$2 -i -t wurstmeister/kafka /bin/bash +docker run --rm -v /var/run/docker.sock:/var/run/docker.sock -e DOCKER_HOST_IP=$DOCKER_HOST_IP -e ZK=DOCKER_HOST_IP:2181 -i -t wurstmeister/kafka /bin/bash