compose.yaml

version: '3'

services:
  zookeeper:
    build:
      context: ./kafka
      dockerfile: ./Dockerfile
    container_name: zookeeper
    ports:
      - "2181:2181"
    environment:
      KAFKA_ACTION: start-zk
    networks: 
      tapRobM:
        ipv4_address: 10.0.100.22

  kafkaserver:
    build:
      context: ./kafka
      dockerfile: ./Dockerfile
    container_name: kafkaserver
    hostname: kafkaServer
    depends_on:
      - zookeeper
    ports:
      - "9092:9092"
    environment:
      KAFKA_ACTION: start-kafka
    healthcheck:
      test: nc -z localhost 9092 || exit -1
      start_period: 15s
      interval: 5s
      timeout: 10s
      retries: 10
    networks: 
      tapRobM:
        ipv4_address: 10.0.100.23

  logstash:
    build: 
      context: ./logstash
      dockerfile: ./Dockerfile
    container_name: logstash
    depends_on: 
      kafkaserver:
        condition: service_healthy
    networks: 
      - tapRobM

  elasticsearch:
    build: 
      context: ./elasticsearch
      dockerfile: ./Dockerfile
    container_name: elasticsearch
    ports:
      - "9200:9200"
    environment:
      - node.name=elasticsearch
      - discovery.type=single-node
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    networks: 
      tapRobM:
        ipv4_address: 10.0.100.51

  spark:
    build: 
      context: ./spark
      dockerfile: ./Dockerfile
    container_name: spark
    command: ["TESTkafka_structuredstream.py", "org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.1"]
    environment: 
      SPARK_ACTION: "spark-submit-python"
    ports: 
      - "4040:4040"
    depends_on:
      - elasticsearch
      - kafkaserver
    volumes:
      - type: bind
        source: ./.dataset
        target: /opt/tap/.dataset
    networks: 
      - tapRobM

  twitterproducer:
    build: 
      context: ./python
      dockerfile: ./Dockerfile
    container_name: python
    environment:
      PYTHON_APP: "TwitterProducer.py"
    depends_on: 
      - kafkaserver
      - spark
    networks: 
      - tapRobM

  kibana:
    build: 
      context: ./kibana
      dockerfile: ./Dockerfile
    container_name: kibana
    ports:
      - "5601:5601"
    networks: 
      - tapRobM

networks:
    tapRobM:
        ipam:
            driver: default
            config:
                - subnet: 10.0.100.0/25