docker-compose.yml

version: '3.7'
services:

  #-------------------------------#
  # logstash                      #
  #-------------------------------#

  logstash:
    image: docker.elastic.co/logstash/logstash:8.2.0
    container_name: logstash_green
    volumes: 
        - ./logstash/pipeline/:/usr/share/logstash/pipeline/
        - ./dataset/:/usr/share/logstash/logs/
    depends_on:
        - init-kafka
    environment:
      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
    networks: 
      - tmda

  #-------------------------------#
  # zookeeper                     #
  #-------------------------------#

  zookeeper:
    image: confluentinc/cp-zookeeper:7.0.1
    container_name: zookeeper_green
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEPER_HEAP_OPTS: -Xmx512m -Xmx512m
    networks:
      - tmda

  #-------------------------------#
  # kafka broker                  #
  #-------------------------------#

  broker:
    image: confluentinc/cp-kafka:7.0.1
    container_name: broker_green
    ports:
      - "9092:9092"
    depends_on:
      - zookeeper
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,PLAINTEXT_INTERNAL://broker:29092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_HEAP_OPTS: -Xmx512m -Xmx512m
    networks:
      - tmda

  #-------------------------------#
  # kafka web UI                  #
  #-------------------------------#

  kafka-ui:
    image: provectuslabs/kafka-ui
    container_name: kafka-ui_green
    depends_on:
      - zookeeper
      - broker
    ports:
      - "8080:8080"
    restart: always
    environment:
      KAFKA_CLUSTERS_0_NAME: local
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: broker:29092
      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181
      KAFKA_CLUSTERS_0_HEAP_OPTS: -Xmx512m -Xmx512m
    networks:
      - tmda

  #-------------------------------#
  # kafka topic                   #
  #-------------------------------#

  init-kafka:
    image: confluentinc/cp-kafka:7.0.1
    depends_on:
      - broker
      - zookeeper
      - kafka-ui
    entrypoint: [ '/bin/sh', '-c' ]
    command: |
      "
      # blocks until kafka is reachable
      kafka-topics --bootstrap-server broker:29092 --list

      echo -e 'Creating kafka topics'
      kafka-topics --bootstrap-server broker:29092 --create --if-not-exists --topic sensors-raw --replication-factor 1 --partitions 1
      kafka-topics --bootstrap-server broker:29092 --create --if-not-exists --topic sensors --replication-factor 1 --partitions 1

      echo -e 'Successfully created the following topics:'
      kafka-topics --bootstrap-server broker:29092 --list
      "
    environment:
      KAFKA_BROKER_ID: ignored
      KAFKA_ZOOKEEPER_CONNECT: ignored
      KAFKA_HEAP_OPTS: -Xmx512m -Xmx512m
    networks:
      - tmda
    
  #-------------------------------#
  # spark (training)              #
  #-------------------------------#

  #spark-training:
  #    build: 
  #        context: spark
  #    container_name: spark-training
  #    image: tmda_spark
  #    environment: 
  #      - "SPARK_ACTION=training"
  #    volumes: 
  #      - ./dataset/training:/opt/tap/spark/dataset
  #      - ./spark/code/training.py:/opt/tap/training.py
  #      - ./spark/model:/opt/tap/model
  #    networks:
  #      - tmda
  
  #-------------------------------#
  # spark (cleaning)              #
  #-------------------------------#

  spark-cleaning:
      build: 
          context: spark
      container_name: spark-cleaning
      image: tmda_spark
      depends_on:
        - init-kafka
      environment: 
        - "SPARK_ACTION=cleaning"
      volumes: 
        - ./spark/code/cleaning.py:/opt/tap/cleaning.py
      deploy:
        resources:
          limits:
            cpus: '1.0'
            memory: 1g
          reservations:
            cpus: '0.5'
            memory: 500m
      networks:
        - tmda

  #-------------------------------#
  # spark (streaming)              #
  #-------------------------------#

  spark-streaming:
      build: 
          context: spark
      container_name: spark-streaming
      image: tmda_spark
      depends_on:
        - init-kafka
        - elasticsearch
      restart: on-failure
      environment: 
        - "SPARK_ACTION=streaming"
      volumes: 
        - ./spark/code/streaming.py:/opt/tap/streaming.py
        - ./spark/model:/opt/tap/model
      deploy:
        resources:
          limits:
            cpus: '1.0'
            memory: 1g
          reservations:
            cpus: '0.5'
            memory: 500m
      networks:
        - tmda
  
  #-------------------------------#
  # zip extractor                 #
  #-------------------------------#

  zip-extractor:
      build: 
          context: dataset
      container_name: zip-extractor
      volumes: 
        - ./dataset/:/usr/src/app
      deploy:
        resources:
          limits:
            cpus: '0.2'
            memory: 500m
          reservations:
            cpus: '0.1'
            memory: 250m
      networks:
        - tmda
  
  #-------------------------------#
  # elasticsearch                 #
  #-------------------------------#
  
  elasticsearch:
    container_name: elasticsearch
    hostname: elasticsearch
    image: docker.elastic.co/elasticsearch/elasticsearch:8.2.0
    ports:
      - "9200:9200"
      - "9300:9300"
    restart: on-failure      
    environment:
      - xpack.security.enabled=false
      - discovery.type=single-node
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    networks: 
    - tmda

  #-------------------------------#
  # kibana                        #
  #-------------------------------#

  kibana:
    image: docker.elastic.co/kibana/kibana:8.2.0
    container_name: kibana
    hostname: kibana
    ports:
      - "5601:5601"
    depends_on: 
      - elasticsearch
    networks: 
      - tmda


networks:
  tmda:
    name: tmda
    driver: bridge