docker-compose.yaml

version: '3'

services:

  raw_data_db:
    image: postgres:latest
    container_name: raw_data_db
    environment:
      POSTGRES_DB: raw_data
      POSTGRES_USER: my_username
      POSTGRES_PASSWORD: my_password
    ports:
      - "5433:5432"
    volumes:
      - /path/to/store/raw_data:/var/lib/postgresql/data

  cleaned_data_db:
    image: postgres:latest
    container_name: cleaned_data_db
    environment:
      POSTGRES_DB: cleaned_data
      POSTGRES_USER: my_username
      POSTGRES_PASSWORD: my_password
    ports:
      - "5435:5432"
    volumes:
      - /path/to/store/cleaned_data:/var/lib/postgresql/data

  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  broker:
    image: confluentinc/cp-enterprise-kafka:latest
    container_name: broker
    ports:
      - "9092:9092"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:9092
    depends_on:
      - zookeeper
    healthcheck:
      test: ["CMD", "nc", "-z", "localhost", "9092"]
      interval: 10s
      timeout: 10s
      retries: 5

  producer:
    build:
      context: .
      dockerfile: ./DockerfileProducer
    volumes:
      - ./producer.py:/app/producer.py
      - ./nyc_yellow_taxi_trip_records.csv:/app/nyc_yellow_taxi_trip_records.csv
      - ./data_cleaned_1000.csv:/app/data_cleaned_1000.csv
    command: python /app/producer.py
    depends_on:
      broker:
        condition: service_healthy
        restart: false

  consumer:
    build:
      context: .
      dockerfile: ./DockerfileConsumer
    volumes:
      - ./consumer.py:/app/consumer.py
    command: python /app/consumer.py
    depends_on:
      producer:
        condition: service_completed_successfully
        restart: false

  jupyter-pyspark-notebook:
    build:
      context: .
      dockerfile: ./DockerfileSpark
    hostname: jupyter-pyspark-notebook
    container_name: jupyter-pyspark-notebook
    ports:
      - "8888:8888"
    #restart: always
    volumes:
      - ./Spark.py:/app/Spark.py
    command: python /app/Spark.py
    depends_on:
      consumer:
        condition: service_completed_successfully
        restart: false

  insight_plot:
    build:
      context: .
      dockerfile: ./DockerfileInsight
    hostname: insight_plot
    container_name: insight_plot
    ports:
      - "5000:5000"
    #restart: always
    volumes:
      - ./insight_plot.py:/app/insight_plot.py
    command: python /app/insight_plot.py
    depends_on:
      jupyter-pyspark-notebook:
        condition: service_completed_successfully
        restart: false