docker-compose.yml

version: "3"
services:
  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    container_name: namenode
    restart: always
    ports:
      - 9870:9870
      - 9000:9000   
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ./hadoop.env

  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    container_name: datanode1
    restart: always
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
    ports:
      - 9864:9864
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    env_file:
      - ./hadoop.env

  datanode2:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    container_name: datanode2
    restart: always
    volumes:
      - hadoop_datanode2:/hadoop/dfs/data
    ports:
      - 9863:9864
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    env_file:
      - ./hadoop.env

  resourcemanager:
    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
    container_name: resourcemanager
    restart: always
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 datanode2:9864"
    ports:
      - 8088:8088
    expose:
      - "8032"
      - "8031"
      - "8030"
    env_file:
      - ./hadoop.env

  nodemanager1:
    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
    container_name: nodemanager
    restart: always
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 datanode2:9864 resourcemanager:8088"
    ports:
      - 8042:8042
    env_file:
      - ./hadoop.env
  
  historyserver:
    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8
    container_name: historyserver
    restart: always
    environment:
      SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 datanode2:9864 resourcemanager:8088"
    volumes:
      - hadoop_historyserver:/hadoop/yarn/timeline
    ports:
      - 8188:8188
    env_file:
      - ./hadoop.env

  spark-master:
    image: bde2020/spark-master:3.1.1-hadoop3.2
    container_name: spark-master
    ports:
      - 8080:8080
      - 7077:7077
    env_file:
      - ./hadoop.env

  spark-worker1:
    image: bde2020/spark-worker:3.1.1-hadoop3.2
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
    env_file:
      - ./hadoop.env
  
  spark-worker2:
    image: bde2020/spark-worker:3.1.1-hadoop3.2
    environment:
      - "SPARK_MASTER=spark://spark-master:7077"
    env_file:
      - ./hadoop.env
 
  zeppelin:
    image: bde2020/zeppelin:0.8.0-hadoop-2.8.0-spark-2.3.1
    ports:
      - 80:8080
    volumes:
      - ./notebook:/opt/zeppelin/notebook
    environment:
      SPARK_MASTER: "spark://spark-master:7077"
      MASTER: "spark:spark-master:7077"
    env_file:
      - ./hadoop.env
volumes:
  hadoop_namenode:
  hadoop_datanode:
  hadoop_datanode2:
  hadoop_historyserver: