docker-compose.yml

services:
  spark-master:
    image: tabulario/spark-iceberg
    container_name: spark-master
    build: spark/
    
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
      - SPARK_MODE=master

    ports:
      - 8888:8888
      - 8080:8080
      - 10000:10000
      - 10001:10001
      - 4040:4040
      - 4041:4041
    volumes:
      - ./lakehouse/spark:/home/iceberg/warehouse
      - ./notebooks:/home/iceberg/notebooks/notebooks
      - ./spark_etls:/home/iceberg/etl_scripts

    networks:
      - iceberg_net

    depends_on:
      - rest
      - minio  
    

  spark-worker-1:
    image: tabulario/spark-iceberg
    container_name: spark-worker-1
    build: spark/

    volumes:
      - ./spark_workers.sh:/opt/spark/spark_workers.sh
      - ./lakehouse/spark:/home/iceberg/warehouse
      - ./notebooks:/home/iceberg/notebooks/notebooks
    
    networks:
      - iceberg_net
      
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077 
      - SPARK_WORKER_CORES=2
      - SPARK_WORKER_MEMORY=1G

    depends_on:
      - spark-master

    # command: ["/bin/bash", "/opt/spark/spark_workers.sh"]
    # command: ["start-worker.sh", SPARK_MASTER_URL, "--cores", "2", "--memory", "1G"]
    # command: ["/bin/bash", "-c", "start-worker.sh spark://spark-master:7077 --cores 2 --memory 1G"]
    # command: ["/bin/bash", "spark_workers.sh"]


  spark-worker-2:
    image: tabulario/spark-iceberg
    container_name: spark-worker-2
    build: spark/

    volumes:
      - ./spark_workers.sh:/opt/spark/spark_workers.sh
      - ./lakehouse/spark:/home/iceberg/warehouse
      - ./notebooks:/home/iceberg/notebooks/notebooks

    networks:
      - iceberg_net
      
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077 
      - SPARK_WORKER_CORES=2
      - SPARK_WORKER_MEMORY=1G

    depends_on:
      - spark-master

  spark-worker-3:
    image: tabulario/spark-iceberg
    container_name: spark-worker-3
    build: spark/

    volumes:
      - ./spark_workers.sh:/opt/spark/spark_workers.sh
      - ./lakehouse/spark:/home/iceberg/warehouse
      - ./notebooks:/home/iceberg/notebooks/notebooks

    networks:
      - iceberg_net
      
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark-master:7077 
      - SPARK_WORKER_CORES=2
      - SPARK_WORKER_MEMORY=1G

    depends_on:
      - spark-master


  # spark-worker-4:
  #   image: tabulario/spark-iceberg
  #   container_name: spark-worker-4
  #   build: spark/

  #   volumes:
  #     - ./spark_workers.sh:/opt/spark/spark_workers.sh
  #     - ./lakehouse/spark:/home/iceberg/warehouse
  #     - ./notebooks:/home/iceberg/notebooks/notebooks

  #   networks:
  #     - iceberg_net
      
  #   environment:
  #     - AWS_ACCESS_KEY_ID=admin
  #     - AWS_SECRET_ACCESS_KEY=password
  #     - AWS_REGION=us-east-1
  #     - SPARK_MODE=worker
  #     - SPARK_MASTER_URL=spark://spark-master:7077 
  #     - SPARK_WORKER_CORES=2
  #     - SPARK_WORKER_MEMORY=1G

  #   depends_on:
  #     - spark-master




  rest:
    image: apache/iceberg-rest-fixture
    container_name: iceberg-rest
    networks:
      - iceberg_net
    ports:
      - 8181:8181
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
      - CATALOG_WAREHOUSE=s3://warehouse/
      - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
      - CATALOG_S3_ENDPOINT=http://minio:9000


  minio:
    image: minio/minio
    container_name: minio
    environment:
      - MINIO_ROOT_USER=admin
      - MINIO_ROOT_PASSWORD=password
      - MINIO_DOMAIN=minio
    volumes:
      # - ./lakehouse/iceberg_minio:/data
      - iceberg-minio-data:/data
    networks:
      iceberg_net:
        aliases:
          - warehouse.minio
    ports:
      - 9001:9001
      - 9000:9000
    command: ["server", "/data", "--console-address", ":9001"]



  mc:
    depends_on:
      - minio
    image: minio/mc
    container_name: mc
    networks:
      - iceberg_net
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
    entrypoint: >
      /bin/sh -c "
      until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
      /usr/bin/mc mb --ignore-existing minio/warehouse;
      /usr/bin/mc policy set public minio/warehouse;
      tail -f /dev/null
      "


volumes:
  iceberg-minio-data:
    name: iceberg-minio-data


networks:
  iceberg_net:
    name: iceberg_net