docker-compose.yml

version: "3.9"

services:
  # This service runs the reverse proxy server for easier access to the other services.
  # It allows the use of more human-friendly domains instead of ports.
  traefik:
    profiles: [ "dagster", "mlflow" ]
    image: "traefik:v2.7"
    container_name: traefik
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.traefik.rule=Host(`traefik.127.0.0.1.nip.io`)"
      - "traefik.http.routers.traefik.entrypoints=web"
      - "traefik.http.services.traefik.loadbalancer.server.port=8080"
    command:
      - "--log.level=DEBUG"
      - "--api.insecure=true"
      - "--providers.docker=true"
      - "--providers.docker.exposedbydefault=false"
      - "--entrypoints.web.address=:80"
      - "--metrics.prometheus.buckets=0.1,0.3,1.2,5.0"
      - "--metrics.prometheus.addrouterslabels=true"
      - "--entryPoints.metrics.address=:8082"
      - "--metrics.prometheus.entryPoint=metrics"
    ports:
      - "80:80"
      - "8080:8080"
      - "8082:8082"
    volumes:
      - "/var/run/docker.sock:/var/run/docker.sock:ro"
    networks:
      - mlops_bridge

  minio:
    profiles: [ "mlflow" ]
    image: "minio/minio:RELEASE.2022-05-08T23-50-31Z"
    container_name: minio
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.minio.rule=Host(`minio.127.0.0.1.nip.io`)"
      - "traefik.http.routers.minio.entrypoints=web"
      - "traefik.http.services.minio.loadbalancer.server.port=9001"
    restart: unless-stopped
    ports:
      - "9000:9000"
      - "9001:9001"
    environment:
      - MINIO_ROOT_USER=$AWS_ACCESS_KEY_ID
      - MINIO_ROOT_PASSWORD=$AWS_SECRET_ACCESS_KEY
    command: server /data --console-address ":9001"
    volumes:
      - minio_data:/data
    networks:
      - mlops_bridge

  prometheus:
    profiles: [ "monitoring" ]
    image: "prom/prometheus:v2.35.0"
    container_name: prometheus
    restart: unless-stopped
    ports:
      - "9090:9090"
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.prometheus.rule=Host(`prometheus.127.0.0.1.nip.io`)"
      - "traefik.http.routers.prometheus.entrypoints=web"
    command:
      - "--config.file=/config/prometheus.yml"
    volumes:
      - ./services/prometheus:/config
      - prometheus_data:/prometheus
    networks:
      - mlops_bridge

  grafana:
    profiles: [ "monitoring" ]
    image: "grafana/grafana:8.5.1"
    container_name: grafana
    restart: unless-stopped
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.grafana.rule=Host(`grafana.127.0.0.1.nip.io`)"
      - "traefik.http.routers.grafana.entrypoints=web"
    ports:
      - "2000:3000"
    volumes:
      - grafana_data:/var/lib/grafana
    networks:
      - mlops_bridge

  #########################################################################################

  # This service runs the postgres DB used by dagster for run storage, schedule storage,
  # and event log storage.
  dagster_postgres:
    profiles: [ "dagster" ]
    image: postgres:14
    container_name: dagster-postgres
    restart: unless-stopped
    healthcheck:
      test: [ "CMD-SHELL", "pg_isready" ]
      interval: 10s
      timeout: 5s
      retries: 5
    environment:
      POSTGRES_USER: $DAGSTER_POSTGRES_USER
      POSTGRES_PASSWORD: $DAGSTER_POSTGRES_PASSWORD
      POSTGRES_DB: $DAGSTER_POSTGRES_DB
    networks:
      - mlops_bridge
    volumes:
      - dagster_db:/var/lib/postgresql/data

  # This service runs the gRPC server that loads your user code, in both dagit
  # and dagster-daemon. By setting DAGSTER_CURRENT_IMAGE to its own image, we tell the
  # run launcher to use this same image when launching runs in a new container as well.
  # Multiple containers like this can be deployed separately - each just needs to run on
  # its own port, and have its own entry in the workspace.yaml file that's loaded by dagit.
  dagster_repo:
    profiles: [ "dagster" ]
    build:
      context: .
      dockerfile: ./code/dagster-repo/Dockerfile
    container_name: dagster-repo
    image: $DAGSTER_CURRENT_IMAGE
    restart: always
    networks:
      - mlops_bridge
    environment:
      DAGSTER_POSTGRES_USER:
      DAGSTER_POSTGRES_PASSWORD:
      DAGSTER_POSTGRES_DB:
      DAGSTER_CURRENT_IMAGE:
      AWS_ACCESS_KEY_ID:
      AWS_SECRET_ACCESS_KEY:

        # This service runs dagit, which loads your user code from the user code container.
        # Since our instance uses the QueuedRunCoordinator, any runs submitted from dagit will be put on
        # a queue and later dequeued and launched by dagster-daemon.
  dagster_dagit:
    profiles: [ "dagster" ]
    build:
      context: .
      dockerfile: ./services/dagster/Dockerfile
    entrypoint:
      - dagit
      - -h
      - "0.0.0.0"
      - -p
      - "3000"
      - -w
      - workspace.yaml
    container_name: dagster-dagit
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.dagit.rule=Host(`dagit.127.0.0.1.nip.io`)"
      - "traefik.http.routers.dagit.entrypoints=web"
    expose:
      - "3000"
    ports:
      - "3000:3000"
    environment:
      - DAGSTER_POSTGRES_USER
      - DAGSTER_POSTGRES_PASSWORD
      - DAGSTER_POSTGRES_DB
    volumes:
      # Make docker client accessible so we can terminate containers from dagit
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - mlops_bridge
    depends_on:
      dagster_postgres:
        condition: service_healthy
      dagster_repo:
        condition: service_started

  # This service runs the dagster-daemon process, which is responsible for taking runs
  # off of the queue and launching them, as well as creating runs from schedules or sensors.
  dagster_daemon:
    profiles: [ "dagster" ]
    build:
      context: .
      dockerfile: ./services/dagster/Dockerfile
    entrypoint:
      - dagster-daemon
      - run
    container_name: dagster-daemon
    restart: on-failure
    environment:
      - DAGSTER_POSTGRES_USER
      - DAGSTER_POSTGRES_PASSWORD
      - DAGSTER_POSTGRES_DB
    volumes:
      # Make docker client accessible so we can launch containers using host docker
      - /var/run/docker.sock:/var/run/docker.sock
    networks:
      - mlops_bridge
    depends_on:
      - dagster_postgres
      - dagster_repo

  #########################################################################################

  mlflow_server:
    profiles: [ "mlflow" ]
    build:
      context: .
      dockerfile: ./services/mlflow/Dockerfile
      args:
        - MLFLOW_POSTGRES_USER
        - MLFLOW_POSTGRES_PASSWORD
        - MLFLOW_POSTGRES_DB
        - MLFLOW_S3_ENDPOINT_URL
        - MINIO_BUCKET_NAME
        - AWS_ACCESS_KEY_ID
        - AWS_SECRET_ACCESS_KEY

    container_name: mlflow-server
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.mlflow.rule=Host(`mlflow.127.0.0.1.nip.io`)"
      - "traefik.http.routers.mlflow.entrypoints=web"
    ports:
      - "4000:4000"
    networks:
      - mlops_bridge
    depends_on:
      - minio
      - mlflow_postgres

  mlflow_postgres:
    profiles: [ "mlflow" ]
    image: postgres:14
    container_name: mlflow-postgres
    restart: unless-stopped
    environment:
      POSTGRES_USER: $MLFLOW_POSTGRES_USER
      POSTGRES_PASSWORD: $MLFLOW_POSTGRES_PASSWORD
      POSTGRES_DB: $MLFLOW_POSTGRES_DB
    networks:
      - mlops_bridge
    volumes:
      - mlflow_db:/var/lib/postgresql/data

#########################################################################################
networks:
  mlops_bridge:
    name: "mlops_bridge"
    driver: bridge

volumes:
  mlflow_db:
  dagster_db:
  minio_data:
  prometheus_data:
  grafana_data: