docker-compose.yaml

x-airflow-common: &airflow-common
  build: './dockerfiles/airflow'
  image: ${AIRFLOW_IMAGE_NAME:-extending_airflow:latest}
  profiles:
    - airflow
    - all
  environment: &airflow-common-env
    AIRFLOW__CORE__EXECUTOR: LocalExecutor
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER:-airflow}:${PG_PASSWORD:-airflow}@postgres/${PG_DATABASE:-airflow}
    AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_USER:-airflow}:${PG_PASSWORD:-airflow}@postgres/${PG_DATABASE:-airflow}
    AIRFLOW__CORE__FERNET_KEY: ''
    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
    AIRFLOW__CORE__LOAD_EXAMPLES: 'true'  # Change to 'false' to remove the tutorials and examples
    AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.session'
    AIRFLOW__CORE__TEST_CONNECTION: 'Enabled'
    AIRFLOW__WEBSERVER__EXPOSE_CONFIG: 'True'
    AIRFLOW__SECRETS__BACKEND: airflow.secrets.local_filesystem.LocalFilesystemBackend
    AIRFLOW__SECRETS__BACKEND_KWARGS: '{"variables_file_path": "/opt/secrets/variables.yaml", "connections_file_path": "/opt/secrets/connections.yaml"}'
    AWS_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY:-minio}
    AWS_SECRET_ACCESS_KEY: ${MINIO_SECRET_ACCESS_KEY:-minio123}
    AWS_ENDPOINT_URL_S3: http://s3:9000
    MLFLOW_S3_ENDPOINT_URL: http://s3:9000
  volumes:
    - ${AIRFLOW_PROJ_DIR:-./airflow}/dags:/opt/airflow/dags
    - ${AIRFLOW_PROJ_DIR:-./airflow}/logs:/opt/airflow/logs
    - ${AIRFLOW_PROJ_DIR:-./airflow}/config:/opt/airflow/config
    - ${AIRFLOW_PROJ_DIR:-./airflow}/plugins:/opt/airflow/plugins
    - ${AIRFLOW_PROJ_DIR:-./airflow}/secrets:/opt/secrets
  networks:
    - frontend
    - backend
  user: "${AIRFLOW_UID:-50000}:${AIRFLOW_GID:-0}"
  depends_on:  &airflow-common-depends-on
    postgres:
      condition: service_healthy

services:
  postgres:
    restart: always
    build: './dockerfiles/postgres'
    image: postgres_system
    container_name: postgres
    profiles:
      - airflow
      - mlflow
      - all
    ports:
      - "${PG_PORT:-5432}:5432"
    networks:
      - backend
    environment:
      - POSTGRES_USER=${PG_USER:-airflow}
      - POSTGRES_PASSWORD=${PG_PASSWORD:-airflow}
      - POSTGRES_DATABASE=${PG_DATABASE:-airflow}
    volumes:
      - db_data:/var/lib/postgresql/data
    healthcheck:
      test: ["CMD", "pg_isready", "-p", "5432", "-U", "${PG_USER:-airflow}"]
      interval: 60s
      timeout: 20s
      retries: 3

  s3:
    restart: always
    image: minio/minio:latest
    container_name: minio
    profiles:
      - mlflow
      - airflow
      - all
    ports:
      - "${MINIO_PORT:-9000}:9000"
      - "${MINIO_PORT_UI:-9001}:9001"
    networks:
      - frontend
      - backend
    environment:
      - MINIO_ACCESS_KEY=${MINIO_ACCESS_KEY:-minio}
      - MINIO_SECRET_KEY=${MINIO_SECRET_ACCESS_KEY:-minio123}
    volumes:
      - minio_data:/data
    command: server /data --console-address :9001
    healthcheck:
      test: ["CMD", "mc", "ready", "local"]
      interval: 60s
      timeout: 20s
      retries: 3

  create_s3_buckets:
    image: minio/mc:latest
    container_name: minio_create_bucket
    profiles:
      - mlflow
      - airflow
      - all
    depends_on:
      - s3
    networks:
      - backend
    entrypoint: >
      /bin/sh -c '
      sleep 5;
      /usr/bin/mc config host add s3 http://s3:9000 ${MINIO_ACCESS_KEY:-minio} ${MINIO_SECRET_ACCESS_KEY:-minio123} --api S3v4;
      [[ ! -z "`/usr/bin/mc ls s3 | grep challenge`" ]] || /usr/bin/mc mb s3/${MLFLOW_BUCKET_NAME:-mlflow};
      /usr/bin/mc policy download s3/${MLFLOW_BUCKET_NAME:-mlflow};
      [[ ! -z "`/usr/bin/mc ls s3 | grep challenge`" ]] || /usr/bin/mc mb s3/${DATA_REPO_BUCKET_NAME:-data};
      /usr/bin/mc policy download s3/${DATA_REPO_BUCKET_NAME:-data};
      exit 0;
      '

  mlflow:
    restart: always
    build: "./dockerfiles/mlflow"
    image: mlflow
    container_name: mlflow
    profiles:
      - mlflow
      - all
    depends_on:
      postgres:
        condition: service_healthy
    ports:
      - "${MLFLOW_PORT:-5000}:5000"
    networks:
      - frontend
      - backend
    environment:
      - AWS_ACCESS_KEY_ID=${MINIO_ACCESS_KEY:-minio}
      - AWS_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minio123}
      - MLFLOW_S3_ENDPOINT_URL=http://s3:9000
    command: >
      mlflow server 
      --backend-store-uri postgresql://${PG_USER:-airflow}:${PG_PASSWORD:-airflow}@postgres:5432/mlflow_db
      --host 0.0.0.0
      --default-artifact-root s3://${MLFLOW_BUCKET_NAME:-mlflow}/
    healthcheck:
      test: wget --no-verbose --tries=1 --spider http://mlflow:5000 || exit 1
      interval: 60s
      timeout: 10s
      retries: 3

  fastapi:
    restart: always
    build: "./dockerfiles/fastapi"
    image: backend_fastapi
    container_name: fastapi
    profiles:
      - all
    depends_on:
      mlflow:
        condition: service_healthy
      airflow-webserver:
        condition: service_healthy
    ports:
      - "${FASTAPI_PORT:-8800}:8800"
    networks:
      - frontend
      - backend
    environment:
      - AWS_ACCESS_KEY_ID=${MINIO_ACCESS_KEY:-minio}
      - AWS_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minio123}
      - MLFLOW_S3_ENDPOINT_URL=http://s3:9000
      - AWS_ENDPOINT_URL_S3=http://s3:9000
    command: >
      uvicorn app:app 
      --host 0.0.0.0
      --port 8800
    healthcheck:
      test: curl --include --request GET http://fastapi:8800/ || exit 1
      interval: 60s
      timeout: 10s
      start_period: 120s
      retries: 3

  airflow-webserver:
    <<: *airflow-common
    container_name: airflow_webserver
    command: webserver
    ports:
      - "${AIRFLOW_PORT:-8080}:8080"
    healthcheck:
      test: [ "CMD", "curl", "--fail", "http://localhost:8080/health" ]
      interval: 60s
      timeout: 10s
      retries: 5
    restart: always
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-cli:
    <<: *airflow-common
    container_name: airflow_cli
    profiles:
      - debug
    environment:
      <<: *airflow-common-env
      CONNECTION_CHECK_MAX_COUNT: "0"
    # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252
    command:
      - bash
      - -c
      - airflow

  airflow-scheduler:
    <<: *airflow-common
    container_name: airflow_scheduler
    command: scheduler
    healthcheck:
      test: [ "CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"' ]
      interval: 60s
      timeout: 10s
      retries: 5
    restart: always
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  # it should not be necessary for a basic implementation
  #airflow-triggerer:
  #  <<: *airflow-common
  #  container_name: airflow_triggerer
  #  command: triggerer
  #  healthcheck:
  #    test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"']
  #    interval: 60s
  #    timeout: 10s
  #    retries: 5
  #    start_period: 30s
  #  restart: always
  #  depends_on:
  #    <<: *airflow-common-depends-on
  #    airflow-init:
  #      condition: service_completed_successfully

  airflow-init:
    <<: *airflow-common
    container_name: airflow_init
    entrypoint: /bin/bash
    # yamllint disable rule:line-length
    command:
      - -c
      - |
        if [[ -z "${AIRFLOW_UID:-50000}" ]]; then
          echo
          echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m"
          echo "If you are on Linux, you SHOULD follow the instructions below to set "
          echo "AIRFLOW_UID environment variable, otherwise files will be owned by root."
          echo "For other operating systems you can get rid of the warning with manually created .env file:"
          echo "    See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user"
          echo
        fi
        one_meg=1048576
        mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg))
        cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat)
        disk_available=$$(df / | tail -1 | awk '{print $$4}')
        warning_resources="false"
        if (( mem_available < 4000 )) ; then
          echo
          echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m"
          echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))"
          echo
          warning_resources="true"
        fi
        if (( cpus_available < 2 )); then
          echo
          echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m"
          echo "At least 2 CPUs recommended. You have $${cpus_available}"
          echo
          warning_resources="true"
        fi
        if (( disk_available < one_meg * 10 )); then
          echo
          echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m"
          echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))"
          echo
          warning_resources="true"
        fi
        if [[ $${warning_resources} == "true" ]]; then
          echo
          echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m"
          echo "Please follow the instructions to increase amount of resources available:"
          echo "   https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#before-you-begin"
          echo
        fi
        mkdir -p /sources/logs /sources/dags /sources/plugins
        chown -R "${AIRFLOW_UID:-50000}:${AIRFLOW_GID:-0}" /sources/{logs,dags,plugins}
        exec /entrypoint airflow version
    # yamllint enable rule:line-length
    environment:
      <<: *airflow-common-env
      _AIRFLOW_DB_MIGRATE: 'true'
      _AIRFLOW_WWW_USER_CREATE: 'true'
      _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
      _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
      _PIP_ADDITIONAL_REQUIREMENTS: ''
    user: "0:0"
    volumes:
      - ${AIRFLOW_PROJ_DIR:-./airflow}:/sources

networks:
  frontend:
    driver: bridge
  backend:
    driver: bridge

volumes:
  db_data:
  minio_data: