reorg

learn/airflow/Dockerfile

@@ -0,0 +1,13 @@
+FROM apache/airflow:${AIRFLOW_VERSION:-2.8.1}-python${PYTHON_VERSION:-3.11}
+
+# Switch to root to create directory with correct permissions
+USER root
+WORKDIR /
+RUN mkdir -p /requirements && chown -R airflow:root /requirements
+
+# Switch back to airflow user for pip install
+USER airflow
+COPY --chown=airflow:root requirements.txt /requirements/requirements.txt
+RUN pip install --no-cache-dir -r /requirements/requirements.txt
+
+WORKDIR /opt/airflow

learn/airflow/dags/example_dag.py

@@ -0,0 +1,29 @@
+from airflow import DAG
+from airflow.operators.python import PythonOperator
+from datetime import datetime, timedelta
+
+default_args = {
+    'owner': 'airflow',
+    'depends_on_past': False,
+    'start_date': datetime(2024, 1, 1),
+    'email_on_failure': False,
+    'email_on_retry': False,
+    'retries': 1,
+    'retry_delay': timedelta(minutes=5),
+}
+
+def print_hello():
+    return 'Hello from Airflow!'
+
+with DAG(
+    'example_dag',
+    default_args=default_args,
+    description='A simple example DAG',
+    schedule_interval=timedelta(days=1),
+    catchup=False
+) as dag:
+
+    task = PythonOperator(
+        task_id='print_hello',
+        python_callable=print_hello,
+    )

learn/airflow/docker-compose.yml

@@ -0,0 +1,129 @@
+version: '3.8'
+x-airflow-common: &airflow-common
+  build:
+    context: .
+    args:
+      AIRFLOW_VERSION: ${AIRFLOW_VERSION}
+      PYTHON_VERSION: ${PYTHON_VERSION}
+  environment: &airflow-common-env
+    AIRFLOW__CORE__EXECUTOR: CeleryExecutor
+    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}
+    AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}
+    AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
+    AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY}
+    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
+    AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
+    AIRFLOW__API__AUTH_BACKEND: 'airflow.api.auth.backend.basic_auth'
+  volumes:
+    - ./dags:/opt/airflow/dags
+    - ./logs:/opt/airflow/logs
+    - ./plugins:/opt/airflow/plugins
+    - ./config:/opt/airflow/config
+  user: "${AIRFLOW_UID}:${AIRFLOW_GID}"
+  depends_on: &airflow-common-depends-on
+    redis:
+      condition: service_healthy
+    postgres:
+      condition: service_healthy
+
+services:
+  postgres:
+    image: postgres:15-alpine
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+    volumes:
+      - postgres-db-volume:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}"]
+      interval: 10s
+      retries: 5
+      start_period: 5s
+    restart: always
+
+  redis:
+    image: redis:7-alpine
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 30s
+      retries: 50
+      start_period: 30s
+    restart: always
+
+  airflow-webserver:
+    <<: *airflow-common
+    command: webserver
+    ports:
+      - "8080:8080"
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    restart: always
+
+  airflow-scheduler:
+    <<: *airflow-common
+    command: scheduler
+    healthcheck:
+      test: ["CMD", "curl", "--fail", "http://localhost:8974/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    restart: always
+
+  airflow-worker:
+    <<: *airflow-common
+    command: celery worker
+    healthcheck:
+      test:
+        - "CMD-SHELL"
+        - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}" || exit 1'
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    restart: always
+
+  airflow-triggerer:
+    <<: *airflow-common
+    command: triggerer
+    healthcheck:
+      test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"']
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    restart: always
+
+  airflow-init:
+      image: apache/airflow:${AIRFLOW_VERSION:-2.8.1}-python${PYTHON_VERSION:-3.11}
+      entrypoint: /bin/bash
+      command:
+        - -c
+        - |
+          airflow db init && \
+          airflow users create \
+            --username $${AIRFLOW_USERNAME} \
+            --firstname Admin \
+            --lastname User \
+            --role Admin \
+            --email $${AIRFLOW_EMAIL} \
+            --password $${AIRFLOW_PASSWORD}
+      environment:
+        AIRFLOW_USERNAME: ${AIRFLOW_USERNAME}
+        AIRFLOW_PASSWORD: ${AIRFLOW_PASSWORD}
+        AIRFLOW_EMAIL: ${AIRFLOW_EMAIL}
+        AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres/${POSTGRES_DB}
+      depends_on:
+        postgres:
+          condition: service_healthy
+
+volumes:
+  postgres-db-volume:

learn/airflow/logs/scheduler/latest

@@ -0,0 +1 @@
+2024-12-24

learn/airflow/requirements.txt

@@ -0,0 +1,5 @@
+apache-airflow-providers-amazon==8.15.0
+apache-airflow-providers-postgres==5.10.0
+apache-airflow-providers-http==4.9.0
+apache-airflow-providers-ssh==3.10.0
+pandas==2.2.0

learn/airflow/scripts/init.sh

@@ -0,0 +1,8 @@
+docker compose down --volumes --remove-orphans
+# docker compose down -v
+rm -rf logs/* plugins/* dags/__pycache__
+docker images | grep airflow-docker
+docker rmi $(docker images -q 'airflow-docker*')
+mkdir -p ./dags ./logs ./plugins ./config
+# docker compose up -d
+docker compose up -d --build

learn/superset/Dockerfile

@@ -0,0 +1,23 @@
+FROM apache/superset:latest
+
+USER root
+
+# Install additional dependencies
+RUN apt-get update && \
+    apt-get install -y \
+    netcat-openbsd \
+    postgresql-client \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy the initialization script and entrypoint
+COPY superset-init.sh /app/superset-init.sh
+COPY docker-entrypoint.sh /app/docker-entrypoint.sh
+
+# Set permissions
+RUN chmod +x /app/superset-init.sh && \
+    chmod +x /app/docker-entrypoint.sh
+
+# Switch back to superset user
+USER superset
+
+ENTRYPOINT ["/app/docker-entrypoint.sh"]

learn/superset/docker-compose.yml

@@ -0,0 +1,78 @@
+version: '3.8'
+services:
+  superset:
+    build: 
+      context: .
+      dockerfile: Dockerfile
+    container_name: superset
+    ports:
+      - "8088:8088"
+    environment:
+      - SUPERSET_SECRET_KEY=your_secure_key_here
+      - SUPERSET_LOAD_EXAMPLES=yes
+      - SUPERSET_DB_USER=superset
+      - SUPERSET_DB_PASSWORD=superset
+      - SUPERSET_DB_HOST=db
+      - SUPERSET_DB_PORT=5432
+      - SUPERSET_DB=superset
+    volumes:
+      - superset_home:/app/superset_home
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8088/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    deploy:
+      resources:
+        limits:
+          cpus: '1'
+          memory: 4G
+        reservations:
+          cpus: '0.5'
+          memory: 2G
+    depends_on:
+      db:
+        condition: service_healthy
+    networks:
+      - superset_network
+
+  db:
+    image: postgres:14-alpine
+    container_name: superset_db
+    ports:
+      - "5432:5432"
+    environment:
+      - POSTGRES_USER=superset
+      - POSTGRES_PASSWORD=superset
+      - POSTGRES_DB=superset
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "superset"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    deploy:
+      resources:
+        limits:
+          cpus: '1'
+          memory: 2G
+        reservations:
+          cpus: '0.25'
+          memory: 1G
+    networks:
+      - superset_network
+
+volumes:
+  postgres_data:
+    name: superset_postgres_data
+    driver: local
+  superset_home:
+    name: superset_home
+    driver: local
+
+networks:
+  superset_network:
+    name: superset_network
+    driver: bridge

learn/superset/docker-entrypoint.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+# Run the initialization script if the database hasn't been initialized
+if [ ! -f /app/superset_home/.initialized ]; then
+    echo "Initializing Superset..."
+    /app/superset-init.sh
+    touch /app/superset_home/.initialized
+else
+    echo "Superset already initialized, starting normally..."
+fi
+
+# Start Superset using gunicorn
+gunicorn \
+    --bind "0.0.0.0:8088" \
+    --workers 10 \
+    --timeout 120 \
+    --limit-request-line 0 \
+    --limit-request-field_size 0 \
+    "superset.app:create_app()"

learn/superset/superset-init.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -e
+
+echo "Installing required packages..."
+pip install psycopg2-binary
+
+# Wait for database to be ready
+until nc -z db 5432; do
+    echo "Waiting for postgres database to be ready..."
+    sleep 2
+done
+
+echo "Database is ready, initializing Superset..."
+
+# Initialize the database
+superset db upgrade
+
+# Create admin user
+superset fab create-admin \
+    --username admin \
+    --firstname Superset \
+    --lastname Admin \
+    --email [email protected] \
+    --password admin
+
+# Load example data
+superset load_examples
+
+# Initialize roles
+superset init
+
+echo "Superset initialization completed successfully!"