Keycloak Breeze integration

Work is starting on multi-team Airflow, and this project has many
dependencies on a auth manager that can support authn and authz and also
support the changes to the auth manager api (upcoming). An option for
this is Keycloak.

This PR adds a Breeze integartion for Keycloak which creates a container
running keycloak, which uses the existing Postgres container as the DB
and disables the requirements for ssl/https/certificates to use the
Keycloak admin console (since this is only for development purposes not
production).

.pre-commit-config.yaml

@@ -651,6 +651,8 @@ repos:
           ^docs/exts/removemarktransform.py$|
           ^newsfragments/41761.significant.rst$|
           ^scripts/ci/pre_commit/vendor_k8s_json_schema.py$|
+          ^scripts/ci/docker-compose/integration-keycloak.yml$|
+          ^scripts/ci/docker-compose/keycloak/keycloak-entrypoint.sh$|
           ^tests/|
           ^providers/tests/|
           ^.pre-commit-config\.yaml$|

contributing-docs/testing/integration_tests.rst

@@ -62,6 +62,8 @@ The following integrations are available:
 +--------------+----------------------------------------------------+
 | kerberos     | Integration that provides Kerberos authentication. |
 +--------------+----------------------------------------------------+
+| keycloak     | Integration for manual testing multi team Airflow. |
++--------------+----------------------------------------------------+
 | mongo        | Integration required for MongoDB hooks.            |
 +--------------+----------------------------------------------------+
 | mssql        | Integration required for mssql hooks.              |

dev/breeze/doc/images/output_shell.txt

@@ -1 +1 @@
-94eabd0345df65038958fc4842a0ff4b
+5a625bc4f2da217fdb3eb9a77127abd7

dev/breeze/doc/images/output_start-airflow.txt

@@ -1 +1 @@
-f32bc14609bdfe554743cf2752bf7701
+211d32d88b39716844eaf897d68a7cae

dev/breeze/doc/images/output_testing_integration-tests.txt

@@ -1 +1 @@
-5436131180cd928292c8234d15e0496f
+633eca64e9397259c7400814ce4c2877

dev/breeze/doc/images/output_testing_tests.txt

@@ -1 +1 @@
-2be847f8a1f4e2b80c6da1e4433203aa
+57156c9e849bdef9070035aad71181a3

dev/breeze/src/airflow_breeze/global_constants.py

@@ -68,7 +68,7 @@
     "trino",
     "ydb",
 ]
-OTHER_INTEGRATIONS = ["statsd", "otel", "openlineage"]
+OTHER_INTEGRATIONS = ["statsd", "otel", "openlineage", "keycloak"]
 ALLOWED_DEBIAN_VERSIONS = ["bookworm"]
 ALL_INTEGRATIONS = sorted(
     [

scripts/ci/docker-compose/integration-keycloak.yml

@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+services:
+  keycloak:
+    image: quay.io/keycloak/keycloak:23.0.6
+    labels:
+      breeze.description: "Integration for manual testing multi team Airflow."
+    entrypoint: /opt/keycloak/keycloak-entrypoint.sh
+    environment:
+      KC_HOSTNAME: localhost
+      KC_HOSTNAME_PORT: 38080
+      KC_HOSTNAME_STRICT_BACKCHANNEL: false
+      KC_HTTP_ENABLED: true
+      KC_HOSTNAME_STRICT: true
+
+      KEYCLOAK_ADMIN: admin
+      KEYCLOAK_ADMIN_PASSWORD: admin
+
+      KC_DB: postgres
+      KC_DB_URL: jdbc:postgresql://postgres/keycloak
+      KC_DB_USERNAME: keycloak
+      KC_DB_PASSWORD: keycloak
+    ports:
+      - 38080:38080
+    restart: always
+    depends_on:
+      - postgres
+    volumes:
+      - ./keycloak/keycloak-entrypoint.sh:/opt/keycloak/keycloak-entrypoint.sh
+
+  postgres:
+    volumes:
+      - ./keycloak/init-keycloak-db.sh:/docker-entrypoint-initdb.d/init-keycloak-db.sh
+    environment:
+      KC_POSTGRES_DB: keycloak
+      KC_POSTGRES_USER: keycloak
+      KC_POSTGRES_PASSWORD: keycloak
+    healthcheck:
+      test: ["CMD", "psql", "-h", "localhost", "-U", "keycloak"]
+      interval: 10s
+      timeout: 10s
+      retries: 5
+
+  airflow:
+    depends_on:
+      - keycloak

scripts/ci/docker-compose/keycloak/init-keycloak-db.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eu
+
+psql -v ON_ERROR_STOP=1 --username "${POSTGRES_USER}" > /dev/null <<-EOSQL
+  CREATE USER ${KC_POSTGRES_USER};
+  ALTER USER ${KC_POSTGRES_USER} WITH PASSWORD '${KC_POSTGRES_PASSWORD}';
+  CREATE DATABASE ${KC_POSTGRES_DB};
+  GRANT ALL PRIVILEGES ON DATABASE ${KC_POSTGRES_DB} TO ${KC_POSTGRES_USER};
+EOSQL

scripts/ci/docker-compose/keycloak/keycloak-entrypoint.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# We exit in case cd fails
+cd /opt/keycloak/bin/ || exit
+
+# Start Keycloak in the background
+./kc.sh start-dev --http-port=38080 &
+
+# Wait for Keycloak to be ready
+echo "Waiting for Keycloak to start..."
+while ! (echo > /dev/tcp/localhost/38080) 2>/dev/null; do
+  echo "keycloak still not started"
+  sleep 5
+done
+sleep 3
+echo "Keycloak is running (probably...)"
+
+# The below commands are used to disable the ssl requirement to use the admin panel of keycloak
+echo "Configuring admin console access without ssl/https"
+# Get credentials to make the below update to the realm settings
+./kcadm.sh config credentials --server http://localhost:38080 --realm master --user admin --password admin
+./kcadm.sh update realms/master -s sslRequired=NONE --server http://localhost:38080
+echo "Configuring complete!"
+
+# Keep the container running
+wait