refactor: use make for api/keycloak db overrides

Makefile

@@ -94,13 +94,21 @@ KUBECTL = $(realpath ./local-dev/kubectl)
 JQ = $(realpath ./local-dev/jq)
 K3D = $(realpath ./local-dev/k3d)
 
+# which database vendor type to use, can be mariadb (default) or mysql
+# DATABASE_VENDOR = mariadb
+DATABASE_VENDOR = mysql
+DATABASE_DOCKERFILE = Dockerfile
+ifeq ($(DATABASE_VENDOR), mysql)
+DATABASE_DOCKERFILE = Dockerfile.mysql
+endif
+
 #######
 ####### Functions
 #######
 
 # Builds a docker image. Expects as arguments: name of the image, location of Dockerfile, path of
 # Docker Build Context
-docker_build = PLATFORMS=$(PLATFORM_ARCH) IMAGE_REPO=$(CI_BUILD_TAG) UPSTREAM_REPO=$(UPSTREAM_REPO) UPSTREAM_TAG=$(UPSTREAM_TAG) TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --load $(1)
+docker_build = PLATFORMS=$(PLATFORM_ARCH) IMAGE_REPO=$(CI_BUILD_TAG) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) UPSTREAM_REPO=$(UPSTREAM_REPO) UPSTREAM_TAG=$(UPSTREAM_TAG) TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --load $(1)
 
 docker_buildx_create = 	docker buildx create --name $(CI_BUILD_TAG) || echo  -e '$(CI_BUILD_TAG) builder already present\n'
 
@@ -186,13 +194,13 @@ $(build-services):
 
 # Dependencies of Service Images
 build/auth-server build/webhook-handler build/webhooks2tasks build/api: build/yarn-workspace-builder
-build/api-db: services/api-db/Dockerfile
+build/api-db: services/api-db/$(DATABASE_DOCKERFILE)
 build/api-redis: services/api-redis/Dockerfile
 build/actions-handler: services/actions-handler/Dockerfile
 build/backup-handler: services/backup-handler/Dockerfile
 build/broker: services/broker/Dockerfile
 build/api-sidecar-handler: services/api-sidecar-handler/Dockerfile
-build/keycloak-db: services/keycloak-db/Dockerfile
+build/keycloak-db: services/keycloak-db/$(DATABASE_DOCKERFILE)
 build/keycloak: services/keycloak/Dockerfile
 build/logs2notifications: services/logs2notifications/Dockerfile
 build/tests: tests/Dockerfile
@@ -305,14 +313,17 @@ webhooks-test-services-up: main-test-services-up $(foreach image,$(webhooks-test
 
 .PHONY: publish-testlagoon-images
 publish-testlagoon-images:
-	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/testlagoon TAG=$(BRANCH_NAME) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/testlagoon TAG=$(BRANCH_NAME) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	# PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/testlagoon TAG=$(BRANCH_NAME) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
 
 # tag and push all images
 
 .PHONY: publish-uselagoon-images
 publish-uselagoon-images:
-	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=$(LAGOON_VERSION) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
-	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/uselagoon TAG=$(LAGOON_VERSION) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	PLATFORMS=$(PUBLISH_PLATFORM_ARCH) DATABASE_VENDOR=$(DATABASE_VENDOR) DATABASE_DOCKERFILE=$(DATABASE_DOCKERFILE) IMAGE_REPO=docker.io/uselagoon TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	# PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=$(LAGOON_VERSION) LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
+	# PLATFORMS=$(PUBLISH_PLATFORM_ARCH) IMAGE_REPO=docker.io/uselagoon TAG=latest LAGOON_VERSION=$(LAGOON_VERSION) docker buildx bake -f docker-bake.hcl --builder $(CI_BUILD_TAG) --push
 
 .PHONY: clean
 clean:
@@ -352,19 +363,6 @@ else
 endif
 	$(MAKE) wait-for-keycloak
 
-# Start all Lagoon Services with mysql instead of mariadb
-up-mysql:
-ifeq ($(ARCH), darwin)
-	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) -f docker-compose.yaml -f docker-compose.local-dev-mysql.yaml -f docker-compose.local-dev.yaml --compatibility up -d
-else
-	# once this docker issue is fixed we may be able to do away with this
-	# linux-specific workaround: https://github.com/docker/cli/issues/2290
-	KEYCLOAK_URL=$$(docker network inspect -f '{{(index .IPAM.Config 0).Gateway}}' bridge):8088 \
-		IMAGE_REPO=$(CI_BUILD_TAG) \
-		docker compose -p $(CI_BUILD_TAG) -f docker-compose.yaml -f docker-compose.local-dev-mysql.yaml -f docker-compose.local-dev.yaml --compatibility up -d
-endif
-	$(MAKE) wait-for-keycloak
-
 down:
 	IMAGE_REPO=$(CI_BUILD_TAG) docker compose -p $(CI_BUILD_TAG) --compatibility down -v --remove-orphans
 
@@ -405,15 +403,6 @@ ui-logs-development: build-ui-logs-development
 api-logs-development: build-ui-logs-development
 	IMAGE_REPO=$(CI_BUILD_TAG) COMPOSE_STACK_NAME=$(CI_BUILD_TAG) ADDITIONAL_FLAGS="-f docker-compose.yaml -f docker-compose.local-dev.yaml" ADDITIONAL_SERVICES="" $(MAKE) compose-api-logs-development
 
-# MySQL specific targets for local dev
-.PHONY: ui-logs-development-mysql
-ui-logs-development-mysql: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) COMPOSE_STACK_NAME=$(CI_BUILD_TAG) ADDITIONAL_FLAGS="-f docker-compose.yaml -f docker-compose.local-dev-mysql.yaml -f docker-compose.local-dev.yaml" ADDITIONAL_SERVICES="ui" $(MAKE) compose-api-logs-development
-
-.PHONY: api-logs-development-mysql
-api-logs-development-mysql: build-ui-logs-development
-	IMAGE_REPO=$(CI_BUILD_TAG) COMPOSE_STACK_NAME=$(CI_BUILD_TAG) ADDITIONAL_FLAGS="-f docker-compose.yaml -f docker-compose.local-dev-mysql.yaml -f docker-compose.local-dev.yaml" ADDITIONAL_SERVICES="" $(MAKE) compose-api-logs-development
-
 # compose-api-logs-development can be consumed by other repositories to start a local api
 # supported make variable passthrough are
 # IMAGE_REPO - the docker repostory to use (uselagoon, testlagoon, other)

docker-bake.hcl

@@ -15,6 +15,14 @@ variable "UPSTREAM_REPO" {
   default = "uselagoon"
 }
 
+variable "DATABASE_DOCKERFILE" {
+  default = "Dockerfile"
+}
+
+variable "DATABASE_VENDOR" {
+  default = "mariadb"
+}
+
 variable "UPSTREAM_TAG" {
   default = "latest"
 }
@@ -39,6 +47,7 @@ target "default"{
     LAGOON_VERSION = "${LAGOON_VERSION}"
     UPSTREAM_REPO = "${UPSTREAM_REPO}"
     UPSTREAM_TAG = "${UPSTREAM_TAG}"
+    DATABASE_VENDOR = "${DATABASE_VENDOR}"
   }
 }
 
@@ -131,6 +140,7 @@ target "api" {
 target "api-db" {
   inherits = ["default"]
   context = "services/api-db"
+  dockerfile = "${DATABASE_DOCKERFILE}"
   labels = {
     "org.opencontainers.image.title": "lagoon-core/api-db - the MariaDB database service for Lagoon API"
   }
@@ -206,6 +216,7 @@ target "keycloak" {
 target "keycloak-db" {
   inherits = ["default"]
   context = "services/keycloak-db"
+  dockerfile = "${DATABASE_DOCKERFILE}"
   labels = {
     "org.opencontainers.image.title": "lagoon-core/keycloak-db - the MariaDB database service for Lagoon Keycloak"
   }

services/api-db/Dockerfile.mysql

@@ -0,0 +1,27 @@
+ARG UPSTREAM_REPO
+ARG UPSTREAM_TAG
+FROM ${UPSTREAM_REPO:-uselagoon}/mysql-8.0:${UPSTREAM_TAG:-latest}
+
+ARG LAGOON_VERSION
+ENV LAGOON_VERSION=$LAGOON_VERSION
+
+USER root
+
+COPY ./legacy-migration-scripts/* /legacy-migration-scripts/
+RUN chown -R mysql /legacy-migration-scripts/ \
+    && /bin/fix-permissions /legacy-migration-scripts/
+
+USER mysql
+
+ENV MYSQL_DATABASE=infrastructure \
+    MYSQL_USER=api \
+    MYSQL_PASSWORD=api
+    # are these needed??
+    # MYSQL_CHARSET=utf8 \
+    # MYSQL_COLLATION=utf8_general_ci
+
+# do we need to keep these any more? v2.10.0 was long ago...
+# COPY ./rerun_initdb.sh /rerun_initdb.sh
+# COPY ./legacy_rerun_initdb.sh /legacy_rerun_initdb.sh
+
+CMD ["mysqld", "--sql_mode", ""]

services/keycloak-db/Dockerfile.mysql

@@ -0,0 +1,19 @@
+ARG UPSTREAM_REPO
+ARG UPSTREAM_TAG
+FROM ${UPSTREAM_REPO:-uselagoon}/mysql-8.0:${UPSTREAM_TAG:-latest}
+
+ARG LAGOON_VERSION
+ENV LAGOON_VERSION=$LAGOON_VERSION
+
+ENV MYSQL_DATABASE=keycloak \
+    MYSQL_USER=keycloak \
+    MYSQL_PASSWORD=keycloak
+    # are these needed??
+    # MYSQL_CHARSET=utf8 \
+    # MYSQL_COLLATION=utf8_general_ci
+
+# not used in mysql8
+# COPY my_query-cache.cnf /etc/mysql/conf.d/my_query-cache.cnf
+# USER root
+# RUN fix-permissions /etc/mysql/conf.d/
+# USER mysql

services/keycloak/Dockerfile

@@ -22,6 +22,8 @@ COPY --from=ubi-micro-build /mnt/rootfs /
 
 ARG LAGOON_VERSION
 ENV LAGOON_VERSION=$LAGOON_VERSION
+ARG DATABASE_VENDOR
+ENV DATABASE_VENDOR=$DATABASE_VENDOR
 
 USER root
 
@@ -58,7 +60,7 @@ ENV TMPDIR=/tmp \
     KEYCLOAK_ADMIN_PASSWORD=admin \
     KEYCLOAK_REALM=lagoon \
     KEYCLOAK_REALM_ROLES=admin \
-    DB_VENDOR=mariadb \
+    DB_VENDOR=${DATABASE_VENDOR} \
     DB_ADDR=keycloak-db \
     DB_USER=keycloak \
     DB_PASSWORD=keycloak \
@@ -70,7 +72,7 @@ ENV TMPDIR=/tmp \
     KEYCLOAK_AUTH_SERVER_CLIENT_SECRET=f605b150-7636-4447-abd3-70988786b330 \
     KEYCLOAK_SERVICE_API_CLIENT_SECRET=d3724d52-34d1-4967-a802-4d178678564b \
     KEYCLOAK_LAGOON_UI_OIDC_CLIENT_SECRET=20580a56-6fbc-11ef-9a5b-3b4da292aa54 \
-    LAGOON_DB_VENDOR=mariadb \
+    LAGOON_DB_VENDOR=${DATABASE_VENDOR} \
     LAGOON_DB_DATABASE=infrastructure \
     LAGOON_DB_USER=api \
     LAGOON_DB_PASSWORD=api \