HMS-2678 feat: script to deploy in ephe

Add the set of makefiles to provide the rules to custom deployment in
the ephemeral environment. Align this repo with the backend repository.

Signed-off-by: Alejandro Visiedo <[email protected]>

scripts/mk/container.mk

@@ -0,0 +1,66 @@
+##
+# General rules for interacting with container
+# manager (podman or docker).
+##
+
+ifneq (,$(shell command podman -v 2>/dev/null))
+CONTAINER_ENGINE ?= podman
+else
+ifneq (,$(shell command docker -v 2>/dev/null))
+CONTAINER_ENGINE ?= docker
+else
+CONTAINER_ENGINE ?= false
+endif
+endif
+export CONTAINER_ENGINE
+
+CONTAINER_HEALTH_PATH ?= .State.Health.Status
+
+ifneq (,$shell(selinuxenabled 2>/dev/null))
+CONTAINER_VOL_SUFFIX ?= :Z
+else
+CONTAINER_VOL_SUFFIX ?=
+endif
+
+CONTAINER_REGISTRY_USER ?= $(USER)
+CONTAINER_REGISTRY ?= quay.io
+CONTAINER_CONTEXT_DIR ?= .
+CONTAINERFILE ?= build/package/Dockerfile
+CONTAINER_IMAGE_BASE ?= $(CONTAINER_REGISTRY)/$(CONTAINER_REGISTRY_USER)/$(APP)
+CONTAINER_IMAGE_TAG ?= $(shell git rev-parse --short HEAD)
+CONTAINER_IMAGE ?= $(CONTAINER_IMAGE_BASE):$(CONTAINER_IMAGE_TAG)
+# CONTAINER_BUILD_OPTS
+# CONTAINER_ENGINE_OPTS
+# CONTAINER_RUN_ARGS
+
+# if go is available, mount user's Go module and build cache to speed up dev builds.
+ifneq (,$(shell command go 2>&1 >/dev/null))
+USE_GO_CACHE = true
+CONTAINER_BUILD_OPTS += -v "$(shell go env GOCACHE):/opt/app-root/src/.cache/go-build$(CONTAINER_VOL_SUFFIX)"
+CONTAINER_BUILD_OPTS += -v "$(shell go env GOMODCACHE):/opt/app-root/src/go/pkg/mod$(CONTAINER_VOL_SUFFIX)"
+else
+USE_GO_CACHE = false
+endif
+
+.PHONY: registry-login
+registry-login:
+	$(CONTAINER_ENGINE) login -u "$(CONTAINER_REGISTRY_USER)" -p "$(CONTAINER_REGISTRY_TOKEN)" $(CONTAINER_REGISTRY)
+
+.PHONY: container-build
+container-build: QUAY_EXPIRATION ?= 1d
+container-build:  ## Build image CONTAINER_IMAGE from CONTAINERFILE using the CONTAINER_CONTEXT_DIR
+	$(CONTAINER_ENGINE) build \
+	  --label "quay.expires-after=$(QUAY_EXPIRATION)" \
+	  $(CONTAINER_BUILD_OPTS) \
+	  -t "$(CONTAINER_IMAGE)" \
+	  $(CONTAINER_CONTEXT_DIR) \
+	  -f "$(CONTAINERFILE)"
+.PHONY: container-push
+container-push:  ## Push image to remote registry
+	$(CONTAINER_ENGINE) push "$(CONTAINER_IMAGE)"
+
+# TODO Indicate in the options the IP assigned to the postgres container
+# .PHONY: container-run
+# container-run: CONTAINER_ENGINE_OPTS += --env-file .env
+# container-run:  ## Run with CONTAINER_ENGINE_OPTS the CONTAINER_IMAGE using CONTAINER_RUN_ARGS as arguments (eg. make container-run CONTAINER_ENGINE_OPTS="-p 9000:9000")
+# 	$(CONTAINER_ENGINE) run $(CONTAINER_ENGINE_OPTS) $(CONTAINER_IMAGE) $(CONTAINER_RUN_ARGS)

scripts/mk/ephemeral.mk

@@ -0,0 +1,186 @@
+
+# .PHONY: ephemeral-setup
+# ephemeral-setup: ## Configure bonfire to run locally
+# 	bonfire config write-default > $(PROJECT_DIR)/config/bonfire-config.yaml
+
+ifeq (,$(APP))
+$(error APP is empty; did you miss to set APP=my-app at your scripts/mk/variables.mk)
+endif
+
+APP_COMPONENT ?= frontend
+
+NAMESPACE ?= $(shell oc project -q 2>/dev/null)
+# POOL could be:
+#   default
+#   minimal
+#   managed-kafka
+#   real-managed-kafka
+POOL ?= default
+export NAMESPACE
+export POOL
+
+
+# CLIENTS_RBAC_BASE_URL ?= http://localhost:8801/api/rbac/v1  # For local workstation
+# CLIENTS_RBAC_BASE_URL ?= http://rbac-service:8080/api/rbac/v1
+# export CLIENTS_RBAC_BASE_URL
+
+# Set the default duration for the namespace reservation and extension
+EPHEMERAL_DURATION ?= 4h
+
+ifneq (default,$(POOL))
+EPHEMERAL_OPTS += --no-single-replicas
+else
+EPHEMERAL_OPTS += --single-replicas
+endif
+
+ifeq (False,$(CLIENTS_RBAC_ENABLED))
+EPHEMERAL_OPTS += --set-parameter "$(APP_COMPONENT)/CLIENTS_RBAC_ENABLED=False"
+else
+ifneq (,$(CLIENTS_RBAC_BASE_URL))
+EPHEMERAL_OPTS += --set-parameter "$(APP_COMPONENT)/CLIENTS_RBAC_BASE_URL=$(CLIENTS_RBAC_BASE_URL)"
+endif
+endif
+
+EPHEMERAL_BONFIRE_PATH ?= config/bonfire.yaml
+
+# Enable frontend deployment
+EPHEMERAL_OPTS += --frontends true
+
+# https://consoledot.pages.redhat.com/docs/dev/creating-a-new-app/using-ee/bonfire/getting-started-with-ees.html
+# Checkout this: https://github.com/RedHatInsights/bonfire/commit/15ac80bfcf9c386eabce33cb219b015a58b756c8
+.PHONY: ephemeral-login
+ephemeral-login: .old-ephemeral-login ## Help in login to the ephemeral cluster
+	@#if [ "$(GH_SESSION_COOKIE)" != "" ]; then python3 $(GO_OUTPUT)/get-token.py; else $(MAKE) .old-ephemeral-login; fi
+
+.PHONY: .old-ephemeral-login
+.old-ephemeral-login:
+	xdg-open "https://oauth-openshift.apps.c-rh-c-eph.8p0c.p1.openshiftapps.com/oauth/token/request"
+	@echo "- Login with github"
+	@echo "- Do click on 'Display Token'"
+	@echo "- Copy 'Log in with this token' command"
+	@echo "- Paste the command in your terminal"
+	@echo ""
+	@echo "Now you should have access to the cluster, remember to use bonfire to manage namespace lifecycle:"
+	@echo '# make ephemeral-namespace-create'
+	@echo ""
+	@echo "Check the namespaces reserved to you by:"
+	@echo '# make ephemeral-namespace-list'
+	@echo ""
+	@echo "If you need to extend 1hour the time for the namespace reservation"
+	@echo '# make ephemeral-namespace-extend-1h'
+	@echo ""
+	@echo "Finally if you don't need the reserved namespace or just you want to cleanup and restart with a fresh namespace you run:"
+	@echo '# make ephemeral-namespace-delete-all'
+
+# Download https://gitlab.cee.redhat.com/klape/get-token/-/blob/main/get-token.py
+$(GO_OUTPUT/get-token.py):
+	curl -Ls -o "$(GO_OUTPUT/get-token.py)" "https://gitlab.cee.redhat.com/klape/get-token/-/raw/main/get-token.py"
+
+# NOTE Changes to config/bonfire.yaml could impact to this rule
+# make ephemeral-deploy EPHEMERAL_NO_BUILD=y CONTAINER_IMAGE_BASE=quay.io/cloudservices/idmsvc-frontend CONTAINER_IMAGE_TAG=7b4abc3
+.PHONY: ephemeral-deploy
+ephemeral-deploy:  ## Deploy application using 'config/bonfire.yaml' file
+	[ "$(EPHEMERAL_NO_BUILD)" == "y" ] || $(MAKE) ephemeral-build-deploy
+	source .venv/bin/activate && \
+	bonfire deploy \
+	    --source appsre \
+		--local-config-path "$(EPHEMERAL_BONFIRE_PATH)" \
+		--local-config-method override \
+		--secrets-dir "$(PROJECT_DIR)/secrets/ephemeral" \
+		--import-secrets \
+		--namespace "$(NAMESPACE)" \
+		--set-parameter "$(APP_COMPONENT)/IMAGE=$(CONTAINER_IMAGE_BASE)" \
+		--set-parameter "$(APP_COMPONENT)/IMAGE_TAG=$(CONTAINER_IMAGE_TAG)" \
+		$(EPHEMERAL_OPTS) \
+		"$(APP)"
+
+# NOTE Changes to config/bonfire.yaml could impact to this rule
+.PHONY: ephemeral-undeploy
+ephemeral-undeploy: ## Undeploy application from the current namespace
+	source .venv/bin/activate && \
+	bonfire process \
+	    --source appsre \
+		--local-config-path "$(EPHEMERAL_BONFIRE_PATH)" \
+		--local-config-method override \
+		--namespace "$(NAMESPACE)" \
+		--set-parameter "$(APP_COMPONENT)/IMAGE=$(CONTAINER_IMAGE_BASE)" \
+		--set-parameter "$(APP_COMPONENT)/IMAGE_TAG=$(CONTAINER_IMAGE_TAG)" \
+		$(EPHEMERAL_OPTS) \
+		"$(APP)" 2>/dev/null | json2yaml | oc delete -f -
+	! oc get secrets/content-sources-certs &>/dev/null || oc delete secrets/content-sources-certs
+
+.PHONY: ephemeral-process
+ephemeral-process: ## Process application from the current namespace
+	source .venv/bin/activate && \
+	bonfire process \
+	    --source appsre \
+		--local-config-path "$(EPHEMERAL_BONFIRE_PATH)" \
+		--namespace "$(NAMESPACE)" \
+		--set-parameter "$(APP_COMPONENT)/IMAGE=$(CONTAINER_IMAGE_BASE)" \
+		--set-parameter "$(APP_COMPONENT)/IMAGE_TAG=$(CONTAINER_IMAGE_TAG)" \
+		$(EPHEMERAL_OPTS) \
+		"$(APP)" 2>/dev/null | json2yaml
+
+# TODO Add command to specify to bonfire the clowdenv template to be used
+.PHONY: ephemeral-namespace-create
+ephemeral-namespace-create:  ## Create a namespace (requires ephemeral environment)
+	oc project "$(shell source .venv/bin/activate && bonfire namespace reserve --force --pool "$(POOL)" -d "$(EPHEMERAL_DURATION)" 2>/dev/null)"
+
+.PHONY: ephemeral-namespace-delete
+ephemeral-namespace-delete: ## Delete current namespace (requires ephemeral environment)
+	source .venv/bin/activate && \
+	bonfire namespace release --force "$(oc project -q)"
+
+.PHONY: ephemeral-namespace-delete-all
+ephemeral-namespace-delete-all: ## Delete all namespace created by us (requires ephemeral environment)
+	source .venv/bin/activate && \
+	for item in $$( bonfire namespace list --mine --output json | jq -r '. | to_entries | map(select(.key | match("ephemeral-*";"i"))) | map(.key) | .[]' ); do \
+	  bonfire namespace release --force $$item ; \
+	done
+
+.PHONY: ephemeral-namespace-list
+ephemeral-namespace-list: ## List all the namespaces reserved to the current user (requires ephemeral environment)
+	source .venv/bin/activate && \
+	bonfire namespace list --mine
+
+.PHONY: ephemeral-namespace-extend
+ephemeral-namespace-extend: ## Extend for EPHEMERAL_DURATION ("4h" default) the usage of the current ephemeral environment
+	source .venv/bin/activate && \
+	bonfire namespace extend --duration "$(EPHEMERAL_DURATION)" "$(NAMESPACE)"
+
+.PHONY: ephemeral-namespace-describe
+ephemeral-namespace-describe: ## Display information about the current namespace
+	@source .venv/bin/activate && \
+	bonfire namespace describe "$(NAMESPACE)"
+
+
+# CONTAINER_IMAGE_BASE should be a public image
+# Tested by 'make ephemeral-build-deploy CONTAINER_IMAGE_BASE=quay.io/avisied0/hmsidm-frontend'
+.PHONY: ephemeral-build-deploy
+ephemeral-build-deploy:  ## Build and deploy image using 'build_deploy.sh' scripts; It requires to pass DOCKER_IMAGE_BASE
+	@$(MAKE) registry-login \
+		CONTAINER_REGISTRY_USER="$(QUAY_USER)" \
+		CONTAINER_REGISTRY_TOKEN="$(QUAY_TOKEN)" \
+		CONTAINER_REGISTRY="quay.io"
+	$(MAKE) container-build
+	$(MAKE) container-push
+	IMAGE="$(CONTAINER_IMAGE_BASE)" IMAGE_TAG="$(CONTAINER_IMAGE_TAG)" $(DOCKER) build ./build_deploy.sh 2>&1 | tee build_deploy.log
+
+.PHONY: ephemeral-pr-checks
+ephemeral-pr-checks:
+	IMAGE="$(CONTAINER_IMAGE_BASE)" bash ./pr_checks.sh
+
+# FIXME This rule will require some updates but it will be something similar
+.PHONY: ephemeral-test-backend
+ephemeral-test-backend:  ## Run IQE tests in the ephemeral environment (require to run ephemeral-deploy before)
+	source .venv/bin/activate && \
+	bonfire deploy-iqe-cji \
+	  --env clowder_smoke \
+	  --cji-name "$(APP)-$(APP_COMPONENT)" \
+	  --namespace "$(NAMESPACE)" \
+	  "$(APP)"
+
+# https://kubernetes.io/docs/tasks/administer-cluster/dns-debugging-resolution/
+.PHONY: ephemeral-run-dnsutil
+ephemeral-run-dnsutil:  ## Run a shell in a new pod to debug dns situations
+	oc run dnsutil --rm --image=registry.k8s.io/e2e-test-images/jessie-dnsutils:1.3 -it -- bash

scripts/mk/includes.mk

@@ -4,8 +4,16 @@
 include scripts/mk/projectdir.mk
 
 -include secrets/private.mk
+include scripts/mk/venv.mk
 include scripts/mk/variables.mk
 
 include scripts/mk/help.mk
 include scripts/mk/printvars.mk
 include scripts/mk/crc-frontend.mk
+
+include scripts/mk/meta-container.mk
+include scripts/mk/container.mk
+
+include scripts/mk/meta-ephemeral.mk
+include scripts/mk/ephemeral.mk
+

scripts/mk/meta-container.mk

@@ -0,0 +1 @@
+##@ Container rules

scripts/mk/meta-ephemeral.mk

@@ -0,0 +1 @@
+##@ Ephemeral environment rules

scripts/mk/private.example.mk

@@ -0,0 +1,35 @@
+##
+# This file contains custom variables definition to override
+# other values used in the different makefiles
+##
+
+# Your quay user as it is used for setting CONTAINER_IMAGE_BASE
+# The link below is to regenerate the token:
+# NOTE Don't forget to grant write permission to the robot account
+#      only for the repository/ies you need
+#   https://quay.io/repository/YOURUSER/YOURIMAGE?tab=settings
+# TODO Update with your robot account
+export QUAY_USER := user+robot_account
+export QUAY_TOKEN := MYROBOTACCOUNT
+export QUAY_LOGIN := $(firstword $(subst +, ,$(QUAY_USER)))
+# TODO Update with the image name created in your repository
+export QUAY_REPOSITORY := YOURIMAGE
+
+# Point out to your local repository
+CONTAINER_IMAGE_BASE ?= quay.io/$(QUAY_LOGIN)/$(QUAY_REPOSITORY)
+
+# https://access.redhat.com/RegistryAuthentication
+# https://access.redhat.com/RegistryAuthentication#creating-registry-service-accounts-6
+# To retrieve your token or regenerate it
+# https://access.redhat.com/terms-based-registry/#/token/YOUR_USERNAME
+# TODO Update your account details
+export RH_REGISTRY_USER := 
+export RH_REGISTRY_TOKEN := 
+
+# Ephemeral pool
+POOL ?= default
+# POOL ?= real-managed-kafka
+
+# NOTE This could be necessary when too many deployments are made.
+# export GITLAB_TOKEN :=
+

scripts/mk/variables.mk

@@ -2,6 +2,6 @@
 # General variables leveraged on the repository.
 ##
 
-APP := hmsidm
+APP := idmsvc
 
 

scripts/mk/venv.mk

@@ -0,0 +1,20 @@
+# install Python tools in a virtual environment
+
+PYTHON_VENV := .venv
+BONFIRE := $(PYTHON_VENV)/bin/bonfire
+PRE_COMMIT := $(PYTHON_VENV)/bin/pre-commit
+JSON2YAML := $(PYTHON_VENV)/bin/json2yaml
+
+$(PYTHON_VENV):
+	python3 -m venv $(PYTHON_VENV)
+	$(PYTHON_VENV)/bin/pip install -U pip setuptools
+
+$(BONFIRE) $(PRE_COMMIT) $(JSON2YAML): $(PYTHON_VENV)
+	$(PYTHON_VENV)/bin/pip3 install -r requirements-dev.txt
+	touch $(BONFIRE) $(PRE_COMMIT)
+
+.PHONY: install-python-tools
+install-python-tools:
+	$(MAKE) $(BONFIRE)
+	$(MAKE) $(PRE_COMMIT)
+	$(MAKE) $(JSON2YAML)