diff --git a/.azure-pipelines/templates/02_docker_build.yaml b/.azure-pipelines/templates/02_docker_build.yaml index 8ff94e6..9d33e54 100644 --- a/.azure-pipelines/templates/02_docker_build.yaml +++ b/.azure-pipelines/templates/02_docker_build.yaml @@ -9,10 +9,9 @@ stages: ) dependsOn: [] jobs: - - job: build + - job: build_saleor_core displayName: Build saleor-core image condition: always() - depends_on: [] steps: - template: ../init_step.yaml @@ -27,10 +26,9 @@ stages: script: | make build_saleor_core - - job: build + - job: build_saleor_core_dev displayName: Build saleor-core-dev image condition: always() - depends_on: [] steps: - template: ../init_step.yaml @@ -45,10 +43,9 @@ stages: script: | make build_saleor_core_dev - - job: build + - job: build_saleor_dashboard displayName: Build saleor-dashboard image condition: always() - depends_on: [] steps: - template: ../init_step.yaml @@ -63,10 +60,9 @@ stages: script: | make build_saleor_dashboard - - job: build - displayName: Build saleor-storefront + - job: build_saleor_storefront + displayName: Build saleor-storefront image condition: always() - depends_on: [] steps: - template: ../init_step.yaml diff --git a/.azure-pipelines/templates/03_docker_push.yaml b/.azure-pipelines/templates/03_docker_push.yaml new file mode 100644 index 0000000..495098f --- /dev/null +++ b/.azure-pipelines/templates/03_docker_push.yaml @@ -0,0 +1,91 @@ +# @format +--- +stages: + - stage: Push + displayName: Push images + condition: eq(variables['Build.SourceBranch'], 'refs/heads/master') + dependsOn: [] + jobs: + - job: push_saleor_core + displayName: Push saleor-core image + condition: always() + steps: + - template: ../init_step.yaml + + - task: Bash@3 + displayName: Push saleor-core image + inputs: + targetType: "inline" + workingDirectory: "$(Build.Repository.LocalPath)" + failOnStderr: false + noProfile: true + noRc: true + script: | + make build_saleor_core + make push_saleor_core + env: + REGISTRY_TOKEN: $(registryToken) + FORCE_PUSH_IMAGES: $(forcePush) + + - job: push_saleor_core_dev + displayName: Push saleor-core-dev image + condition: always() + steps: + - template: ../init_step.yaml + + - task: Bash@3 + displayName: Push saleor-core-dev image + inputs: + targetType: "inline" + workingDirectory: "$(Build.Repository.LocalPath)" + failOnStderr: false + noProfile: true + noRc: true + script: | + make build_saleor_core_dev + make push_saleor_core_dev + env: + REGISTRY_TOKEN: $(registryToken) + FORCE_PUSH_IMAGES: $(forcePush) + + - job: push_saleor_dashboard + displayName: Push saleor-dashboard image + condition: always() + steps: + - template: ../init_step.yaml + + - task: Bash@3 + displayName: Push saleor-dashboard image + inputs: + targetType: "inline" + workingDirectory: "$(Build.Repository.LocalPath)" + failOnStderr: false + noProfile: true + noRc: true + script: | + make build_saleor_dashboard + make push_saleor_dashboard + env: + REGISTRY_TOKEN: $(registryToken) + FORCE_PUSH_IMAGES: $(forcePush) + + - job: push_saleor_storefront + displayName: Push saleor-storefront image + condition: always() + steps: + - template: ../init_step.yaml + + - task: Bash@3 + displayName: Push saleor-storefront image + inputs: + targetType: "inline" + workingDirectory: "$(Build.Repository.LocalPath)" + failOnStderr: false + noProfile: true + noRc: true + script: | + make build_saleor_storefront + make push_saleor_storefront + env: + REGISTRY_TOKEN: $(registryToken) + FORCE_PUSH_IMAGES: $(forcePush) diff --git a/.gitignore b/.gitignore index c231608..1e3b14b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ node_modules .env saleor_core +saleor_core_dev saleor_dashboard saleor_storefront charts/saleor-platform/charts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e5abe6..5874cfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,10 @@ **_Added_** -- Saleor source code built as docker images rebuilt with customizations required for the k8s deployments +- Saleor source code built as docker images rebuilt with customizations required for the k8s deployments. - Added initial helm charts with each component in it's own chart, namely: - saleor-core - saleor-storefront - saleor-dashboard - Added an umbrella chart with all chart dependencies (sub-charts) specified for consumption. -- Added scripts and makefile for CI operations +- Added scripts and makefile for CI operations. diff --git a/Makefile b/Makefile index 83e5e36..6db6021 100644 --- a/Makefile +++ b/Makefile @@ -11,11 +11,19 @@ SHELL := /bin/bash init_all \ format_all \ lint_all \ + prepare_saleor_sources \ build_saleor_core \ + build_saleor_core_dev \ + build_saleor_dashboard \ + build_saleor_storefront \ push_saleor_core \ + push_saleor_core_dev \ + push_saleor_dashboard \ + push_saleor_storefront \ get_image \ get_image_version + ## Dependency installation targets init_yarn: @@ -70,15 +78,53 @@ lint_all: init_all ./scripts/make.sh lint_dockerfiles get_image: - @./scripts/make.sh get_image saleor_core + @if test -z "$(FILTER)"; then \ + echo "env variable FILTER is required" && \ + exit 1; \ + fi; + @./scripts/make.sh get_image "${FILTER}" + unset FILTER get_image_version: - @./scripts/make.sh get_image_version - -build_saleor_core: + @if test -z "$(SALEOR_REPO)"; then \ + echo "env variable SALEOR_REPO is required" && \ + exit 1; \ + fi; + @./scripts/make.sh get_image_version "${SALEOR_REPO}" + unset SALEOR_REPO + +push_image: + @if test -z "$(IMAGE_VERSION)"; then \ + echo "env variable IMAGE_VERSION is required" && \ + exit 1; \ + fi; + @if test -z "$(DOCKER_REPO)"; then \ + echo "env variable DOCKER_REPO is required" && \ + exit 1; \ + fi; + @./scripts/make.sh push_image "${IMAGE_VERSION}" "${DOCKER_REPO}" "${FORCE_PUSH}" + unset IMAGE_VERSION + unset DOCKER_REPO + +prepare_saleor_sources: + ./scripts/make.sh prepare_saleor_source \ + "https://github.com/mirumee/saleor.git" \ + "Core.Dockerfile" && \ + ./scripts/make.sh prepare_saleor_source \ + "https://github.com/mirumee/saleor.git" \ + "Core.Dev.Dockerfile" && \ + ./scripts/make.sh prepare_saleor_source \ + "https://github.com/mirumee/saleor-dashboard.git" \ + "Dashboard.Dockerfile" && \ + ./scripts/make.sh prepare_saleor_source \ + "https://github.com/mirumee/saleor-storefront.git" \ + "Storefront.Dockerfile" + +build_saleor_core: prepare_saleor_sources ./scripts/make.sh set_env_saleor_core - ./scripts/make.sh prepare_saleor_source "https://github.com/mirumee/saleor.git" "Core.Dockerfile" docker-compose build saleor_core + docker image ls + make -s get_image FILTER=saleor_core docker run \ --rm "$$(make -s get_image FILTER=saleor_core)" \ /bin/bash -c 'echo "build ok"' || \ @@ -87,10 +133,11 @@ build_saleor_core: "$$(make -s get_image FILTER=saleor_core)" rm .env -build_saleor_core_dev: +build_saleor_core_dev: prepare_saleor_sources ./scripts/make.sh set_env_saleor_core_dev - ./scripts/make.sh prepare_saleor_source "https://github.com/mirumee/saleor.git" "Core.Dev.Dockerfile" docker-compose build saleor_core_dev + docker image ls + make -s get_image FILTER=saleor_core_dev docker run \ --rm "$$(make -s get_image FILTER=saleor_core_dev)" \ /bin/bash -c 'echo "build ok"' || \ @@ -99,10 +146,11 @@ build_saleor_core_dev: "$$(make -s get_image FILTER=saleor_core_dev)" rm .env -build_saleor_dashboard: +build_saleor_dashboard: prepare_saleor_sources ./scripts/make.sh set_env_saleor_dashboard - ./scripts/make.sh prepare_saleor_source "https://github.com/mirumee/saleor-dashboard.git" "Dashboard.Dockerfile" docker-compose build saleor_dashboard + docker image ls + make -s get_image FILTER=saleor_dashboard docker run \ --rm "$$(make -s get_image FILTER=saleor_dashboard)" \ /bin/bash -c 'echo "build ok"' || \ @@ -111,10 +159,11 @@ build_saleor_dashboard: "$$(make -s get_image FILTER=saleor_dashboard)" rm .env -build_saleor_storefront: +build_saleor_storefront: prepare_saleor_sources ./scripts/make.sh set_env_saleor_storefront - ./scripts/make.sh prepare_saleor_source "https://github.com/mirumee/saleor-storefront.git" "Dashboard.Dockerfile" docker-compose build saleor_storefront + docker image ls + make -s get_image FILTER=saleor_storefront docker run \ --rm "$$(make -s get_image FILTER=saleor_storefront)" \ /bin/bash -c 'echo "build ok"' || \ @@ -123,19 +172,78 @@ build_saleor_storefront: "$$(make -s get_image FILTER=saleor_storefront)" rm .env -#push_saleor_core: -# @if test -z "$(REGISTRY_TOKEN)"; then \ -# echo "env variable REGISTRY_TOKEN is required" && \ -# exit 1; \ -# fi; -# docker tag \ -# "$$(make -s get_image FILTER=saleor_core)" \ -# "ghcr.io/eirenauts/saleor-core:$$(make -s get_image_version)" -# docker tag \ -# "$$(make -s get_image FILTER=saleor_core)" \ -# "ghcr.io/eirenauts/saleor-core:latest" -# echo "${REGISTRY_TOKEN}" | docker login ghcr.io -u eirenauts --password-stdin -# docker push "ghcr.io/eirenauts/saleor-core:$$(make -s get_image_version)" -# docker push "ghcr.io/eirenauts/saleor-core:latest" -# docker logout ghcr.io -# if [[ -e /home/vsts/.docker/config.json ]]; then rm /home/vsts/.docker/config.json; fi +push_saleor_core: + @if test -z "$(REGISTRY_TOKEN)"; then \ + echo "env variable REGISTRY_TOKEN is required" && \ + exit 1; \ + fi; + docker tag \ + "$$(make -s get_image FILTER=saleor_core)" \ + "ghcr.io/eirenauts/saleor-core:$$(make -s get_image_version SALEOR_REPO=https://github.com/mirumee/saleor.git)" + docker tag \ + "$$(make -s get_image FILTER=saleor_core)" \ + "ghcr.io/eirenauts/saleor-core:latest" + echo "${REGISTRY_TOKEN}" | docker login ghcr.io -u eirenauts --password-stdin + make -s push_image \ + IMAGE_VERSION="$$(make -s get_image_version SALEOR_REPO=https://github.com/mirumee/saleor.git)" \ + DOCKER_REPO=saleor-core \ + FORCE_PUSH="${FORCE_PUSH_IMAGES}" + docker logout ghcr.io + if [[ -e /home/vsts/.docker/config.json ]]; then rm /home/vsts/.docker/config.json; fi + +push_saleor_core_dev: + @if test -z "$(REGISTRY_TOKEN)"; then \ + echo "env variable REGISTRY_TOKEN is required" && \ + exit 1; \ + fi; + docker tag \ + "$$(make -s get_image FILTER=saleor_core_dev)" \ + "ghcr.io/eirenauts/saleor-core:dev-$$(make -s get_image_version SALEOR_REPO=https://github.com/mirumee/saleor.git)" + docker tag \ + "$$(make -s get_image FILTER=saleor_core_dev)" \ + "ghcr.io/eirenauts/saleor-core:dev-latest" + echo "${REGISTRY_TOKEN}" | docker login ghcr.io -u eirenauts --password-stdin + make -s push_image \ + IMAGE_VERSION="$$(make -s get_image_version SALEOR_REPO=https://github.com/mirumee/saleor.git)" \ + DOCKER_REPO=saleor-core \ + FORCE_PUSH="${FORCE_PUSH_IMAGES}" + docker logout ghcr.io + if [[ -e /home/vsts/.docker/config.json ]]; then rm /home/vsts/.docker/config.json; fi + +push_saleor_dashboard: + @if test -z "$(REGISTRY_TOKEN)"; then \ + echo "env variable REGISTRY_TOKEN is required" && \ + exit 1; \ + fi; + docker tag \ + "$$(make -s get_image FILTER=saleor_dashboard)" \ + "ghcr.io/eirenauts/saleor-dashboard:$$(make -s get_image_version SALEOR_REPO=https://github.com/mirumee/saleor-dashboard.git)" + docker tag \ + "$$(make -s get_image FILTER=saleor_dashboard)" \ + "ghcr.io/eirenauts/saleor-dashboard:latest" + echo "${REGISTRY_TOKEN}" | docker login ghcr.io -u eirenauts --password-stdin + make -s push_image \ + IMAGE_VERSION="$$(make -s get_image_version SALEOR_REPO=https://github.com/mirumee/saleor-dashboard.git)" \ + DOCKER_REPO=saleor-dashboard \ + FORCE_PUSH="${FORCE_PUSH_IMAGES}" + docker logout ghcr.io + if [[ -e /home/vsts/.docker/config.json ]]; then rm /home/vsts/.docker/config.json; fi + +push_saleor_storefront: + @if test -z "$(REGISTRY_TOKEN)"; then \ + echo "env variable REGISTRY_TOKEN is required" && \ + exit 1; \ + fi; + docker tag \ + "$$(make -s get_image FILTER=saleor_storefront)" \ + "ghcr.io/eirenauts/saleor-storefront:$$(make -s get_image_version SALEOR_REPO=https://github.com/mirumee/saleor-storefront.git)" + docker tag \ + "$$(make -s get_image FILTER=saleor_storefront)" \ + "ghcr.io/eirenauts/saleor-storefront:latest" + echo "${REGISTRY_TOKEN}" | docker login ghcr.io -u eirenauts --password-stdin + make -s push_image \ + IMAGE_VERSION="$$(make -s get_image_version SALEOR_REPO=https://github.com/mirumee/saleor-storefront.git)" \ + DOCKER_REPO=saleor-storefront \ + FORCE_PUSH="${FORCE_PUSH_IMAGES}" + docker logout ghcr.io + if [[ -e /home/vsts/.docker/config.json ]]; then rm /home/vsts/.docker/config.json; fi diff --git a/docker-compose.yaml b/docker-compose.yaml index dbba638..ba63ced 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -23,17 +23,17 @@ services: UID: 1001 GUID: 1001 - saleor_core_dashboard: + saleor_dashboard: build: - context: ./saleor_core_dashboard + context: ./saleor_dashboard dockerfile: Dockerfile args: SHORT_SHA: ${SHORT_SHA} VERSION: ${VERSION} - saleor_core_storefront: + saleor_storefront: build: - context: ./saleor_core_storefront + context: ./saleor_storefront dockerfile: Dockerfile args: SHORT_SHA: ${SHORT_SHA} diff --git a/images/Core.Dev.Dockerfile b/images/Core.Dev.Dockerfile index fcda274..d3f97e6 100644 --- a/images/Core.Dev.Dockerfile +++ b/images/Core.Dev.Dockerfile @@ -46,7 +46,7 @@ LABEL \ \ org.opencontainers.image.title="saleor-core" \ org.opencontainers.image.description="Docker image for saleor core k8s containers" \ - org.opencontainers.image.url="ghcr.io/eirenauts/saleor-core:dev-${VERSION}" \ + org.opencontainers.image.url="ghcr.io/eirenauts/saleor-core:${VERSION}" \ org.opencontainers.image.source="https://github.com/mirumee/saleor" \ org.opencontainers.image.revision="${SHORT_SHA}" \ org.opencontainers.image.version="${VERSION}" \ @@ -63,7 +63,7 @@ RUN \ \ \ apt-get update -y && \ - apt-get install -y --no-install-recommends \ + apt-get install -y --no-install-recommends --allow-downgrades \ libxml2='2.9.4+dfsg1-7+deb10u1' \ libssl1.1='1.1.1d-0+deb10u3' \ libcairo2='1.16.0-4' \ diff --git a/images/Core.Dockerfile b/images/Core.Dockerfile index c6988da..1a1ecb6 100644 --- a/images/Core.Dockerfile +++ b/images/Core.Dockerfile @@ -56,7 +56,7 @@ WORKDIR /app SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] RUN \ apt-get update -y && \ - apt-get install -y --no-install-recommends \ + apt-get install -y --no-install-recommends --allow-downgrades \ libxml2='2.9.4+dfsg1-7+deb10u1' \ libssl1.1='1.1.1d-0+deb10u3' \ libcairo2='1.16.0-4' \ diff --git a/scripts/ci.sh b/scripts/ci.sh index 78c424e..e272032 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -219,6 +219,26 @@ function get_git_tag() { git tag --points-at HEAD 2>/dev/null || echo "" } +function get_branch_from_azure_devops_ci() { + if [[ -n "${SYSTEM_PULLREQUEST_SOURCEBRANCH}" ]]; then + sed --regexp-extended 's|refs/heads/||g' <<<"${SYSTEM_PULLREQUEST_SOURCEBRANCH}" + elif [[ -n "${BUILD_SOURCEBRANCH}" ]]; then + sed --regexp-extended 's|refs/heads/||g' <<<"${BUILD_SOURCEBRANCH}" + fi +} + +function get_git_branch() { + if [[ -z "$(get_branch_from_azure_devops_ci)" ]]; then + git branch --show-current + else + get_branch_from_azure_devops_ci + fi +} + +function get_redacted_git_branch() { + sed --regexp-extended 's|\W|-|g' <<<"$(get_git_branch)" +} + function get_image_version() { local saleor_repo="${1}" local branch @@ -228,6 +248,8 @@ function get_image_version() { return fi + branch="$(get_redacted_git_branch)" + required_version="$(get_required_app_version "${saleor_repo}")" if [[ -z "$(get_git_tag)" ]]; then @@ -305,12 +327,13 @@ function prepare_saleor_source() { working_dir="$(get_working_directory "${dockerfile}")" if [[ -d "./${working_dir}" ]]; then + echo "Removing ./${working_dir}" rm -rf "./${working_dir}" fi saleor_release="$(get_required_app_version "${saleor_repo}")" - echo "Checking out version ${saleor_release} of saleor" + echo "Checking out version ${saleor_release} of ${saleor_repo}" git clone "${saleor_repo}" "./${working_dir}" && cd "./${working_dir}" && git checkout "${saleor_release}" && @@ -318,9 +341,36 @@ function prepare_saleor_source() { echo "Copying gunicorn config file into saleor repo" && cp ../images/config/gunicorn_conf.py "./saleor/gunicorn_conf.py" echo "Removing unecessary static files" && - rm ./saleor/static/populatedb_data.json && - rm -rf ./saleor/static/placeholders + rm ./saleor/static/populatedb_data.json && + rm -rf ./saleor/static/placeholders fi && cp "../images/${dockerfile}" ./Dockerfile && cat Dockerfile } + +function push_image() { + local image_version="${1}" + local docker_repo="${2}" + local force_push="${3}" + + if [[ -z "${image_version}" ]]; then + echo "Variable image_version is required" >>/dev/stderr + return + fi + + if [[ -z "${docker_repo}" ]]; then + echo "Variable docker_repo is required" >>/dev/stderr + return + fi + + if [[ -n "${force_push}" ]]; then + echo "Force pushing images" + docker push "ghcr.io/eirenauts/${docker_repo}:${image_version}" && + docker push "ghcr.io/eirenauts/${docker_repo}:latest" + else + docker manifest inspect "ghcr.io/eirenauts/${docker_repo}:${image_version}" >/dev/null 2>&1 && + echo "Image already exists, skipping push" || + docker push "ghcr.io/eirenauts/${docker_repo}:${image_version}" && + docker push "ghcr.io/eirenauts/${docker_repo}:latest" + fi +} diff --git a/scripts/env.sh b/scripts/env.sh index f11f689..23b77f6 100755 --- a/scripts/env.sh +++ b/scripts/env.sh @@ -13,7 +13,15 @@ function set_env_saleor_core() { } function set_env_saleor_core_dev() { - set_env_saleor_core + if [[ -e .env ]]; then + rm .env && + touch .env + fi + + { + echo "SHORT_SHA=$(get_short_sha)" + echo "VERSION=dev-$(get_image_version https://github.com/mirumee/saleor.git)" + } >>.env } function set_env_saleor_dashboard() { @@ -24,7 +32,7 @@ function set_env_saleor_dashboard() { { echo "SHORT_SHA=$(get_short_sha)" - echo "VERSION=dev-$(get_image_version https://github.com/mirumee/saleor-dashboard.git)" + echo "VERSION=$(get_image_version https://github.com/mirumee/saleor-dashboard.git)" } >>.env } @@ -36,6 +44,6 @@ function set_env_saleor_storefront() { { echo "SHORT_SHA=$(get_short_sha)" - echo "VERSION=dev-$(get_image_version https://github.com/mirumee/saleor-storefront.git)" + echo "VERSION=$(get_image_version https://github.com/mirumee/saleor-storefront.git)" } >>.env }