From 7a8e2a352448f7f5ec80e6e0375f3382e21770b3 Mon Sep 17 00:00:00 2001 From: Daniel Skinstad Drabitzius Date: Thu, 13 Feb 2025 14:05:20 +0100 Subject: [PATCH] chore: trigger integration tests * Trigger integration tests * Move the order and make the release jobs depend on trigger-integration to ensure that we tag and publish the images after they're tested Ticket: QA-828 Signed-off-by: Daniel Skinstad Drabitzius --- .gitlab-ci.yml | 2 +- gitlab-pipeline/stage/release.yml | 45 ++++++++++---- gitlab-pipeline/stage/trigger-integration.yml | 58 ++++++++++++++++--- 3 files changed, 83 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 59fcbd1d..7e64e7c4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -233,8 +233,8 @@ stages: - build - yocto:build-n-test - test - - release - trigger:packages - trigger:images - trigger:integration + - release - .post diff --git a/gitlab-pipeline/stage/release.yml b/gitlab-pipeline/stage/release.yml index 049818ab..a1ae8120 100644 --- a/gitlab-pipeline/stage/release.yml +++ b/gitlab-pipeline/stage/release.yml @@ -51,6 +51,7 @@ release:testing-boards:beagleboneblack:automatic: dependencies: - init:workspace - build:acceptance:beagleboneblack + - trigger:integration-tests only: variables: - $PUBLISH_RELEASE_AUTOMATIC == "true" && $BUILD_BEAGLEBONEBLACK == "true" @@ -150,6 +151,7 @@ release:mender-cli:automatic: dependencies: - init:workspace - build:mender-cli + - trigger:integration-tests extends: .template_release_binary_tools .template_release_mender-monitor: @@ -160,9 +162,6 @@ release:mender-cli:automatic: variables: S3_BUCKET_NAME: "mender-binaries" S3_BUCKET_PATH: "mender-monitor/yocto" - dependencies: - - init:workspace - - build:mender-monitor:package before_script: # Install dependencies - apt update && apt install -yyq awscli git wget python3 python3-pip xz-utils @@ -184,10 +183,17 @@ release:mender-cli:automatic: - aws s3 cp stage-artifacts/mender-monitor-*.tar.gz s3://$S3_BUCKET_NAME/$S3_BUCKET_PATH/$mender_monitor_version/ release:mender-monitor:manual: + dependencies: + - init:workspace + - build:mender-monitor:package when: manual extends: .template_release_mender-monitor release:mender-monitor:automatic: + dependencies: + - init:workspace + - build:mender-monitor:package + - trigger:integration-tests only: variables: - $PUBLISH_RELEASE_AUTOMATIC == "true" @@ -202,9 +208,6 @@ release:mender-monitor:automatic: S3_BUCKET_NAME: "mender-binaries" S3_BUCKET_PATH_YOCTO: "mender-gateway/yocto" S3_BUCKET_PATH_EXAMPLES: "mender-gateway/examples" - dependencies: - - init:workspace - - build:mender-gateway:package before_script: # Early exit when building an integration version without mender-gateway - apt update && apt install -yyq xz-utils @@ -231,10 +234,17 @@ release:mender-monitor:automatic: - aws s3 cp stage-artifacts/mender-gateway-examples-*.tar s3://$S3_BUCKET_NAME/$S3_BUCKET_PATH_EXAMPLES/$mender_gateway_version/ release:mender-gateway:manual: + dependencies: + - init:workspace + - build:mender-gateway:package when: manual extends: .template_release_mender-gateway release:mender-gateway:automatic: + dependencies: + - init:workspace + - build:mender-gateway:package + - trigger:integration-tests only: variables: - $PUBLISH_RELEASE_AUTOMATIC == "true" @@ -247,10 +257,6 @@ release:mender-gateway:automatic: image: docker services: - docker:dind - dependencies: - - init:workspace - - build:client:qemu - - build:client:docker before_script: # Check correct dind setup - docker version @@ -272,10 +278,16 @@ release:mender-gateway:automatic: - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" "$CI_REGISTRY" --password-stdin script: # Load, tag and push mender-client-* images + # Historical note, + # The version here used to be determined by the release_tool, which could distinguish between + # git repo version and container image version. Today this is kind of messy, as we moved out + # the backend to a separate repo but we haven't done deep changes in the release process for + # the rest, and "--version-of" option gives bogus results. + # TLDR; hardcode to mender-${INTEGRATION_REV} version for the Virtual Devices images - | version=mender-${INTEGRATION_REV} - images=("mender-client-docker" "mender-client-docker-addons" "mender-client-qemu" "mender-client-qemu-rofs" "mender-gateway-qemu-commercial" "mender-monitor-qemu-commercial" "mender-qemu-rofs-commercial") - for image in "${images[@]}"; do + images="mender-client-docker mender-client-docker-addons mender-client-qemu mender-client-qemu-rofs mender-gateway-qemu-commercial mender-monitor-qemu-commercial mender-qemu-rofs-commercial" + for image in $images; do docker pull ${GITLAB_REGISTRY_PREFIX}-${image} tagged_image=docker.io/mendersoftware/${image}:${version} docker tag ${GITLAB_REGISTRY_PREFIX}-${image} ${tagged_image} @@ -285,12 +297,21 @@ release:mender-gateway:automatic: # This job allows mender repo to publish the related Docker client images on # merges to master or release branches. release:virtual-client:manual: + dependencies: + - init:workspace + - build:client:qemu + - build:client:docker when: manual extends: .template_release_docker_images:client-only # This job allows a release to publish only the client images. Specifically designed for Mender 3.7 # series, to complement job release_docker_multiplatform_images:manual release:virtual-client:automatic: + dependencies: + - init:workspace + - build:client:qemu + - build:client:docker + - trigger:integration-tests only: variables: - $PUBLISH_DOCKER_CLIENT_IMAGES == "true" diff --git a/gitlab-pipeline/stage/trigger-integration.yml b/gitlab-pipeline/stage/trigger-integration.yml index d45c0459..22ce561d 100644 --- a/gitlab-pipeline/stage/trigger-integration.yml +++ b/gitlab-pipeline/stage/trigger-integration.yml @@ -3,19 +3,59 @@ inherit: variables: false variables: - MENDER_CLIENT_TAG: mender-$INTEGRATION_REV - MENDER_GATEWAY_QEMU_TAG: mender-$INTEGRATION_REV - MENDER_GATEWAY_TAG: $MENDER_GATEWAY_REV + MENDER_CLIENT_REGISTRY: registry.gitlab.com + MENDER_CLIENT_ENTERPRISE_REGISTRY: registry.gitlab.com + MENDER_CLIENT_REPOSITORY: northern.tech/mender + + MENDER_CLIENT_DOCKER_IMAGE: mender-qa + MENDER_CLIENT_DOCKER_TAG: ${CI_PIPELINE_ID}-mender-client-docker + + MENDER_CLIENT_DOCKER_ADDONS_IMAGE: mender-qa + MENDER_CLIENT_DOCKER_ADDONS_TAG: ${CI_PIPELINE_ID}-mender-client-docker-addons + + MENDER_CLIENT_QEMU_IMAGE: mender-qa + MENDER_CLIENT_QEMU_TAG: ${CI_PIPELINE_ID}-mender-client-qemu + + MENDER_CLIENT_QEMU_ROFS_IMAGE: mender-qa + MENDER_CLIENT_QEMU_ROFS_TAG: ${CI_PIPELINE_ID}-mender-client-qemu-rofs + + MENDER_GATEWAY_QEMU_COMMERCIAL_IMAGE: mender-qa + MENDER_GATEWAY_QEMU_COMMERCIAL_TAG: ${CI_PIPELINE_ID}-mender-gateway-qemu-commercial + + MENDER_MONITOR_QEMU_COMMERCIAL_IMAGE: mender-qa + MENDER_MONITOR_QEMU_COMMERCIAL_TAG: ${CI_PIPELINE_ID}-mender-monitor-qemu-commercial + + MENDER_QEMU_ROFS_COMMERCIAL_IMAGE: mender-qa + MENDER_QEMU_ROFS_COMMERCIAL_TAG: ${CI_PIPELINE_ID}-mender-qemu-rofs-commercial + + MENDER_GATEWAY_TAG: ${MENDER_GATEWAY_REV} + RUN_TESTS_FULL_INTEGRATION: "true" # TODO: remove these parallel 1 overrides whenever tests are stable CI_JOBS_IN_PARALLEL_INTEGRATION: 1 XDIST_JOBS_IN_PARALLEL_INTEGRATION: 1 + +trigger:generate-gitlab-integration-rev: + stage: trigger:integration + script: + # Convert INTEGRATION_REV on format `pull/0000/head` to `pr_0000` to specify which gitlab branch to trigger + - | + GITLAB_INTEGRATION_REV=$INTEGRATION_REV + if [[ "$INTEGRATION_REV" =~ ^pull/([0-9]+)/head$ ]]; then + GITLAB_INTEGRATION_REV="pr_${BASH_REMATCH[1]}" + fi + - echo "GITLAB_INTEGRATION_REV=$GITLAB_INTEGRATION_REV" >> gitlab_integration_rev.env + artifacts: + reports: + dotenv: gitlab_integration_rev.env + +trigger:integration-tests: + extends: .template:trigger:integration-tests + needs: + - trigger:generate-gitlab-integration-rev + - build:client:qemu + when: always trigger: project: Northern.tech/Mender/integration - branch: $INTEGRATION_REV + branch: $GITLAB_INTEGRATION_REV strategy: depend - -trigger:integration-tests:manual: - needs: [] - when: manual - extends: .template:trigger:integration-tests