QA-828 - trigger integration tests

.gitlab-ci.yml

@@ -233,8 +233,8 @@ stages:
   - build
   - yocto:build-n-test
   - test
-  - release
   - trigger:packages
   - trigger:images
   - trigger:integration
+  - release
   - .post

gitlab-pipeline/stage/build.yml

@@ -43,24 +43,17 @@ build:client:docker:
   script:
     - echo ${CI_REGISTRY_PASSWORD} | docker login --username ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY}
     # First build mender's repo Docker image
-    - docker_url=$($WORKSPACE/integration/extra/release_tool.py --map-name docker mender-client-docker docker_url)
     - cd go/src/github.com/mendersoftware/mender
-    - ./tests/build-docker -t $docker_url:pr
-    - docker tag ${docker_url}:pr ${GITLAB_REGISTRY_PREFIX}-mender-client-docker
+    - ./tests/build-docker -t ${GITLAB_REGISTRY_PREFIX}-mender-client-docker
     - docker push ${GITLAB_REGISTRY_PREFIX}-mender-client-docker
-    # Then, if available, build integration's repo Docker image (Mender 2.7 and later)
-    - if $($WORKSPACE/integration/extra/release_tool.py -l docker | egrep -q '^mender-client-docker-addons$'); then
-    -   docker_url=$($WORKSPACE/integration/extra/release_tool.py --map-name docker mender-client-docker-addons docker_url)
-    -   cd $WORKSPACE/integration/extra/mender-client-docker-addons
-    -   docker build
-        --build-arg MENDER_CLIENT_REV=$MENDER_REV
-        --build-arg MENDER_CONNECT_REV=$MENDER_CONNECT_REV
-        --build-arg MENDER_SETUP_REV=$MENDER_SETUP_REV
-        --tag $docker_url:pr
-        .
-    -   docker tag ${docker_url}:pr ${GITLAB_REGISTRY_PREFIX}-mender-client-docker-addons
-    -   docker push ${GITLAB_REGISTRY_PREFIX}-mender-client-docker-addons
-    - fi
+    - cd $WORKSPACE/integration/extra/mender-client-docker-addons
+    - docker build
+      --build-arg MENDER_CLIENT_REV=$MENDER_REV
+      --build-arg MENDER_CONNECT_REV=$MENDER_CONNECT_REV
+      --build-arg MENDER_SETUP_REV=$MENDER_SETUP_REV
+      --tag ${GITLAB_REGISTRY_PREFIX}-mender-client-docker-addons
+      --push
+      .
     - echo "success" > /JOB_RESULT.txt
   after_script:
     - ${CI_PROJECT_DIR}/scripts/github_pull_request_status $(cat /JOB_RESULT.txt) "Gitlab ${CI_JOB_NAME} finished" "${CI_JOB_URL}" "${CI_JOB_NAME}/${INTEGRATION_REV}"

gitlab-pipeline/stage/release.yml

@@ -51,6 +51,7 @@ release:testing-boards:beagleboneblack:automatic:
   dependencies:
     - init:workspace
     - build:acceptance:beagleboneblack
+    - trigger:integration-tests
   only:
     variables:
       - $PUBLISH_RELEASE_AUTOMATIC == "true" && $BUILD_BEAGLEBONEBLACK == "true"
@@ -150,6 +151,7 @@ release:mender-cli:automatic:
   dependencies:
     - init:workspace
     - build:mender-cli
+    - trigger:integration-tests
   extends: .template_release_binary_tools
 
 .template_release_mender-monitor:
@@ -160,9 +162,6 @@ release:mender-cli:automatic:
   variables:
     S3_BUCKET_NAME: "mender-binaries"
     S3_BUCKET_PATH: "mender-monitor/yocto"
-  dependencies:
-    - init:workspace
-    - build:mender-monitor:package
   before_script:
     # Install dependencies
     - apt update && apt install -yyq awscli git wget python3 python3-pip xz-utils
@@ -184,10 +183,17 @@ release:mender-cli:automatic:
     - aws s3 cp stage-artifacts/mender-monitor-*.tar.gz s3://$S3_BUCKET_NAME/$S3_BUCKET_PATH/$mender_monitor_version/
 
 release:mender-monitor:manual:
+  dependencies:
+    - init:workspace
+    - build:mender-monitor:package
   when: manual
   extends: .template_release_mender-monitor
 
 release:mender-monitor:automatic:
+  dependencies:
+    - init:workspace
+    - build:mender-monitor:package
+    - trigger:integration-tests
   only:
     variables:
       - $PUBLISH_RELEASE_AUTOMATIC == "true"
@@ -202,9 +208,6 @@ release:mender-monitor:automatic:
     S3_BUCKET_NAME: "mender-binaries"
     S3_BUCKET_PATH_YOCTO: "mender-gateway/yocto"
     S3_BUCKET_PATH_EXAMPLES: "mender-gateway/examples"
-  dependencies:
-    - init:workspace
-    - build:mender-gateway:package
   before_script:
     # Early exit when building an integration version without mender-gateway
     - apt update && apt install -yyq xz-utils
@@ -231,10 +234,17 @@ release:mender-monitor:automatic:
     - aws s3 cp stage-artifacts/mender-gateway-examples-*.tar s3://$S3_BUCKET_NAME/$S3_BUCKET_PATH_EXAMPLES/$mender_gateway_version/
 
 release:mender-gateway:manual:
+  dependencies:
+    - init:workspace
+    - build:mender-gateway:package
   when: manual
   extends: .template_release_mender-gateway
 
 release:mender-gateway:automatic:
+  dependencies:
+    - init:workspace
+    - build:mender-gateway:package
+    - trigger:integration-tests
   only:
     variables:
       - $PUBLISH_RELEASE_AUTOMATIC == "true"
@@ -247,10 +257,6 @@ release:mender-gateway:automatic:
   image: docker
   services:
     - docker:dind
-  dependencies:
-    - init:workspace
-    - build:client:qemu
-    - build:client:docker
   before_script:
     # Check correct dind setup
     - docker version
@@ -272,29 +278,40 @@ release:mender-gateway:automatic:
     - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" "$CI_REGISTRY" --password-stdin
   script:
     # Load, tag and push mender-client-* images
-    - for image in $($WORKSPACE/integration/extra/release_tool.py --list docker | egrep 'mender-client|mender-qemu|mender-monitor|mender-gateway-qemu-commercial'); do
-    #   Historical note,
-    #   The version here used to be determined by the release_tool, which could distinguish between
-    #   git repo version and container image version. Today this is kind of messy, as we moved out
-    #   the backend to a separate repo but we haven't done deep changes in the release process for
-    #   the rest, and "--version-of" option gives bogus results.
-    #   TLDR; hardcode to mender-${INTEGRATION_REV} version for the Virtual Devices images
-    -   version=mender-${INTEGRATION_REV}
-    -   docker_url=$($WORKSPACE/integration/extra/release_tool.py --map-name docker $image docker_url)
-    -   docker pull ${GITLAB_REGISTRY_PREFIX}-${image}
-    -   docker tag ${GITLAB_REGISTRY_PREFIX}-${image} ${docker_url}:${version}
-    -   docker push $docker_url:${version}
-    - done
+    # Historical note,
+    # The version here used to be determined by the release_tool, which could distinguish between
+    # git repo version and container image version. Today this is kind of messy, as we moved out
+    # the backend to a separate repo but we haven't done deep changes in the release process for
+    # the rest, and "--version-of" option gives bogus results.
+    # TLDR; hardcode to mender-${INTEGRATION_REV} version for the Virtual Devices images
+    - |
+      version=mender-${INTEGRATION_REV}
+      images="mender-client-docker mender-client-docker-addons mender-client-qemu mender-client-qemu-rofs mender-gateway-qemu-commercial mender-monitor-qemu-commercial mender-qemu-rofs-commercial"
+      for image in $images; do
+          docker pull ${GITLAB_REGISTRY_PREFIX}-${image}
+          tagged_image=docker.io/mendersoftware/${image}:${version}
+          docker tag ${GITLAB_REGISTRY_PREFIX}-${image} ${tagged_image}
+          docker push ${tagged_image}
+      done
 
 # This job allows mender repo to publish the related Docker client images on
 # merges to master or release branches.
 release:virtual-client:manual:
+  dependencies:
+    - init:workspace
+    - build:client:qemu
+    - build:client:docker
   when: manual
   extends: .template_release_docker_images:client-only
 
 # This job allows a release to publish only the client images. Specifically designed for Mender 3.7
 # series, to complement job release_docker_multiplatform_images:manual
 release:virtual-client:automatic:
+  dependencies:
+    - init:workspace
+    - build:client:qemu
+    - build:client:docker
+    - trigger:integration-tests
   only:
     variables:
       - $PUBLISH_DOCKER_CLIENT_IMAGES == "true"

gitlab-pipeline/stage/trigger-integration.yml

@@ -3,19 +3,59 @@
   inherit:
     variables: false
   variables:
-    MENDER_CLIENT_TAG: mender-$INTEGRATION_REV
-    MENDER_GATEWAY_QEMU_TAG: mender-$INTEGRATION_REV
-    MENDER_GATEWAY_TAG: $MENDER_GATEWAY_REV
+    MENDER_CLIENT_REGISTRY: registry.gitlab.com
+    MENDER_CLIENT_ENTERPRISE_REGISTRY: registry.gitlab.com
+    MENDER_CLIENT_REPOSITORY: northern.tech/mender
+
+    MENDER_CLIENT_DOCKER_IMAGE: mender-qa
+    MENDER_CLIENT_DOCKER_TAG: ${CI_PIPELINE_ID}-mender-client-docker
+
+    MENDER_CLIENT_DOCKER_ADDONS_IMAGE: mender-qa
+    MENDER_CLIENT_DOCKER_ADDONS_TAG: ${CI_PIPELINE_ID}-mender-client-docker-addons
+
+    MENDER_CLIENT_QEMU_IMAGE: mender-qa
+    MENDER_CLIENT_QEMU_TAG: ${CI_PIPELINE_ID}-mender-client-qemu
+
+    MENDER_CLIENT_QEMU_ROFS_IMAGE: mender-qa
+    MENDER_CLIENT_QEMU_ROFS_TAG: ${CI_PIPELINE_ID}-mender-client-qemu-rofs
+
+    MENDER_GATEWAY_QEMU_COMMERCIAL_IMAGE: mender-qa
+    MENDER_GATEWAY_QEMU_COMMERCIAL_TAG: ${CI_PIPELINE_ID}-mender-gateway-qemu-commercial
+
+    MENDER_MONITOR_QEMU_COMMERCIAL_IMAGE: mender-qa
+    MENDER_MONITOR_QEMU_COMMERCIAL_TAG: ${CI_PIPELINE_ID}-mender-monitor-qemu-commercial
+
+    MENDER_QEMU_ROFS_COMMERCIAL_IMAGE: mender-qa
+    MENDER_QEMU_ROFS_COMMERCIAL_TAG: ${CI_PIPELINE_ID}-mender-qemu-rofs-commercial
+
+    MENDER_GATEWAY_TAG: ${MENDER_GATEWAY_REV}
+
     RUN_TESTS_FULL_INTEGRATION: "true"
     # TODO: remove these parallel 1 overrides whenever tests are stable
     CI_JOBS_IN_PARALLEL_INTEGRATION: 1
     XDIST_JOBS_IN_PARALLEL_INTEGRATION: 1
+
+trigger:generate-gitlab-integration-rev:
+  stage: trigger:integration
+  script:
+  # Convert INTEGRATION_REV on format `pull/0000/head` to `pr_0000` to specify which gitlab branch to trigger
+  - |
+    GITLAB_INTEGRATION_REV=$INTEGRATION_REV
+    if [[ "$INTEGRATION_REV" =~ ^pull/([0-9]+)/head$ ]]; then
+      GITLAB_INTEGRATION_REV="pr_${BASH_REMATCH[1]}"
+    fi
+  - echo "GITLAB_INTEGRATION_REV=$GITLAB_INTEGRATION_REV" >> gitlab_integration_rev.env
+  artifacts:
+    reports:
+      dotenv: gitlab_integration_rev.env
+
+trigger:integration-tests:
+  extends: .template:trigger:integration-tests
+  needs:
+    - trigger:generate-gitlab-integration-rev
+    - build:client:qemu
+  when: always
   trigger:
     project: Northern.tech/Mender/integration
-    branch: $INTEGRATION_REV
+    branch: $GITLAB_INTEGRATION_REV
     strategy: depend
-
-trigger:integration-tests:manual:
-  needs: []
-  when: manual
-  extends: .template:trigger:integration-tests